@skill-harness/cli 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/decision.js +914 -287
- package/package.json +3 -3
package/dist/decision.js
CHANGED
|
@@ -1,20 +1,72 @@
|
|
|
1
1
|
// experiments/decision-shadow/main.mjs
|
|
2
|
-
import { createHash as
|
|
3
|
-
import { open as
|
|
4
|
-
import { resolve as
|
|
2
|
+
import { createHash as createHash7 } from "node:crypto";
|
|
3
|
+
import { open as open5, readFile as readFile2, stat } from "node:fs/promises";
|
|
4
|
+
import { resolve as resolve3, join as join7 } from "node:path";
|
|
5
|
+
import { openSync as openSync3, writeFileSync as writeFileSync3, fsyncSync as fsyncSync3, closeSync as closeSync3 } from "node:fs";
|
|
5
6
|
|
|
6
|
-
// experiments/decision-shadow/
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
7
|
+
// experiments/decision-shadow/data-root.mjs
|
|
8
|
+
import { closeSync, fsyncSync, lstatSync, mkdirSync, mkdtempSync, openSync, writeFileSync } from "node:fs";
|
|
9
|
+
import { createHash } from "node:crypto";
|
|
10
|
+
import { homedir } from "node:os";
|
|
11
|
+
import { isAbsolute, join, parse, relative, resolve, sep } from "node:path";
|
|
12
|
+
function resolveDataRoot({ env = process.env, home = homedir(), override } = {}) {
|
|
13
|
+
const value = override !== void 0 ? override : env.SKILL_HARNESS_DATA_ROOT !== void 0 ? env.SKILL_HARNESS_DATA_ROOT : join(home, ".skill-harness");
|
|
14
|
+
if (typeof value !== "string" || !value.trim() || /[\p{Cc}\p{Cf}]/u.test(value) || !isAbsolute(value))
|
|
15
|
+
throw new TypeError("Skill Harness data root must be a nonempty absolute path");
|
|
16
|
+
return resolve(value);
|
|
17
|
+
}
|
|
18
|
+
function ensurePrivateDirectory(path) {
|
|
19
|
+
const directory = resolveDataRoot({ override: path });
|
|
20
|
+
const anchor = parse(directory).root;
|
|
21
|
+
let current = anchor;
|
|
22
|
+
for (const part of relative(anchor, directory).split(sep).filter(Boolean)) {
|
|
23
|
+
current = join(current, part);
|
|
24
|
+
try {
|
|
25
|
+
mkdirSync(current, { mode: 448 });
|
|
26
|
+
} catch (error) {
|
|
27
|
+
if (error.code !== "EEXIST") throw error;
|
|
28
|
+
}
|
|
29
|
+
const info2 = lstatSync(current);
|
|
30
|
+
if (!info2.isDirectory() || info2.isSymbolicLink())
|
|
31
|
+
throw new Error("Skill Harness collection path must contain ordinary directories, not links");
|
|
32
|
+
}
|
|
33
|
+
const info = lstatSync(directory);
|
|
34
|
+
if (process.platform !== "win32" && ((info.mode & 63) !== 0 || process.getuid && info.uid !== process.getuid()))
|
|
35
|
+
throw new Error("Skill Harness collection directory must be owned by the current user with mode 0700");
|
|
36
|
+
return directory;
|
|
37
|
+
}
|
|
38
|
+
function writePrivateNew(path, bytes) {
|
|
39
|
+
const fd = openSync(path, "wx", 384);
|
|
40
|
+
try {
|
|
41
|
+
writeFileSync(fd, bytes);
|
|
42
|
+
fsyncSync(fd);
|
|
43
|
+
} finally {
|
|
44
|
+
closeSync(fd);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
function createSessionCollection(root, kind, sessionId) {
|
|
48
|
+
if (!["jev-workflow", "jev-manual"].includes(kind)) throw new TypeError("Unsupported collection kind");
|
|
49
|
+
if (typeof sessionId !== "string" || !sessionId.trim() || sessionId.length > 256 || /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/u.test(sessionId))
|
|
50
|
+
throw new TypeError("Collection requires an explicit valid session identity");
|
|
51
|
+
const base = ensurePrivateDirectory(root);
|
|
52
|
+
const family = ensurePrivateDirectory(join(base, kind));
|
|
53
|
+
const session = ensurePrivateDirectory(join(family, `session-${createHash("sha256").update(sessionId).digest("hex")}`));
|
|
54
|
+
return mkdtempSync(join(session, kind === "jev-workflow" ? "selection-" : "run-"));
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// experiments/decision-shadow/collections.mjs
|
|
58
|
+
import { constants as constants2 } from "node:fs";
|
|
59
|
+
import { open as open2, opendir } from "node:fs/promises";
|
|
60
|
+
import { join as join3, parse as parse2, relative as relative2, sep as sep2 } from "node:path";
|
|
9
61
|
|
|
10
62
|
// experiments/decision-shadow/learning-data.mjs
|
|
11
|
-
import { createHash as
|
|
63
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
12
64
|
import { constants } from "node:fs";
|
|
13
65
|
import { open, mkdir, writeFile } from "node:fs/promises";
|
|
14
|
-
import { isAbsolute, join, normalize } from "node:path";
|
|
66
|
+
import { isAbsolute as isAbsolute2, join as join2, normalize } from "node:path";
|
|
15
67
|
|
|
16
68
|
// experiments/decision-shadow/dataset.mjs
|
|
17
|
-
import { createHash } from "node:crypto";
|
|
69
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
18
70
|
var CASE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$/;
|
|
19
71
|
var SHA256 = /^[0-9a-f]{64}$/;
|
|
20
72
|
var MAX_CASES = 100;
|
|
@@ -55,7 +107,7 @@ function canonical(value) {
|
|
|
55
107
|
return JSON.stringify(value);
|
|
56
108
|
}
|
|
57
109
|
function hashCase(value) {
|
|
58
|
-
return
|
|
110
|
+
return createHash2("sha256").update(canonical(value), "utf8").digest("hex");
|
|
59
111
|
}
|
|
60
112
|
function parseCases(value) {
|
|
61
113
|
const root = plainObject(value, "dataset");
|
|
@@ -419,7 +471,7 @@ function trainingAssets() {
|
|
|
419
471
|
}
|
|
420
472
|
|
|
421
473
|
// experiments/decision-shadow/learning-data.mjs
|
|
422
|
-
var learningDigest = (value) =>
|
|
474
|
+
var learningDigest = (value) => createHash3("sha256").update(typeof value === "string" || Buffer.isBuffer(value) ? value : canonical2(value)).digest("hex");
|
|
423
475
|
function canonical2(value) {
|
|
424
476
|
return Array.isArray(value) ? `[${value.map(canonical2).join(",")}]` : value !== null && typeof value === "object" ? `{${Object.keys(value).sort().map((k) => `${JSON.stringify(k)}:${canonical2(value[k])}`).join(",")}}` : JSON.stringify(value);
|
|
425
477
|
}
|
|
@@ -448,7 +500,7 @@ function parsedCases(cases) {
|
|
|
448
500
|
check(Array.isArray(cases), "cases must be parsed cases");
|
|
449
501
|
const raw = cases.map((c) => {
|
|
450
502
|
keys(c, ["id", "input", "question", "provenance", "source", "visibility", "hash"], "parsed case");
|
|
451
|
-
const { hash:
|
|
503
|
+
const { hash: hash4, ...rest } = c;
|
|
452
504
|
return rest;
|
|
453
505
|
});
|
|
454
506
|
const validated = parseCases({ schema: 1, cases: raw });
|
|
@@ -520,10 +572,15 @@ function parseExperiment(cases, manifest) {
|
|
|
520
572
|
return { ...structuredClone(manifest), entries };
|
|
521
573
|
}
|
|
522
574
|
async function readExplicit(path, expected, limit = 1024 * 1024) {
|
|
575
|
+
sha2(expected);
|
|
576
|
+
const bytes = await readBoundedArtifact(path, limit);
|
|
577
|
+
check(learningDigest(bytes) === expected, "artifact hash mismatch");
|
|
578
|
+
return bytes;
|
|
579
|
+
}
|
|
580
|
+
async function readBoundedArtifact(path, limit = 1024 * 1024) {
|
|
523
581
|
text(path, "artifact path", 4096);
|
|
524
582
|
check(process.platform === "linux", "Evidence review requires Linux no-follow directory descriptors");
|
|
525
|
-
check(
|
|
526
|
-
sha2(expected);
|
|
583
|
+
check(isAbsolute2(path) && normalize(path) === path && !path.endsWith("/"), "artifact path must be canonical and absolute");
|
|
527
584
|
const components = path.split("/").filter(Boolean);
|
|
528
585
|
check(components.length <= 128, "artifact path too deep");
|
|
529
586
|
check(!components.some((p) => ["sessions", "native-sessions", "auth.json", ".env"].includes(p.toLowerCase())), "private session or credential paths are unsupported");
|
|
@@ -565,9 +622,7 @@ async function readExplicit(path, expected, limit = 1024 * 1024) {
|
|
|
565
622
|
await h.close();
|
|
566
623
|
}
|
|
567
624
|
}
|
|
568
|
-
|
|
569
|
-
check(learningDigest(bytes) === expected, "artifact hash mismatch");
|
|
570
|
-
return bytes;
|
|
625
|
+
return b.subarray(0, n);
|
|
571
626
|
} finally {
|
|
572
627
|
if (file) await file.close();
|
|
573
628
|
await Promise.allSettled(directories.map((d) => d.handle.close()));
|
|
@@ -606,6 +661,18 @@ async function reviewLabelEvidence({ cases, labels, references }) {
|
|
|
606
661
|
}
|
|
607
662
|
return { schema: 1, kind: "decision-label-evidence-review", reviewed, meaning: "Exact receipt bytes and declared provenance checked; human identity, truth, independence and rights remain explicit reviewer assertions." };
|
|
608
663
|
}
|
|
664
|
+
function redactSelectedInput(input, replacements) {
|
|
665
|
+
check(Array.isArray(replacements) && replacements.length <= 32, "invalid redactions");
|
|
666
|
+
let textValue = input;
|
|
667
|
+
for (const r of replacements) {
|
|
668
|
+
keys(r, ["from", "to"], "redaction");
|
|
669
|
+
text(r.from, "redaction source", 4096);
|
|
670
|
+
check(typeof r.to === "string" && /^<[A-Z0-9_ -]{1,80}>$/.test(r.to), "redaction must use a placeholder");
|
|
671
|
+
check(textValue.includes(r.from), "redaction source absent");
|
|
672
|
+
textValue = textValue.replaceAll(r.from, r.to);
|
|
673
|
+
}
|
|
674
|
+
return { input: textValue, replacements: replacements.map((r) => ({ fromSha256: learningDigest(r.from), fromCodepoints: Array.from(r.from).length, to: r.to })) };
|
|
675
|
+
}
|
|
609
676
|
async function importSelectedCase({ caseDocument, selection, sessionConsent, experimentEntry }) {
|
|
610
677
|
const cases = parseCases(caseDocument);
|
|
611
678
|
check(cases.length === 1 && cases[0].provenance === "observed", "import requires exactly one observed case");
|
|
@@ -643,16 +710,9 @@ async function importSelectedCase({ caseDocument, selection, sessionConsent, exp
|
|
|
643
710
|
const points = Array.from(s);
|
|
644
711
|
check(Number.isSafeInteger(f.start) && Number.isSafeInteger(f.end) && f.start >= 0 && f.end > f.start && f.end <= points.length, "invalid excerpt range");
|
|
645
712
|
let excerpt = points.slice(f.start, f.end).join("");
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
text(r.from, "redaction source", 4096);
|
|
650
|
-
check(typeof r.to === "string" && /^<[A-Z0-9_ -]{1,80}>$/.test(r.to), "redaction must use a placeholder");
|
|
651
|
-
check(excerpt.includes(r.from), "redaction source absent");
|
|
652
|
-
excerpt = excerpt.replaceAll(r.from, r.to);
|
|
653
|
-
}
|
|
654
|
-
parts.push(excerpt);
|
|
655
|
-
refs.push({ ...f, replacements: f.replacements.map((r) => ({ fromSha256: learningDigest(r.from), fromCodepoints: Array.from(r.from).length, to: r.to })) });
|
|
713
|
+
const redacted = redactSelectedInput(excerpt, f.replacements);
|
|
714
|
+
parts.push(redacted.input);
|
|
715
|
+
refs.push({ ...f, replacements: redacted.replacements });
|
|
656
716
|
}
|
|
657
717
|
check(parts.join("\n\n") === c.input, "case input must equal exact selected redacted fragments joined by two newlines");
|
|
658
718
|
return { caseDocument: structuredClone(caseDocument), experimentEntry: e, receipt: { schema: 1, kind: "decision-selected-public-import", caseId: c.id, caseHash: c.hash, sessionId: e.sessionId, consentHash: learningDigest(consent), source: structuredClone(selection.source), fragments: refs, trainingEligible: false, automaticCollection: false } };
|
|
@@ -703,7 +763,7 @@ async function writePreparedExport({ prepared, directory }) {
|
|
|
703
763
|
keys(prepared, ["schema", "kind", "metadata", "files"], "prepared export");
|
|
704
764
|
check(prepared.schema === 2 && prepared.metadata?.schema === 2 && prepared.kind === "prepared-decision-learning-export", "unsupported prepared export");
|
|
705
765
|
text(directory, "output directory", 4096);
|
|
706
|
-
check(
|
|
766
|
+
check(isAbsolute2(directory), "output directory must be absolute");
|
|
707
767
|
const allowed = ["train.jsonl", "validation.jsonl", "test.jsonl", "export-manifest.json", "train-lora.py", "training-config.example.json", "requirements-training.txt", "TRAINING.md"];
|
|
708
768
|
check(Object.keys(prepared.files).sort().join("|") === [...allowed].sort().join("|"), "unexpected export files");
|
|
709
769
|
check(prepared.files["export-manifest.json"] === JSON.stringify(prepared.metadata, null, 2) + "\n", "export metadata mismatch");
|
|
@@ -712,10 +772,497 @@ async function writePreparedExport({ prepared, directory }) {
|
|
|
712
772
|
if (name !== "export-manifest.json") check(prepared.metadata.files[name]?.sha256 === learningDigest(s) && prepared.metadata.files[name]?.bytes === Buffer.byteLength(s), "prepared export bytes changed");
|
|
713
773
|
}
|
|
714
774
|
await mkdir(directory, { mode: 448 });
|
|
715
|
-
for (const [name, s] of Object.entries(prepared.files)) await writeFile(
|
|
775
|
+
for (const [name, s] of Object.entries(prepared.files)) await writeFile(join2(directory, name), s, { encoding: "utf8", flag: "wx", mode: 384 });
|
|
716
776
|
return { directory, trainingEligible: prepared.metadata.trainingEligible, files: Object.fromEntries(Object.entries(prepared.files).map(([name, s]) => [name, { sha256: learningDigest(s), bytes: Buffer.byteLength(s) }])) };
|
|
717
777
|
}
|
|
718
778
|
|
|
779
|
+
// experiments/decision-shadow/collections.mjs
|
|
780
|
+
var DIRECTORY = constants2.O_RDONLY | constants2.O_DIRECTORY | constants2.O_NOFOLLOW;
|
|
781
|
+
var LIMITS = Object.freeze({ entries: 1e4, directories: 1e4, bytes: 32 * 1024 * 1024 });
|
|
782
|
+
var SESSION = /^session-([a-f0-9]{64})$/;
|
|
783
|
+
var SELECTION = /^selection-[A-Za-z0-9]{6}$/;
|
|
784
|
+
var RUN = /^run-[A-Za-z0-9]{6}$/;
|
|
785
|
+
var IDENTIFIER = /^[A-Za-z0-9~@][A-Za-z0-9~._:/@+-]{0,255}$/;
|
|
786
|
+
var object = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
|
|
787
|
+
var sessionIdentity = (value) => typeof value === "string" && value.trim() && value.length <= 256 && !/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/u.test(value);
|
|
788
|
+
async function openRoot(root) {
|
|
789
|
+
const anchor = parse2(root).root;
|
|
790
|
+
let handle = await open2(anchor, DIRECTORY);
|
|
791
|
+
try {
|
|
792
|
+
for (const name of relative2(anchor, root).split(sep2).filter(Boolean)) {
|
|
793
|
+
const next = await open2(`/proc/self/fd/${handle.fd}/${name}`, DIRECTORY);
|
|
794
|
+
await handle.close();
|
|
795
|
+
handle = next;
|
|
796
|
+
}
|
|
797
|
+
return handle;
|
|
798
|
+
} catch (error) {
|
|
799
|
+
await handle.close();
|
|
800
|
+
throw error;
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
async function listCollections(value) {
|
|
804
|
+
const root = resolveDataRoot({ override: value });
|
|
805
|
+
if (process.platform !== "linux") throw Error("Collection inventory requires Linux no-follow directory descriptors");
|
|
806
|
+
const result = {
|
|
807
|
+
schema: 1,
|
|
808
|
+
kind: "decision-collection-inventory",
|
|
809
|
+
meaning: "inventory-only-not-validation",
|
|
810
|
+
collections: [],
|
|
811
|
+
count: 0,
|
|
812
|
+
gaps: [],
|
|
813
|
+
truncated: false,
|
|
814
|
+
limits: { ...LIMITS },
|
|
815
|
+
trainingEligible: false,
|
|
816
|
+
exportEligible: false
|
|
817
|
+
};
|
|
818
|
+
let entries = 0, directories = 0, bytesRead = 0;
|
|
819
|
+
const gap = (path, reason) => result.gaps.push({ path, reason });
|
|
820
|
+
const limit = (path) => {
|
|
821
|
+
if (!result.truncated) gap(path, "inventory-limit-reached");
|
|
822
|
+
result.truncated = true;
|
|
823
|
+
};
|
|
824
|
+
const directoryFailure = (path, error) => gap(path, error.code === "ENOENT" ? "directory-missing" : "linked-or-unreadable-directory");
|
|
825
|
+
async function visit(parent, name, path, fn, optional = false) {
|
|
826
|
+
if (result.truncated) return;
|
|
827
|
+
if (directories >= LIMITS.directories) {
|
|
828
|
+
limit(path);
|
|
829
|
+
return;
|
|
830
|
+
}
|
|
831
|
+
let handle2;
|
|
832
|
+
try {
|
|
833
|
+
handle2 = await open2(`/proc/self/fd/${parent.fd}/${name}`, DIRECTORY);
|
|
834
|
+
directories++;
|
|
835
|
+
} catch (error) {
|
|
836
|
+
if (!optional || error.code !== "ENOENT") directoryFailure(path, error);
|
|
837
|
+
return;
|
|
838
|
+
}
|
|
839
|
+
try {
|
|
840
|
+
await fn(handle2);
|
|
841
|
+
} finally {
|
|
842
|
+
await handle2.close();
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
async function each(handle2, path, fn) {
|
|
846
|
+
let directory;
|
|
847
|
+
try {
|
|
848
|
+
directory = await opendir(`/proc/self/fd/${handle2.fd}`);
|
|
849
|
+
} catch {
|
|
850
|
+
gap(path, "unreadable-directory");
|
|
851
|
+
return;
|
|
852
|
+
}
|
|
853
|
+
try {
|
|
854
|
+
for await (const entry of directory) {
|
|
855
|
+
if (entries >= LIMITS.entries) {
|
|
856
|
+
limit(path);
|
|
857
|
+
break;
|
|
858
|
+
}
|
|
859
|
+
entries++;
|
|
860
|
+
await fn(entry);
|
|
861
|
+
if (result.truncated) break;
|
|
862
|
+
}
|
|
863
|
+
} catch {
|
|
864
|
+
gap(path, "directory-changed-or-unreadable");
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
async function selection(path, kind, expectedSession) {
|
|
868
|
+
const file = kind === "jev-workflow" ? "selection.json" : "selected-data.json";
|
|
869
|
+
const relativePath = `${path}/${file}`;
|
|
870
|
+
const maximum = kind === "jev-workflow" ? 1024 * 1024 : 4 * 1024 * 1024;
|
|
871
|
+
if (LIMITS.bytes - bytesRead < maximum) {
|
|
872
|
+
limit(path);
|
|
873
|
+
return;
|
|
874
|
+
}
|
|
875
|
+
let bytes;
|
|
876
|
+
try {
|
|
877
|
+
bytes = await readBoundedArtifact(join3(root, relativePath), maximum);
|
|
878
|
+
} catch (error) {
|
|
879
|
+
gap(relativePath, error.code === "ENOENT" ? "selection-missing" : "linked-oversized-or-unreadable-selection");
|
|
880
|
+
return;
|
|
881
|
+
}
|
|
882
|
+
bytesRead += bytes.length;
|
|
883
|
+
let record;
|
|
884
|
+
try {
|
|
885
|
+
record = JSON.parse(bytes.toString("utf8"));
|
|
886
|
+
} catch {
|
|
887
|
+
gap(relativePath, "invalid-selection-json");
|
|
888
|
+
return;
|
|
889
|
+
}
|
|
890
|
+
const workflow = kind === "jev-workflow";
|
|
891
|
+
const recognized = object(record) && (workflow ? record.schema === 2 && record.kind === "skill-harness-selected-decision-v2" || record.schema === 3 && record.kind === "skill-harness-selected-decision-v3" : record.schema === 1 && record.kind === "decision-session-selected-data");
|
|
892
|
+
if (!recognized || !sessionIdentity(record.sessionId) || record.trainingEligible !== false || (workflow ? record.exportEligible !== false : !Array.isArray(record.cases))) {
|
|
893
|
+
gap(relativePath, "unrecognized-or-incomplete-selection");
|
|
894
|
+
return;
|
|
895
|
+
}
|
|
896
|
+
const sessionSha256 = learningDigest(record.sessionId);
|
|
897
|
+
if (expectedSession && expectedSession !== sessionSha256) {
|
|
898
|
+
gap(relativePath, "session-partition-mismatch");
|
|
899
|
+
return;
|
|
900
|
+
}
|
|
901
|
+
let provider = null, model = null;
|
|
902
|
+
if (workflow) {
|
|
903
|
+
provider = typeof record.provider === "string" && IDENTIFIER.test(record.provider) ? record.provider : null;
|
|
904
|
+
model = typeof record.model === "string" && IDENTIFIER.test(record.model) ? record.model : null;
|
|
905
|
+
if (provider === null || model === null) gap(relativePath, "invalid-provider-model-metadata");
|
|
906
|
+
}
|
|
907
|
+
result.collections.push({
|
|
908
|
+
path: relativePath,
|
|
909
|
+
sha256: learningDigest(bytes),
|
|
910
|
+
bytes: bytes.length,
|
|
911
|
+
sessionSha256,
|
|
912
|
+
recordType: record.kind,
|
|
913
|
+
recordSchema: record.schema,
|
|
914
|
+
caseCount: workflow ? 1 : record.cases.length,
|
|
915
|
+
provider,
|
|
916
|
+
model,
|
|
917
|
+
trainingEligible: false,
|
|
918
|
+
exportEligible: false
|
|
919
|
+
});
|
|
920
|
+
}
|
|
921
|
+
async function family(handle2, path, kind) {
|
|
922
|
+
await each(handle2, path, async (entry) => {
|
|
923
|
+
const session = SESSION.exec(entry.name);
|
|
924
|
+
const legacy = kind === "jev-workflow" && SELECTION.test(entry.name);
|
|
925
|
+
if (session) {
|
|
926
|
+
const sessionPath = `${path}/${entry.name}`;
|
|
927
|
+
await visit(handle2, entry.name, sessionPath, async (sessionHandle) => {
|
|
928
|
+
await each(sessionHandle, sessionPath, async (child) => {
|
|
929
|
+
const matches = (kind === "jev-workflow" ? SELECTION : RUN).test(child.name);
|
|
930
|
+
if (!matches) {
|
|
931
|
+
if (child.isSymbolicLink() || /^(selection|run)-/.test(child.name)) gap(sessionPath, "unrecognized-collection-entry");
|
|
932
|
+
return;
|
|
933
|
+
}
|
|
934
|
+
const childPath = `${sessionPath}/${child.name}`;
|
|
935
|
+
await visit(sessionHandle, child.name, childPath, () => selection(childPath, kind, session[1]));
|
|
936
|
+
});
|
|
937
|
+
});
|
|
938
|
+
} else if (legacy) {
|
|
939
|
+
const childPath = `${path}/${entry.name}`;
|
|
940
|
+
await visit(handle2, entry.name, childPath, () => selection(childPath, kind, null));
|
|
941
|
+
} else if (entry.isSymbolicLink() || /^(session|selection|run)-/.test(entry.name)) {
|
|
942
|
+
gap(path, "unrecognized-collection-entry");
|
|
943
|
+
}
|
|
944
|
+
});
|
|
945
|
+
}
|
|
946
|
+
let handle;
|
|
947
|
+
try {
|
|
948
|
+
handle = await openRoot(root);
|
|
949
|
+
} catch (error) {
|
|
950
|
+
directoryFailure(".", error);
|
|
951
|
+
return result;
|
|
952
|
+
}
|
|
953
|
+
try {
|
|
954
|
+
for (const kind of ["jev-workflow", "jev-manual"])
|
|
955
|
+
await visit(handle, kind, kind, (child) => family(child, kind, kind), true);
|
|
956
|
+
} finally {
|
|
957
|
+
await handle.close();
|
|
958
|
+
}
|
|
959
|
+
result.collections.sort((a, b) => a.path.localeCompare(b.path));
|
|
960
|
+
result.gaps.sort((a, b) => a.path.localeCompare(b.path) || a.reason.localeCompare(b.reason));
|
|
961
|
+
result.count = result.collections.length;
|
|
962
|
+
return result;
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
// experiments/decision-shadow/research-commands.mjs
|
|
966
|
+
import { mkdir as mkdir2, writeFile as writeFile2 } from "node:fs/promises";
|
|
967
|
+
import { join as join5, resolve as resolve2 } from "node:path";
|
|
968
|
+
|
|
969
|
+
// experiments/decision-shadow/workflow-import.mjs
|
|
970
|
+
import { dirname, join as join4, isAbsolute as isAbsolute3, normalize as normalize2 } from "node:path";
|
|
971
|
+
import { isDeepStrictEqual } from "node:util";
|
|
972
|
+
|
|
973
|
+
// experiments/decision-shadow/native-jev.mjs
|
|
974
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
975
|
+
var APIS = /* @__PURE__ */ new Set(["typesafe-system-one", "cloudflare-workers-ai-system-one"]);
|
|
976
|
+
var IDENTIFIER2 = /^[A-Za-z0-9@~][A-Za-z0-9@~._:/-]{0,255}$/;
|
|
977
|
+
var JEV = /(?:^|[/:])jev(?:-latest|-[0-9][A-Za-z0-9._-]*)?$/;
|
|
978
|
+
var keys2 = (value) => Object.keys(value).sort().join(",");
|
|
979
|
+
var isRecord = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
|
|
980
|
+
var identifier = (value) => typeof value === "string" && IDENTIFIER2.test(value);
|
|
981
|
+
var sha256 = (value) => createHash4("sha256").update(value).digest("hex");
|
|
982
|
+
function check2(condition, message) {
|
|
983
|
+
if (!condition) throw new TypeError(message);
|
|
984
|
+
}
|
|
985
|
+
function routeIdentity(value) {
|
|
986
|
+
return identifier(value.provider) && value.provider.toLowerCase() !== "openai" && identifier(value.model) && JEV.test(value.model.toLowerCase()) && APIS.has(value.api);
|
|
987
|
+
}
|
|
988
|
+
function validateNativeRoute(route) {
|
|
989
|
+
check2(isRecord(route) && keys2(route) === "api,model,modelSha256,provider,transport", "Invalid native JEV route fields");
|
|
990
|
+
check2(route.transport === "pi-classifier" && routeIdentity(route) && typeof route.modelSha256 === "string" && /^[a-f0-9]{64}$/.test(route.modelSha256), "Invalid native JEV route");
|
|
991
|
+
return Object.freeze({ ...route });
|
|
992
|
+
}
|
|
993
|
+
function text2(value, maximum, name) {
|
|
994
|
+
check2(typeof value === "string" && value.trim().length > 0 && Array.from(value).length <= maximum && !/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/u.test(value), `Invalid native JEV ${name}`);
|
|
995
|
+
}
|
|
996
|
+
function makeNativeRequest({ input, question }) {
|
|
997
|
+
text2(input, 16e3, "input");
|
|
998
|
+
text2(question, 8192, "question");
|
|
999
|
+
return {
|
|
1000
|
+
state: { input },
|
|
1001
|
+
questions: { decision: {
|
|
1002
|
+
type: "bool",
|
|
1003
|
+
instructions: question,
|
|
1004
|
+
criteria: { true: "The answer to the question is yes.", false: "The answer to the question is no." }
|
|
1005
|
+
} }
|
|
1006
|
+
};
|
|
1007
|
+
}
|
|
1008
|
+
function nativeRequestSha256(example) {
|
|
1009
|
+
return sha256(JSON.stringify(makeNativeRequest(example)));
|
|
1010
|
+
}
|
|
1011
|
+
function metric(value, integer2 = false) {
|
|
1012
|
+
if (value === void 0) return null;
|
|
1013
|
+
check2(typeof value === "number" && Number.isFinite(value) && value >= 0 && (!integer2 || Number.isSafeInteger(value)), "Invalid native usage");
|
|
1014
|
+
return value;
|
|
1015
|
+
}
|
|
1016
|
+
function usageOf(value) {
|
|
1017
|
+
if (value === void 0) return { inputTokens: null, outputTokens: null, costUsd: null };
|
|
1018
|
+
check2(isRecord(value), "Invalid native usage");
|
|
1019
|
+
for (const key of ["cacheRead", "cacheWrite", "totalTokens"]) metric(value[key], true);
|
|
1020
|
+
if (value.cost !== void 0) {
|
|
1021
|
+
check2(isRecord(value.cost), "Invalid native usage");
|
|
1022
|
+
for (const key of ["input", "output", "cacheRead", "cacheWrite"]) metric(value.cost[key]);
|
|
1023
|
+
}
|
|
1024
|
+
return { inputTokens: metric(value.input, true), outputTokens: metric(value.output, true), costUsd: metric(value.cost?.total) };
|
|
1025
|
+
}
|
|
1026
|
+
function normalizeNativeResult(route, result, latencyMs) {
|
|
1027
|
+
route = validateNativeRoute(route);
|
|
1028
|
+
const output = {
|
|
1029
|
+
status: "error",
|
|
1030
|
+
probability: null,
|
|
1031
|
+
resolvedModel: null,
|
|
1032
|
+
returnedProvider: null,
|
|
1033
|
+
returnedModel: null,
|
|
1034
|
+
usage: { inputTokens: null, outputTokens: null, costUsd: null },
|
|
1035
|
+
costSource: null,
|
|
1036
|
+
latencyMs: typeof latencyMs === "number" && Number.isFinite(latencyMs) && latencyMs >= 0 ? latencyMs : null,
|
|
1037
|
+
error: "invalid provider response"
|
|
1038
|
+
};
|
|
1039
|
+
if (!isRecord(result) || output.latencyMs === null || result.provider !== route.provider || result.model !== route.model || result.api !== route.api) return output;
|
|
1040
|
+
output.returnedProvider = result.provider;
|
|
1041
|
+
output.returnedModel = result.model;
|
|
1042
|
+
if (result.stopReason === "error" || result.stopReason === "aborted")
|
|
1043
|
+
return { ...output, error: result.stopReason === "aborted" ? "provider request aborted" : "provider request failed" };
|
|
1044
|
+
try {
|
|
1045
|
+
check2(result.stopReason === "stop" && isRecord(result.answers) && keys2(result.answers) === "decision", "Invalid answer");
|
|
1046
|
+
const answer = result.answers.decision;
|
|
1047
|
+
check2(isRecord(answer) && answer.type === "bool" && typeof answer.probability === "number" && Number.isFinite(answer.probability) && answer.probability >= 0 && answer.probability <= 1, "Invalid answer");
|
|
1048
|
+
const usage = usageOf(result.usage);
|
|
1049
|
+
return {
|
|
1050
|
+
...output,
|
|
1051
|
+
status: "answered",
|
|
1052
|
+
probability: answer.probability,
|
|
1053
|
+
usage,
|
|
1054
|
+
costSource: usage.costUsd === null ? null : "pi-catalog-estimate",
|
|
1055
|
+
error: null
|
|
1056
|
+
};
|
|
1057
|
+
} catch {
|
|
1058
|
+
return output;
|
|
1059
|
+
}
|
|
1060
|
+
}
|
|
1061
|
+
function validateNativePrediction(route, result) {
|
|
1062
|
+
route = validateNativeRoute(route);
|
|
1063
|
+
try {
|
|
1064
|
+
check2(isRecord(result) && keys2(result) === "costSource,error,latencyMs,probability,resolvedModel,returnedModel,returnedProvider,status,usage", "Invalid native prediction fields");
|
|
1065
|
+
check2(result.resolvedModel === null && (result.returnedProvider === null && result.returnedModel === null || result.returnedProvider === route.provider && result.returnedModel === route.model), "Invalid native prediction identity");
|
|
1066
|
+
check2(isRecord(result.usage) && keys2(result.usage) === "costUsd,inputTokens,outputTokens", "Invalid native prediction usage");
|
|
1067
|
+
for (const key of ["inputTokens", "outputTokens", "costUsd"])
|
|
1068
|
+
if (result.usage[key] !== null) {
|
|
1069
|
+
check2(result.usage[key] !== void 0, "Missing native usage field");
|
|
1070
|
+
metric(result.usage[key], key !== "costUsd");
|
|
1071
|
+
}
|
|
1072
|
+
if (result.latencyMs !== null) {
|
|
1073
|
+
check2(result.latencyMs !== void 0, "Missing native latency");
|
|
1074
|
+
metric(result.latencyMs);
|
|
1075
|
+
}
|
|
1076
|
+
check2(result.costSource === (result.usage.costUsd === null ? null : "pi-catalog-estimate"), "Invalid native cost provenance");
|
|
1077
|
+
if (result.status === "answered") {
|
|
1078
|
+
check2(result.error === null && result.returnedProvider === route.provider && result.returnedModel === route.model && result.latencyMs !== null && typeof result.probability === "number" && Number.isFinite(result.probability) && result.probability >= 0 && result.probability <= 1, "Invalid native answer");
|
|
1079
|
+
} else {
|
|
1080
|
+
check2(result.status === "error" && result.probability === null && ["invalid provider response", "provider request failed", "provider request aborted"].includes(result.error) && Object.values(result.usage).every((value) => value === null), "Invalid native error");
|
|
1081
|
+
}
|
|
1082
|
+
return true;
|
|
1083
|
+
} catch {
|
|
1084
|
+
return false;
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
// experiments/decision-shadow/workflow-import.mjs
|
|
1089
|
+
var QUESTION = "Given the stated stage, unresolved engineering uncertainty, requirements and selected evidence, is the proposed next action justified? Assess that action only, not final acceptance or whether mandatory later review is complete.";
|
|
1090
|
+
var SOURCE_KEYS = ["schema", "kind", "sessionId", "toolCallId", "frozenAt", "input", "inputSha256", "question", "provider", "model", "consent", "authorization", "provenance", "candidateIdentity", "candidateObservation", "sourceBinding", "evidenceRefs", "evidenceClaims", "inputArtifact", "redaction", "rights", "labelStatus", "trainingEligible", "exportEligible", "publicCaptureVerified"];
|
|
1091
|
+
var ENGINEERING_KEYS = ["schema", "kind", "source", "sessionId", "toolCallId", "candidate", "recordedAt", "evidenceRefs", "candidateIdentity", "candidateObservation", "sourceBinding", "independence", "trainingEligible", "exportEligible", "labelStatus"];
|
|
1092
|
+
var LIMIT = 2 * 1024 * 1024;
|
|
1093
|
+
function check3(ok, message) {
|
|
1094
|
+
if (!ok) throw new TypeError(message);
|
|
1095
|
+
}
|
|
1096
|
+
function keys3(value, names, name) {
|
|
1097
|
+
check3(value && typeof value === "object" && !Array.isArray(value) && isDeepStrictEqual(Object.keys(value).sort(), [...names].sort()), name + ": unsupported or missing fields");
|
|
1098
|
+
}
|
|
1099
|
+
function text3(value, name, max = 256) {
|
|
1100
|
+
check3(typeof value === "string" && value.trim().length > 0 && value.length <= max && !value.includes("\0") && !/[\uD800-\uDFFF]/u.test(value), name + ": invalid text");
|
|
1101
|
+
}
|
|
1102
|
+
function hash(value) {
|
|
1103
|
+
check3(typeof value === "string" && /^[a-f0-9]{64}$/.test(value), "invalid SHA-256");
|
|
1104
|
+
}
|
|
1105
|
+
function date2(value) {
|
|
1106
|
+
check3(typeof value === "string" && /^\d{4}-\d\d-\d\dT/.test(value) && Number.isFinite(Date.parse(value)), "invalid timestamp");
|
|
1107
|
+
return Date.parse(value);
|
|
1108
|
+
}
|
|
1109
|
+
function json2(bytes) {
|
|
1110
|
+
return JSON.parse(new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes));
|
|
1111
|
+
}
|
|
1112
|
+
function reference(ref) {
|
|
1113
|
+
keys3(ref, ["path", "sha256"], "selected reference");
|
|
1114
|
+
text3(ref.path, "artifact path", 4096);
|
|
1115
|
+
hash(ref.sha256);
|
|
1116
|
+
}
|
|
1117
|
+
function unlabeled(record) {
|
|
1118
|
+
check3(record.trainingEligible === false && record.exportEligible === false && record.labelStatus === "unlabeled", "retained source must remain unlabeled and ineligible");
|
|
1119
|
+
}
|
|
1120
|
+
function observed(record, sessionId, candidate) {
|
|
1121
|
+
if (record.candidateIdentity === "caller-claimed") {
|
|
1122
|
+
check3(record.candidateObservation === null, "caller-claimed candidate cannot include an observation");
|
|
1123
|
+
return;
|
|
1124
|
+
}
|
|
1125
|
+
check3(record.candidateIdentity === "principal-runtime-observed", "unsupported candidate provenance");
|
|
1126
|
+
const o = record.candidateObservation;
|
|
1127
|
+
keys3(o, ["requestId", "sessionId", "observedAt", "candidate"], "candidate observation");
|
|
1128
|
+
text3(o.requestId, "observation ID");
|
|
1129
|
+
date2(o.observedAt);
|
|
1130
|
+
check3(o.sessionId === sessionId && o.candidate && typeof o.candidate === "object" && !Array.isArray(o.candidate) && o.candidate.algorithm === "principal-candidate-v1" && o.candidate.id === candidate, "candidate observation identity mismatch");
|
|
1131
|
+
text3(o.candidate.root, "candidate root", 4096);
|
|
1132
|
+
check3(isAbsolute3(o.candidate.root) && normalize2(o.candidate.root) === o.candidate.root, "candidate root must be canonical and absolute");
|
|
1133
|
+
}
|
|
1134
|
+
function evidenceRefs(refs, retained = false) {
|
|
1135
|
+
check3(Array.isArray(refs) && refs.length <= 8, "evidence must select at most 8 files");
|
|
1136
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1137
|
+
return refs.map((ref) => {
|
|
1138
|
+
keys3(ref, retained ? ["path", "sha256", "bytes", "retainedPath"] : ["path", "sha256", "bytes"], "evidence reference");
|
|
1139
|
+
text3(ref.path, "original evidence path", 4096);
|
|
1140
|
+
hash(ref.sha256);
|
|
1141
|
+
check3(isAbsolute3(ref.path) && normalize2(ref.path) === ref.path && !seen.has(ref.path), "invalid or duplicate evidence path");
|
|
1142
|
+
seen.add(ref.path);
|
|
1143
|
+
check3(Number.isSafeInteger(ref.bytes) && ref.bytes >= 0 && ref.bytes <= LIMIT, "invalid evidence size");
|
|
1144
|
+
return { path: ref.path, sha256: ref.sha256, bytes: ref.bytes };
|
|
1145
|
+
});
|
|
1146
|
+
}
|
|
1147
|
+
async function snapshot(directory, ref, expectedName) {
|
|
1148
|
+
check3(ref.retainedPath === expectedName, "unexpected retained evidence path");
|
|
1149
|
+
const bytes = await readExplicit(join4(directory, expectedName), ref.sha256, LIMIT);
|
|
1150
|
+
check3(bytes.length === ref.bytes, "retained evidence size mismatch");
|
|
1151
|
+
return { sha256: ref.sha256, bytes: ref.bytes };
|
|
1152
|
+
}
|
|
1153
|
+
function validateSource(source) {
|
|
1154
|
+
const native = source?.schema === 3 && source.kind === "skill-harness-selected-decision-v3";
|
|
1155
|
+
check3(native || source?.schema === 2 && source.kind === "skill-harness-selected-decision-v2", "unsupported workflow selection contract");
|
|
1156
|
+
keys3(source, native ? [...SOURCE_KEYS, "route", "requestSha256"] : SOURCE_KEYS, "workflow selection");
|
|
1157
|
+
text3(source.sessionId, "session ID");
|
|
1158
|
+
text3(source.toolCallId, "tool-call ID");
|
|
1159
|
+
date2(source.frozenAt);
|
|
1160
|
+
text3(source.input, "decision input", 32e3);
|
|
1161
|
+
hash(source.inputSha256);
|
|
1162
|
+
check3(Array.from(source.input).length <= 16e3 && learningDigest(source.input) === source.inputSha256, "decision input digest or size mismatch");
|
|
1163
|
+
check3(source.question === QUESTION, "unsupported workflow question");
|
|
1164
|
+
const route = native ? validateNativeRoute(source.route) : null;
|
|
1165
|
+
check3(route ? source.provider === route.provider && source.model === route.model : source.provider === "jev" && source.model === "typesafe/jev-1.13", "unsupported workflow provider");
|
|
1166
|
+
if (native) check3(source.requestSha256 === nativeRequestSha256(source), "native request digest mismatch");
|
|
1167
|
+
const packet = JSON.parse(source.input);
|
|
1168
|
+
keys3(packet, ["candidate", "stage", "nextAction", "uncertainty", "requirements", "evidence"], "decision packet");
|
|
1169
|
+
for (const key of Object.keys(packet)) text3(packet[key], key, 32e3);
|
|
1170
|
+
check3(["design", "implementation", "verification"].includes(packet.stage), "unsupported decision stage");
|
|
1171
|
+
const { candidate, stage, nextAction, uncertainty, requirements, evidence } = packet;
|
|
1172
|
+
check3(JSON.stringify({ candidate, stage, nextAction, uncertainty, requirements, evidence }) === source.input, "decision input is not the exact producer packet");
|
|
1173
|
+
const consent = parseStorageConsent(source.consent, source.sessionId);
|
|
1174
|
+
check3(consent.decision === "granted" && date2(consent.recordedAt) <= date2(source.frozenAt), "recorded session storage consent is missing or postdates selection");
|
|
1175
|
+
const a = source.authorization;
|
|
1176
|
+
keys3(a, ["kind", "interactionId", "sessionId", "recordedAt", ...native ? ["route"] : ["provider", "model"], "question", "maximumCalls"], "workflow authorization");
|
|
1177
|
+
text3(a.interactionId, "authorization interaction");
|
|
1178
|
+
const authorizationMatches = native ? a.kind === "jev-workflow-paid-scope-v2" && isDeepStrictEqual(validateNativeRoute(a.route), route) : a.kind === "jev-workflow-paid-scope" && a.provider === source.provider && a.model === source.model;
|
|
1179
|
+
check3(authorizationMatches && a.sessionId === source.sessionId && a.question === QUESTION && a.maximumCalls === 3 && date2(a.recordedAt) <= date2(source.frozenAt), "workflow authorization mismatch");
|
|
1180
|
+
check3(source.provenance === "tool-selected-input" && source.evidenceClaims === "unassessed" && source.redaction === "unassessed" && source.rights === "unassessed" && source.publicCaptureVerified === false, "unsupported workflow provenance claims");
|
|
1181
|
+
unlabeled(source);
|
|
1182
|
+
const refs = evidenceRefs(source.evidenceRefs);
|
|
1183
|
+
check3(source.sourceBinding === (refs.length ? "local-reference-digests-verified" : "unassessed"), "source binding mismatch");
|
|
1184
|
+
keys3(source.inputArtifact, ["path", "sha256", "encoding"], "input artifact");
|
|
1185
|
+
check3(source.inputArtifact.path === "input.txt" && source.inputArtifact.sha256 === source.inputSha256 && source.inputArtifact.encoding === "utf8", "unsupported input artifact");
|
|
1186
|
+
observed(source, source.sessionId, candidate);
|
|
1187
|
+
return { packet, consent, refs };
|
|
1188
|
+
}
|
|
1189
|
+
async function importWorkflowCase({ caseDocument, selection, sessionConsent, experimentEntry }) {
|
|
1190
|
+
const cases = parseCases(caseDocument);
|
|
1191
|
+
check3(cases.length === 1 && cases[0].provenance === "observed", "workflow import requires exactly one observed case");
|
|
1192
|
+
const c = cases[0];
|
|
1193
|
+
const entry = parseEntry(c, experimentEntry), currentConsent = parseStorageConsent(sessionConsent, entry.sessionId);
|
|
1194
|
+
check3(currentConsent.decision === "granted", "session storage declined; workflow import refused");
|
|
1195
|
+
check3(entry.decisionTimeReviewed && entry.redactionReviewed, "decision-time and redaction review required");
|
|
1196
|
+
keys3(selection, ["schema", "kind", "source", "replacements", "engineering"], "workflow import selection");
|
|
1197
|
+
check3(selection.schema === 1 && selection.kind === "decision-selected-workflow-input", "unsupported workflow import selection");
|
|
1198
|
+
reference(selection.source);
|
|
1199
|
+
check3(Array.isArray(selection.engineering) && selection.engineering.length <= 8, "select at most 8 engineering records");
|
|
1200
|
+
const source = json2(await readExplicit(selection.source.path, selection.source.sha256));
|
|
1201
|
+
const { packet, consent, refs } = validateSource(source), directory = dirname(selection.source.path);
|
|
1202
|
+
check3(source.sessionId === entry.sessionId && c.source.sha256 === selection.source.sha256 && c.source.recordId === source.toolCallId, "workflow source/session identity mismatch");
|
|
1203
|
+
check3(c.question === source.question, "workflow question must match the selected decision");
|
|
1204
|
+
const input = await readExplicit(join4(directory, "input.txt"), source.inputSha256);
|
|
1205
|
+
check3(input.equals(Buffer.from(source.input, "utf8")), "input artifact differs from decision input");
|
|
1206
|
+
const redacted = redactSelectedInput(source.input, selection.replacements);
|
|
1207
|
+
check3(c.input === redacted.input, "case input must equal exact selected redacted decision input");
|
|
1208
|
+
check3(!selection.replacements.length || c.visibility === "redacted", "redacted case must declare redacted visibility");
|
|
1209
|
+
const mappingBytes = await readBoundedArtifact(join4(directory, "evidence.json"));
|
|
1210
|
+
const mapping = json2(mappingBytes);
|
|
1211
|
+
keys3(mapping, ["schema", "evidenceRefs"], "retained decision evidence");
|
|
1212
|
+
check3(mapping.schema === 1 && isDeepStrictEqual(evidenceRefs(mapping.evidenceRefs, true), refs), "decision evidence mapping mismatch");
|
|
1213
|
+
const decisionEvidence = [];
|
|
1214
|
+
for (const [index, ref] of mapping.evidenceRefs.entries()) decisionEvidence.push(await snapshot(directory, ref, `decision-evidence-${index + 1}.bin`));
|
|
1215
|
+
const engineering = [], seen = /* @__PURE__ */ new Set();
|
|
1216
|
+
for (const ref of selection.engineering) {
|
|
1217
|
+
reference(ref);
|
|
1218
|
+
check3(dirname(ref.path) === directory && !seen.has(ref.path), "engineering record must be a distinct selected sibling");
|
|
1219
|
+
seen.add(ref.path);
|
|
1220
|
+
const record = json2(await readExplicit(ref.path, ref.sha256));
|
|
1221
|
+
keys3(record, ENGINEERING_KEYS, "engineering evidence");
|
|
1222
|
+
check3(record.schema === 1 && record.kind === "skill-harness-engineering-evidence-v1", "provider advice is not engineering evidence");
|
|
1223
|
+
keys3(record.source, ["path", "sha256", "inputSha256"], "engineering source");
|
|
1224
|
+
check3(record.source.path === "selection.json" && record.source.sha256 === selection.source.sha256 && record.source.inputSha256 === source.inputSha256, "engineering source mismatch");
|
|
1225
|
+
text3(record.toolCallId, "engineering tool-call ID");
|
|
1226
|
+
check3(record.sessionId === source.sessionId && record.candidate === packet.candidate && date2(record.recordedAt) >= date2(source.frozenAt), "engineering session/candidate/time mismatch");
|
|
1227
|
+
check3(record.sourceBinding === "local-reference-digests-verified" && record.independence === "unassessed", "unsupported engineering evidence claims");
|
|
1228
|
+
unlabeled(record);
|
|
1229
|
+
observed(record, source.sessionId, packet.candidate);
|
|
1230
|
+
check3(source.candidateIdentity !== "principal-runtime-observed" || record.candidateIdentity === "principal-runtime-observed" && record.candidateObservation.candidate.root === source.candidateObservation.candidate.root, "engineering evidence lost its matching candidate observation");
|
|
1231
|
+
const linkedRefs = evidenceRefs(record.evidenceRefs, true);
|
|
1232
|
+
check3(linkedRefs.length > 0, "engineering evidence requires selected files");
|
|
1233
|
+
const key = learningDigest(JSON.stringify({ candidate: record.candidate, evidenceRefs: linkedRefs }));
|
|
1234
|
+
check3(ref.path === join4(directory, `engineering-${key}.json`), "engineering filename does not match producer identity");
|
|
1235
|
+
const artifacts = [];
|
|
1236
|
+
for (const [index, saved] of record.evidenceRefs.entries()) artifacts.push(await snapshot(directory, saved, `engineering-${key}-${index + 1}.bin`));
|
|
1237
|
+
engineering.push({ sha256: ref.sha256, sessionId: record.sessionId, toolCallId: record.toolCallId, candidateIdentity: record.candidateIdentity, recordedAt: record.recordedAt, artifacts, independence: "unassessed" });
|
|
1238
|
+
}
|
|
1239
|
+
return { caseDocument: structuredClone(caseDocument), experimentEntry: entry, receipt: {
|
|
1240
|
+
schema: 1,
|
|
1241
|
+
kind: "decision-selected-workflow-import",
|
|
1242
|
+
caseId: c.id,
|
|
1243
|
+
caseHash: c.hash,
|
|
1244
|
+
sessionId: source.sessionId,
|
|
1245
|
+
toolCallId: source.toolCallId,
|
|
1246
|
+
source: { sha256: selection.source.sha256, recordId: source.toolCallId },
|
|
1247
|
+
inputSha256: source.inputSha256,
|
|
1248
|
+
...source.schema === 3 ? { sourceContract: source.kind, nativeRouteSha256: learningDigest(source.route), nativeRequestSha256: source.requestSha256 } : {},
|
|
1249
|
+
recordedConsentHash: learningDigest(consent),
|
|
1250
|
+
currentConsentHash: learningDigest(currentConsent),
|
|
1251
|
+
candidateIdentity: source.candidateIdentity,
|
|
1252
|
+
candidateObservationSha256: source.candidateObservation ? learningDigest(source.candidateObservation) : null,
|
|
1253
|
+
replacements: redacted.replacements,
|
|
1254
|
+
decisionEvidence: { mappingSha256: learningDigest(mappingBytes), artifacts: decisionEvidence },
|
|
1255
|
+
engineering,
|
|
1256
|
+
labelStatus: "unlabeled",
|
|
1257
|
+
trainingEligible: false,
|
|
1258
|
+
exportEligible: false,
|
|
1259
|
+
publicCaptureVerified: false,
|
|
1260
|
+
automaticCollection: false,
|
|
1261
|
+
providerPredictionsIncluded: false,
|
|
1262
|
+
meaning: "Selected local bytes and declared producer linkage checked at import time; authenticity, source truth, independence, redaction and rights remain reviewer assertions. Export separately checks curated cases, entries, consent and independent labels; it does not reverify this receipt."
|
|
1263
|
+
} };
|
|
1264
|
+
}
|
|
1265
|
+
|
|
719
1266
|
// experiments/decision-shadow/workflow-fixtures.mjs
|
|
720
1267
|
var WORKFLOW_ORACLE_VERSION = "workflow-convergence-oracles-v1";
|
|
721
1268
|
function evaluateWorkflowFixture(family, facts) {
|
|
@@ -739,7 +1286,7 @@ function evaluateWorkflowFixture(family, facts) {
|
|
|
739
1286
|
}
|
|
740
1287
|
}
|
|
741
1288
|
function workflowFixtureFamilies() {
|
|
742
|
-
const root = "/fixture/pilot",
|
|
1289
|
+
const root = "/fixture/pilot", hash4 = "a".repeat(64);
|
|
743
1290
|
const workspace = {
|
|
744
1291
|
requestedRoot: root,
|
|
745
1292
|
observedRoot: root,
|
|
@@ -748,8 +1295,8 @@ function workflowFixtureFamilies() {
|
|
|
748
1295
|
readablePaths: [`${root}/report.md`]
|
|
749
1296
|
};
|
|
750
1297
|
const identity2 = {
|
|
751
|
-
expected: { algorithm: "principal-candidate-v1", id:
|
|
752
|
-
actual: { algorithm: "principal-candidate-v1", id:
|
|
1298
|
+
expected: { algorithm: "principal-candidate-v1", id: hash4 },
|
|
1299
|
+
actual: { algorithm: "principal-candidate-v1", id: hash4 }
|
|
753
1300
|
};
|
|
754
1301
|
const evidence = {
|
|
755
1302
|
candidateMatched: true,
|
|
@@ -760,9 +1307,9 @@ function workflowFixtureFamilies() {
|
|
|
760
1307
|
const reuse = {
|
|
761
1308
|
operation: "static-inspection",
|
|
762
1309
|
purpose: "same-check",
|
|
763
|
-
currentInput:
|
|
1310
|
+
currentInput: hash4,
|
|
764
1311
|
currentEnvironment: "node-fixture-v1",
|
|
765
|
-
previous: { state: "complete", input:
|
|
1312
|
+
previous: { state: "complete", input: hash4, environment: "node-fixture-v1", outputsVerified: true }
|
|
766
1313
|
};
|
|
767
1314
|
return [
|
|
768
1315
|
{
|
|
@@ -775,7 +1322,7 @@ function workflowFixtureFamilies() {
|
|
|
775
1322
|
family: "identity",
|
|
776
1323
|
split: "train",
|
|
777
1324
|
question: "Do these records contain matching full candidate IDs under the same principal-candidate-v1 algorithm? A diff hash or another algorithm is not an interchangeable candidate ID.",
|
|
778
|
-
facts: [identity2, { ...identity2, actual: { algorithm: "git-diff-sha256", id:
|
|
1325
|
+
facts: [identity2, { ...identity2, actual: { algorithm: "git-diff-sha256", id: hash4 } }, { ...identity2, actual: { algorithm: "principal-candidate-v1", id: "b".repeat(64) } }, { expected: { algorithm: "principal-candidate-v1", id: "aaaaaaa" }, actual: { algorithm: "principal-candidate-v1", id: "aaaaaaa" } }]
|
|
779
1326
|
},
|
|
780
1327
|
{
|
|
781
1328
|
family: "evidence",
|
|
@@ -875,30 +1422,30 @@ function createLearningFixtures({ recordedAt = "2026-10-08T00:00:00.000Z", set =
|
|
|
875
1422
|
}
|
|
876
1423
|
|
|
877
1424
|
// experiments/decision-shadow/research.mjs
|
|
878
|
-
import { createHash as
|
|
879
|
-
import { open as
|
|
1425
|
+
import { createHash as createHash5, randomUUID } from "node:crypto";
|
|
1426
|
+
import { open as open3, readFile } from "node:fs/promises";
|
|
880
1427
|
import { DECISION_SYSTEM_PROMPT } from "@skill-harness/adapters";
|
|
881
1428
|
var PROMPT_REVISION = "decision-input-question-v1";
|
|
882
1429
|
var decisionPrompt = (c) => JSON.stringify({ input: c.input, question: c.question });
|
|
883
1430
|
var fullCaseSetHash = (cases) => learningDigest(cases.map((c) => ({ id: c.id, hash: c.hash })));
|
|
884
|
-
var
|
|
885
|
-
function
|
|
1431
|
+
var hash2 = (s) => createHash5("sha256").update(s).digest("hex");
|
|
1432
|
+
function check4(ok, message) {
|
|
886
1433
|
if (!ok) throw TypeError(message);
|
|
887
1434
|
}
|
|
888
|
-
function
|
|
889
|
-
|
|
1435
|
+
function keys4(value, names, name) {
|
|
1436
|
+
check4(
|
|
890
1437
|
value && typeof value === "object" && !Array.isArray(value) && Object.keys(value).sort().join("|") === [...names].sort().join("|"),
|
|
891
1438
|
name + " has unsupported/missing fields"
|
|
892
1439
|
);
|
|
893
1440
|
}
|
|
894
1441
|
function sha3(value) {
|
|
895
|
-
|
|
1442
|
+
check4(
|
|
896
1443
|
typeof value === "string" && /^[0-9a-f]{64}$/.test(value),
|
|
897
1444
|
"invalid digest"
|
|
898
1445
|
);
|
|
899
1446
|
}
|
|
900
1447
|
function selected(cases, manifest, split) {
|
|
901
|
-
|
|
1448
|
+
check4(
|
|
902
1449
|
["train", "validation", "test", "all"].includes(split),
|
|
903
1450
|
"explicit train/validation/test/all split required"
|
|
904
1451
|
);
|
|
@@ -908,16 +1455,16 @@ function selected(cases, manifest, split) {
|
|
|
908
1455
|
}
|
|
909
1456
|
function previewPi(cases, manifest, { model, thinking, split }) {
|
|
910
1457
|
manifest = parseExperiment(cases, manifest);
|
|
911
|
-
|
|
1458
|
+
check4(
|
|
912
1459
|
/^openai-codex:[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(model),
|
|
913
1460
|
"explicit subscription model required"
|
|
914
1461
|
);
|
|
915
|
-
|
|
1462
|
+
check4(
|
|
916
1463
|
["off", "minimal", "low", "medium", "high", "xhigh"].includes(thinking),
|
|
917
1464
|
"explicit thinking level required"
|
|
918
1465
|
);
|
|
919
1466
|
const subset = selected(cases, manifest, split);
|
|
920
|
-
|
|
1467
|
+
check4(subset.length > 0, "selected split is empty");
|
|
921
1468
|
return {
|
|
922
1469
|
schema: 1,
|
|
923
1470
|
kind: "decision-pi-preview",
|
|
@@ -932,7 +1479,7 @@ function previewPi(cases, manifest, { model, thinking, split }) {
|
|
|
932
1479
|
caseId: c.id,
|
|
933
1480
|
caseHash: c.hash,
|
|
934
1481
|
prompt: decisionPrompt(c),
|
|
935
|
-
promptSha256:
|
|
1482
|
+
promptSha256: hash2(decisionPrompt(c))
|
|
936
1483
|
}))
|
|
937
1484
|
};
|
|
938
1485
|
}
|
|
@@ -969,10 +1516,10 @@ async function runPiBaseline({
|
|
|
969
1516
|
caseHash: r.caseHash
|
|
970
1517
|
})),
|
|
971
1518
|
promptRevision: PROMPT_REVISION,
|
|
972
|
-
systemPromptSha256:
|
|
1519
|
+
systemPromptSha256: hash2(adapter.DECISION_SYSTEM_PROMPT),
|
|
973
1520
|
trainingEligible: false
|
|
974
1521
|
};
|
|
975
|
-
const file = await
|
|
1522
|
+
const file = await open3(out, "wx", 384);
|
|
976
1523
|
const append = async (row) => {
|
|
977
1524
|
await file.writeFile(JSON.stringify(row) + "\n");
|
|
978
1525
|
await file.sync();
|
|
@@ -1021,16 +1568,16 @@ async function runPiBaseline({
|
|
|
1021
1568
|
);
|
|
1022
1569
|
return header;
|
|
1023
1570
|
}
|
|
1024
|
-
function parsePiRun(
|
|
1571
|
+
function parsePiRun(text4, cases, manifest) {
|
|
1025
1572
|
manifest = parseExperiment(cases, manifest);
|
|
1026
1573
|
let rows;
|
|
1027
1574
|
try {
|
|
1028
|
-
rows =
|
|
1575
|
+
rows = text4.trim().split("\n").map((line) => JSON.parse(line));
|
|
1029
1576
|
} catch {
|
|
1030
1577
|
throw Error("Invalid Pi run JSONL");
|
|
1031
1578
|
}
|
|
1032
1579
|
const [header, ...records] = rows;
|
|
1033
|
-
|
|
1580
|
+
keys4(
|
|
1034
1581
|
header,
|
|
1035
1582
|
[
|
|
1036
1583
|
"schema",
|
|
@@ -1049,11 +1596,11 @@ function parsePiRun(text2, cases, manifest) {
|
|
|
1049
1596
|
],
|
|
1050
1597
|
"Pi run header"
|
|
1051
1598
|
);
|
|
1052
|
-
|
|
1599
|
+
check4(
|
|
1053
1600
|
header.schema === 2 && header.kind === "decision-pi-run" && header.trainingEligible === false,
|
|
1054
1601
|
"unsupported Pi run"
|
|
1055
1602
|
);
|
|
1056
|
-
|
|
1603
|
+
check4(
|
|
1057
1604
|
/^[0-9a-f-]{36}$/.test(header.runId) && new Date(header.createdAt).toISOString() === header.createdAt,
|
|
1058
1605
|
"invalid run identity/time"
|
|
1059
1606
|
);
|
|
@@ -1062,7 +1609,7 @@ function parsePiRun(text2, cases, manifest) {
|
|
|
1062
1609
|
thinking: header.thinking,
|
|
1063
1610
|
split: header.split
|
|
1064
1611
|
});
|
|
1065
|
-
|
|
1612
|
+
check4(
|
|
1066
1613
|
header.caseSetHash === preview.caseSetHash && header.experimentHash === preview.experimentHash && header.promptRevision === PROMPT_REVISION && learningDigest(header.selected) === learningDigest(
|
|
1067
1614
|
preview.requests.map((r) => ({
|
|
1068
1615
|
caseId: r.caseId,
|
|
@@ -1071,21 +1618,21 @@ function parsePiRun(text2, cases, manifest) {
|
|
|
1071
1618
|
),
|
|
1072
1619
|
"Pi run belongs to a different experiment/selection"
|
|
1073
1620
|
);
|
|
1074
|
-
|
|
1075
|
-
header.systemPromptSha256 ===
|
|
1621
|
+
check4(
|
|
1622
|
+
header.systemPromptSha256 === hash2(DECISION_SYSTEM_PROMPT),
|
|
1076
1623
|
"Pi system prompt revision mismatch"
|
|
1077
1624
|
);
|
|
1078
|
-
|
|
1625
|
+
check4(records.length <= preview.requests.length, "too many Pi predictions");
|
|
1079
1626
|
let worker = null;
|
|
1080
1627
|
const sessions = /* @__PURE__ */ new Set();
|
|
1081
1628
|
for (const [i, r] of records.entries()) {
|
|
1082
1629
|
const expected = preview.requests[i];
|
|
1083
|
-
|
|
1630
|
+
check4(
|
|
1084
1631
|
r.caseId === expected.caseId && r.caseHash === expected.caseHash && r.promptSha256 === expected.promptSha256 && r.kind === "prediction" && r.trainingEligible === false,
|
|
1085
1632
|
"Pi prediction identity/order mismatch"
|
|
1086
1633
|
);
|
|
1087
1634
|
if (r.status === "error") {
|
|
1088
|
-
|
|
1635
|
+
keys4(
|
|
1089
1636
|
r,
|
|
1090
1637
|
[
|
|
1091
1638
|
"kind",
|
|
@@ -1099,13 +1646,13 @@ function parsePiRun(text2, cases, manifest) {
|
|
|
1099
1646
|
],
|
|
1100
1647
|
"Pi error"
|
|
1101
1648
|
);
|
|
1102
|
-
|
|
1649
|
+
check4(
|
|
1103
1650
|
r.probability === null && r.error === "Pi decision failed; no retry or fallback." && i === records.length - 1,
|
|
1104
1651
|
"invalid Pi error prefix"
|
|
1105
1652
|
);
|
|
1106
1653
|
continue;
|
|
1107
1654
|
}
|
|
1108
|
-
|
|
1655
|
+
keys4(
|
|
1109
1656
|
r,
|
|
1110
1657
|
[
|
|
1111
1658
|
"kind",
|
|
@@ -1126,48 +1673,48 @@ function parsePiRun(text2, cases, manifest) {
|
|
|
1126
1673
|
],
|
|
1127
1674
|
"Pi prediction"
|
|
1128
1675
|
);
|
|
1129
|
-
|
|
1676
|
+
check4(
|
|
1130
1677
|
["answered", "abstained"].includes(r.status) && r.resolvedModel === header.requestedModel && r.piVersion === "1.0.4" && r.systemPromptSha256 === header.systemPromptSha256,
|
|
1131
1678
|
"Pi result route/version mismatch"
|
|
1132
1679
|
);
|
|
1133
|
-
|
|
1680
|
+
check4(
|
|
1134
1681
|
r.status === "abstained" ? r.probability === null : typeof r.probability === "number" && Number.isFinite(r.probability) && r.probability >= 0 && r.probability <= 1,
|
|
1135
1682
|
"invalid Pi probability"
|
|
1136
1683
|
);
|
|
1137
|
-
|
|
1684
|
+
keys4(r.nativeFinal, ["sessionId", "messageId", "sha256"], "native final");
|
|
1138
1685
|
for (const k of ["sessionId", "messageId"])
|
|
1139
|
-
|
|
1686
|
+
check4(
|
|
1140
1687
|
typeof r.nativeFinal[k] === "string" && r.nativeFinal[k].length > 0 && r.nativeFinal[k].length <= 256,
|
|
1141
1688
|
"invalid native identity"
|
|
1142
1689
|
);
|
|
1143
1690
|
sha3(r.nativeFinal.sha256);
|
|
1144
|
-
|
|
1691
|
+
check4(
|
|
1145
1692
|
!sessions.has(r.nativeFinal.sessionId),
|
|
1146
1693
|
"Pi cases must have independent fresh sessions"
|
|
1147
1694
|
);
|
|
1148
1695
|
sessions.add(r.nativeFinal.sessionId);
|
|
1149
|
-
|
|
1150
|
-
|
|
1696
|
+
keys4(r.route, ["oauth", "subscription"], "route");
|
|
1697
|
+
check4(
|
|
1151
1698
|
r.route.oauth === true && r.route.subscription === true,
|
|
1152
1699
|
"non-subscription Pi result"
|
|
1153
1700
|
);
|
|
1154
|
-
|
|
1155
|
-
|
|
1701
|
+
keys4(r.usage, ["inputTokens", "outputTokens", "costUsd"], "usage");
|
|
1702
|
+
check4(r.usage.costUsd === null, "subscription cost must not be estimated");
|
|
1156
1703
|
for (const k of ["inputTokens", "outputTokens"])
|
|
1157
|
-
|
|
1704
|
+
check4(
|
|
1158
1705
|
r.usage[k] === null || Number.isSafeInteger(r.usage[k]) && r.usage[k] >= 0,
|
|
1159
1706
|
"invalid Pi usage"
|
|
1160
1707
|
);
|
|
1161
|
-
|
|
1708
|
+
check4(Number.isFinite(r.latencyMs) && r.latencyMs >= 0, "invalid latency");
|
|
1162
1709
|
sha3(r.workerSha256);
|
|
1163
1710
|
if (worker !== null)
|
|
1164
|
-
|
|
1711
|
+
check4(worker === r.workerSha256, "mixed worker revisions");
|
|
1165
1712
|
worker = r.workerSha256;
|
|
1166
1713
|
}
|
|
1167
1714
|
return {
|
|
1168
1715
|
header,
|
|
1169
1716
|
records,
|
|
1170
|
-
runSha256:
|
|
1717
|
+
runSha256: hash2(text4),
|
|
1171
1718
|
cases: preview.requests.map((r) => cases.find((c) => c.id === r.caseId)),
|
|
1172
1719
|
experimentBound: true
|
|
1173
1720
|
};
|
|
@@ -1216,11 +1763,11 @@ function metrics(cases, labels, records) {
|
|
|
1216
1763
|
}
|
|
1217
1764
|
function compareRuns(cases, labels, manifest, runs) {
|
|
1218
1765
|
manifest = parseExperiment(cases, manifest);
|
|
1219
|
-
|
|
1766
|
+
check4(
|
|
1220
1767
|
Array.isArray(runs) && runs.length >= 1 && runs.length <= 8,
|
|
1221
1768
|
"compare 1..8 explicit runs"
|
|
1222
1769
|
);
|
|
1223
|
-
|
|
1770
|
+
check4(
|
|
1224
1771
|
new Set(runs.map((r) => r.runSha256)).size === runs.length,
|
|
1225
1772
|
"duplicate run files"
|
|
1226
1773
|
);
|
|
@@ -1238,7 +1785,7 @@ function compareRuns(cases, labels, manifest, runs) {
|
|
|
1238
1785
|
run.records.filter((r) => r.status === "answered" || r.status === "abstained").map((r) => r.resolvedModel)
|
|
1239
1786
|
)
|
|
1240
1787
|
];
|
|
1241
|
-
|
|
1788
|
+
check4(
|
|
1242
1789
|
observedModels.length <= 1,
|
|
1243
1790
|
"mixed resolved models in one comparison arm"
|
|
1244
1791
|
);
|
|
@@ -1249,6 +1796,12 @@ function compareRuns(cases, labels, manifest, runs) {
|
|
|
1249
1796
|
runSha256: run.runSha256,
|
|
1250
1797
|
runId: run.header.runId ?? null,
|
|
1251
1798
|
model: run.header.requestedModel,
|
|
1799
|
+
...run.header.kind === "decision-shadow-native-run" ? {
|
|
1800
|
+
route: run.header.route,
|
|
1801
|
+
requestFormat: run.header.requestFormat,
|
|
1802
|
+
costSource: run.header.costSource,
|
|
1803
|
+
modelResolution: "Pi returns the requested model identity; backend resolution is unknown."
|
|
1804
|
+
} : {},
|
|
1252
1805
|
observedModels,
|
|
1253
1806
|
createdAt: run.header.createdAt,
|
|
1254
1807
|
thinking: run.header.thinking ?? null,
|
|
@@ -1313,6 +1866,7 @@ var RESEARCH_OPTIONS = {
|
|
|
1313
1866
|
"out"
|
|
1314
1867
|
],
|
|
1315
1868
|
"import-session": ["cases", "selection", "consent", "entry", "out"],
|
|
1869
|
+
"import-workflow": ["cases", "selection", "consent", "entry", "out"],
|
|
1316
1870
|
fixtures: ["out"]
|
|
1317
1871
|
};
|
|
1318
1872
|
var RESEARCH_OPTIONAL = {
|
|
@@ -1324,11 +1878,11 @@ async function researchCommand(command, flags, helpers, options) {
|
|
|
1324
1878
|
const { emit = console.log, piRunner } = options;
|
|
1325
1879
|
if (command === "fixtures") {
|
|
1326
1880
|
const f = createLearningFixtures({ set: flags.set ?? "mechanical" });
|
|
1327
|
-
const dir =
|
|
1881
|
+
const dir = resolve2(flags.out);
|
|
1328
1882
|
await mkdir2(dir, { mode: 448 });
|
|
1329
1883
|
const refs = [];
|
|
1330
1884
|
for (const r of f.labelReceipts) {
|
|
1331
|
-
const path =
|
|
1885
|
+
const path = join5(dir, r.filename);
|
|
1332
1886
|
await writeFile2(path, r.bytes, { flag: "wx", mode: 384 });
|
|
1333
1887
|
refs.push({ caseId: r.caseId, path, sha256: r.sha256 });
|
|
1334
1888
|
}
|
|
@@ -1339,7 +1893,7 @@ async function researchCommand(command, flags, helpers, options) {
|
|
|
1339
1893
|
"label-evidence.json": refs,
|
|
1340
1894
|
"consents.json": []
|
|
1341
1895
|
}))
|
|
1342
|
-
await writeNew2(
|
|
1896
|
+
await writeNew2(join5(dir, name), value);
|
|
1343
1897
|
emit(
|
|
1344
1898
|
JSON.stringify({
|
|
1345
1899
|
saved: dir,
|
|
@@ -1350,15 +1904,16 @@ async function researchCommand(command, flags, helpers, options) {
|
|
|
1350
1904
|
);
|
|
1351
1905
|
return;
|
|
1352
1906
|
}
|
|
1353
|
-
if (command === "import-session") {
|
|
1354
|
-
const
|
|
1907
|
+
if (command === "import-session" || command === "import-workflow") {
|
|
1908
|
+
const importCase = command === "import-workflow" ? importWorkflowCase : importSelectedCase;
|
|
1909
|
+
const result = await importCase({
|
|
1355
1910
|
caseDocument: await jsonFile2(flags.cases),
|
|
1356
1911
|
selection: await jsonFile2(flags.selection),
|
|
1357
1912
|
sessionConsent: await jsonFile2(flags.consent),
|
|
1358
1913
|
experimentEntry: await jsonFile2(flags.entry)
|
|
1359
1914
|
});
|
|
1360
1915
|
await writeNew2(flags.out, result);
|
|
1361
|
-
emit(JSON.stringify({ saved:
|
|
1916
|
+
emit(JSON.stringify({ saved: resolve2(flags.out), trainingReady: false }));
|
|
1362
1917
|
return;
|
|
1363
1918
|
}
|
|
1364
1919
|
const manifest = parseExperiment(cases, await jsonFile2(flags.experiment));
|
|
@@ -1403,7 +1958,7 @@ async function researchCommand(command, flags, helpers, options) {
|
|
|
1403
1958
|
});
|
|
1404
1959
|
emit(
|
|
1405
1960
|
JSON.stringify({
|
|
1406
|
-
saved:
|
|
1961
|
+
saved: resolve2(flags.out),
|
|
1407
1962
|
runId: header.runId,
|
|
1408
1963
|
trainingEligible: false
|
|
1409
1964
|
})
|
|
@@ -1423,7 +1978,7 @@ async function researchCommand(command, flags, helpers, options) {
|
|
|
1423
1978
|
});
|
|
1424
1979
|
const result = await writePreparedExport({
|
|
1425
1980
|
prepared,
|
|
1426
|
-
directory:
|
|
1981
|
+
directory: resolve2(flags.out)
|
|
1427
1982
|
});
|
|
1428
1983
|
emit(JSON.stringify(result));
|
|
1429
1984
|
return;
|
|
@@ -1436,15 +1991,15 @@ async function researchCommand(command, flags, helpers, options) {
|
|
|
1436
1991
|
});
|
|
1437
1992
|
const runs = [];
|
|
1438
1993
|
for (const path of flags.run) {
|
|
1439
|
-
const
|
|
1994
|
+
const text4 = await textFile2(path);
|
|
1440
1995
|
let first;
|
|
1441
1996
|
try {
|
|
1442
|
-
first = JSON.parse(
|
|
1997
|
+
first = JSON.parse(text4.split("\n")[0]);
|
|
1443
1998
|
} catch {
|
|
1444
1999
|
throw Error("Invalid run header");
|
|
1445
2000
|
}
|
|
1446
2001
|
if (first.kind === "decision-pi-run")
|
|
1447
|
-
runs.push(parsePiRun(
|
|
2002
|
+
runs.push(parsePiRun(text4, cases, manifest));
|
|
1448
2003
|
else
|
|
1449
2004
|
runs.push({
|
|
1450
2005
|
...await readLegacyRun(path, cases),
|
|
@@ -1457,7 +2012,7 @@ async function researchCommand(command, flags, helpers, options) {
|
|
|
1457
2012
|
await writeNew2(flags.out, report);
|
|
1458
2013
|
emit(
|
|
1459
2014
|
JSON.stringify({
|
|
1460
|
-
saved:
|
|
2015
|
+
saved: resolve2(flags.out),
|
|
1461
2016
|
commonLabeledAnswered: report.commonLabeledAnswered,
|
|
1462
2017
|
trainingReady: false
|
|
1463
2018
|
})
|
|
@@ -1471,16 +2026,18 @@ async function researchCommand(command, flags, helpers, options) {
|
|
|
1471
2026
|
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
1472
2027
|
import { createInterface } from "node:readline/promises";
|
|
1473
2028
|
import { stdin, stderr } from "node:process";
|
|
1474
|
-
import { openSync, writeFileSync, fsyncSync, closeSync } from "node:fs";
|
|
2029
|
+
import { openSync as openSync2, writeFileSync as writeFileSync2, fsyncSync as fsyncSync2, closeSync as closeSync2 } from "node:fs";
|
|
1475
2030
|
var STORAGE_QUESTION = "Retain this session\u2019s explicitly selected inputs and questions for future LoRA dataset review? This grants storage only, not training or use of JEV predictions as labels.";
|
|
1476
2031
|
async function requestSessionStorage({
|
|
1477
2032
|
sessionId = randomUUID2(),
|
|
1478
2033
|
storage,
|
|
1479
|
-
promptStorage
|
|
2034
|
+
promptStorage,
|
|
2035
|
+
storageRoot
|
|
1480
2036
|
} = {}) {
|
|
1481
2037
|
let answer = storage;
|
|
2038
|
+
const question = STORAGE_QUESTION + (storageRoot ? ` Local collection root: ${storageRoot}.` : "");
|
|
1482
2039
|
if (answer === void 0) {
|
|
1483
|
-
if (promptStorage) answer = await promptStorage(
|
|
2040
|
+
if (promptStorage) answer = await promptStorage(question);
|
|
1484
2041
|
else {
|
|
1485
2042
|
if (!stdin.isTTY || !stderr.isTTY)
|
|
1486
2043
|
throw Error(
|
|
@@ -1489,7 +2046,7 @@ async function requestSessionStorage({
|
|
|
1489
2046
|
const rl = createInterface({ input: stdin, output: stderr });
|
|
1490
2047
|
try {
|
|
1491
2048
|
answer = /^(y|yes)$/i.test(
|
|
1492
|
-
(await rl.question(
|
|
2049
|
+
(await rl.question(question + " [y/N] ")).trim()
|
|
1493
2050
|
) ? "yes" : "no";
|
|
1494
2051
|
} finally {
|
|
1495
2052
|
rl.close();
|
|
@@ -1531,15 +2088,15 @@ async function retainSelectedSessionData({
|
|
|
1531
2088
|
trainingEligible: false,
|
|
1532
2089
|
labelStatus: "unlabeled",
|
|
1533
2090
|
providerOutputsIncluded: false,
|
|
1534
|
-
cases: cases.map(({ hash:
|
|
2091
|
+
cases: cases.map(({ hash: hash4, ...c }) => ({ caseHash: hash4, ...c }))
|
|
1535
2092
|
};
|
|
1536
2093
|
assertCurrent();
|
|
1537
|
-
const fd =
|
|
2094
|
+
const fd = openSync2(out, "wx", 384);
|
|
1538
2095
|
try {
|
|
1539
|
-
|
|
1540
|
-
|
|
2096
|
+
writeFileSync2(fd, JSON.stringify(document, null, 2) + "\n");
|
|
2097
|
+
fsyncSync2(fd);
|
|
1541
2098
|
} finally {
|
|
1542
|
-
|
|
2099
|
+
closeSync2(fd);
|
|
1543
2100
|
}
|
|
1544
2101
|
return { retained: true, path: out, trainingReady: false };
|
|
1545
2102
|
}
|
|
@@ -1579,7 +2136,7 @@ function makeRequest(provider, model, example) {
|
|
|
1579
2136
|
}
|
|
1580
2137
|
};
|
|
1581
2138
|
}
|
|
1582
|
-
function
|
|
2139
|
+
function isRecord2(value) {
|
|
1583
2140
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
1584
2141
|
}
|
|
1585
2142
|
function probability(value) {
|
|
@@ -1605,11 +2162,11 @@ function isSupportedResolvedModel(value) {
|
|
|
1605
2162
|
if (value === PROVIDER.model) return true;
|
|
1606
2163
|
const prefix = `${PROVIDER.model}-`;
|
|
1607
2164
|
if (typeof value !== "string" || !value.startsWith(prefix)) return false;
|
|
1608
|
-
const
|
|
1609
|
-
if (
|
|
1610
|
-
const year = Number(
|
|
1611
|
-
const month = Number(
|
|
1612
|
-
const day = Number(
|
|
2165
|
+
const snapshot2 = value.slice(prefix.length);
|
|
2166
|
+
if (snapshot2.length !== 8 || !/^\d{8}$/.test(snapshot2)) return false;
|
|
2167
|
+
const year = Number(snapshot2.slice(0, 4));
|
|
2168
|
+
const month = Number(snapshot2.slice(4, 6));
|
|
2169
|
+
const day = Number(snapshot2.slice(6, 8));
|
|
1613
2170
|
const leapYear = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
|
|
1614
2171
|
const monthLengths = [31, leapYear ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
|
1615
2172
|
return year > 0 && month >= 1 && month <= 12 && day >= 1 && day <= monthLengths[month - 1];
|
|
@@ -1620,12 +2177,12 @@ function resolvedModel(value) {
|
|
|
1620
2177
|
}
|
|
1621
2178
|
return value;
|
|
1622
2179
|
}
|
|
1623
|
-
function
|
|
1624
|
-
if (!
|
|
2180
|
+
function normalize3(payload) {
|
|
2181
|
+
if (!isRecord2(payload) || !isRecord2(payload.answers) || Object.keys(payload.answers).length !== 1 || !isRecord2(payload.usage)) {
|
|
1625
2182
|
throw new Error("invalid provider response");
|
|
1626
2183
|
}
|
|
1627
2184
|
const answer = payload.answers.decision;
|
|
1628
|
-
if (!
|
|
2185
|
+
if (!isRecord2(answer) || answer.type !== "noul") {
|
|
1629
2186
|
throw new Error("invalid provider response");
|
|
1630
2187
|
}
|
|
1631
2188
|
return {
|
|
@@ -1726,14 +2283,14 @@ async function callProvider(provider, model, example, options = {}) {
|
|
|
1726
2283
|
timedOut
|
|
1727
2284
|
]);
|
|
1728
2285
|
if (!response || response.ok !== true) throw new Error("provider request failed");
|
|
1729
|
-
const
|
|
2286
|
+
const text4 = await readBounded(response, timedOut);
|
|
1730
2287
|
let payload;
|
|
1731
2288
|
try {
|
|
1732
|
-
payload = JSON.parse(
|
|
2289
|
+
payload = JSON.parse(text4);
|
|
1733
2290
|
} catch {
|
|
1734
2291
|
throw new Error("invalid provider response");
|
|
1735
2292
|
}
|
|
1736
|
-
return { ...
|
|
2293
|
+
return { ...normalize3(payload), latencyMs: Math.max(0, performance.now() - startedAt) };
|
|
1737
2294
|
} catch (error) {
|
|
1738
2295
|
return failure(startedAt, error);
|
|
1739
2296
|
} finally {
|
|
@@ -1744,16 +2301,16 @@ async function callProvider(provider, model, example, options = {}) {
|
|
|
1744
2301
|
}
|
|
1745
2302
|
|
|
1746
2303
|
// experiments/decision-shadow/public-evidence.mjs
|
|
1747
|
-
import { constants as
|
|
1748
|
-
import { open as
|
|
1749
|
-
import { createHash as
|
|
1750
|
-
import { isDeepStrictEqual } from "node:util";
|
|
1751
|
-
import { basename, dirname, isAbsolute as
|
|
2304
|
+
import { constants as constants3 } from "node:fs";
|
|
2305
|
+
import { open as open4 } from "node:fs/promises";
|
|
2306
|
+
import { createHash as createHash6 } from "node:crypto";
|
|
2307
|
+
import { isDeepStrictEqual as isDeepStrictEqual2 } from "node:util";
|
|
2308
|
+
import { basename, dirname as dirname2, isAbsolute as isAbsolute4, join as join6, normalize as normalize4 } from "node:path";
|
|
1752
2309
|
var MAX_PUBLIC_SOURCE_FILE_BYTES = 64 * 1024 * 1024;
|
|
1753
2310
|
var MAX_PUBLIC_SOURCE_TOTAL_BYTES = 128 * 1024 * 1024;
|
|
1754
2311
|
var MAX_PUBLIC_SOURCE_FILES = 256;
|
|
1755
2312
|
var MAX_PUBLIC_SOURCE_MANIFEST_BYTES = 1024 * 1024;
|
|
1756
|
-
var
|
|
2313
|
+
var LIMITS2 = Object.freeze({
|
|
1757
2314
|
fileBytes: MAX_PUBLIC_SOURCE_FILE_BYTES,
|
|
1758
2315
|
totalBytes: MAX_PUBLIC_SOURCE_TOTAL_BYTES,
|
|
1759
2316
|
files: MAX_PUBLIC_SOURCE_FILES,
|
|
@@ -1763,42 +2320,42 @@ var HASH = /^[0-9a-f]{64}$/;
|
|
|
1763
2320
|
var UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
|
1764
2321
|
var TOOLS = ["delegate_describe", "delegate", "delegate_all", "delegate_chain"];
|
|
1765
2322
|
var RUNTIME_PREFIX = "Runtime execution evidence (process settlement; not workspace cleanup or task acceptance):\n```json\n";
|
|
1766
|
-
var DIR_FLAGS =
|
|
1767
|
-
var FILE_FLAGS =
|
|
1768
|
-
var digest = (bytes) =>
|
|
1769
|
-
function
|
|
2323
|
+
var DIR_FLAGS = constants3.O_RDONLY | constants3.O_DIRECTORY | constants3.O_NOFOLLOW | constants3.O_NONBLOCK;
|
|
2324
|
+
var FILE_FLAGS = constants3.O_RDONLY | constants3.O_NOFOLLOW | constants3.O_NONBLOCK;
|
|
2325
|
+
var digest = (bytes) => createHash6("sha256").update(bytes).digest("hex");
|
|
2326
|
+
function check5(ok, message) {
|
|
1770
2327
|
if (!ok) throw new TypeError(message);
|
|
1771
2328
|
}
|
|
1772
|
-
function
|
|
1773
|
-
|
|
2329
|
+
function object2(value, name) {
|
|
2330
|
+
check5(value !== null && typeof value === "object" && !Array.isArray(value), `${name} must be an object`);
|
|
1774
2331
|
}
|
|
1775
|
-
function
|
|
1776
|
-
|
|
1777
|
-
|
|
2332
|
+
function keys5(value, expected, name) {
|
|
2333
|
+
object2(value, name);
|
|
2334
|
+
check5(isDeepStrictEqual2(Object.keys(value).sort(), [...expected].sort()), `${name} has unsupported or missing fields`);
|
|
1778
2335
|
}
|
|
1779
2336
|
function string(value, name, nullable = false) {
|
|
1780
2337
|
if (nullable && value === null) return;
|
|
1781
|
-
|
|
2338
|
+
check5(typeof value === "string" && value.length > 0 && value.length <= 4096 && !value.includes("\0"), `${name} must be a bounded nonempty string`);
|
|
1782
2339
|
}
|
|
1783
2340
|
function publicText(value, name) {
|
|
1784
|
-
|
|
2341
|
+
check5(typeof value === "string", name + " must be a string");
|
|
1785
2342
|
}
|
|
1786
|
-
function
|
|
1787
|
-
|
|
2343
|
+
function hash3(value, name, nullable = false) {
|
|
2344
|
+
check5(nullable && value === null || typeof value === "string" && HASH.test(value), `${name} must be a lowercase SHA-256`);
|
|
1788
2345
|
}
|
|
1789
2346
|
function integer(value, name, min = 0) {
|
|
1790
|
-
|
|
2347
|
+
check5(Number.isSafeInteger(value) && value >= min, `${name} must be a bounded integer`);
|
|
1791
2348
|
}
|
|
1792
2349
|
function enumeration(value, allowed, name) {
|
|
1793
|
-
|
|
2350
|
+
check5(allowed.includes(value), `${name} has an unsupported value`);
|
|
1794
2351
|
}
|
|
1795
2352
|
function array(value, name, max = 64) {
|
|
1796
|
-
|
|
2353
|
+
check5(Array.isArray(value) && value.length <= max, `${name} must be a bounded array`);
|
|
1797
2354
|
}
|
|
1798
2355
|
function canonicalPath(path, name) {
|
|
1799
2356
|
string(path, name);
|
|
1800
|
-
|
|
1801
|
-
|
|
2357
|
+
check5(isAbsolute4(path) && normalize4(path) === path && (path === "/" || !path.endsWith("/")), `${name} must be canonical and absolute`);
|
|
2358
|
+
check5(path.split("/").length <= 128, `${name} exceeds directory depth limit`);
|
|
1802
2359
|
}
|
|
1803
2360
|
function metadata(stat2) {
|
|
1804
2361
|
return ["dev", "ino", "mode", "uid", "gid", "size", "mtimeNs", "ctimeNs"].map((key) => stat2[key].toString()).join(":");
|
|
@@ -1806,7 +2363,7 @@ function metadata(stat2) {
|
|
|
1806
2363
|
function identity(stat2) {
|
|
1807
2364
|
return ["dev", "ino", "mode"].map((key) => stat2[key].toString()).join(":");
|
|
1808
2365
|
}
|
|
1809
|
-
function
|
|
2366
|
+
function json3(bytes, name) {
|
|
1810
2367
|
try {
|
|
1811
2368
|
return JSON.parse(new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes));
|
|
1812
2369
|
} catch {
|
|
@@ -1822,10 +2379,10 @@ var EvidenceReader = class {
|
|
|
1822
2379
|
this.root = root;
|
|
1823
2380
|
}
|
|
1824
2381
|
async directory(parent, component, path, stable = false) {
|
|
1825
|
-
const handle = await
|
|
2382
|
+
const handle = await open4(parent ? `/proc/self/fd/${parent.handle.fd}/${component}` : "/", DIR_FLAGS);
|
|
1826
2383
|
try {
|
|
1827
2384
|
const stat2 = await handle.stat({ bigint: true });
|
|
1828
|
-
|
|
2385
|
+
check5(stat2.isDirectory(), "Evidence path component is not a directory");
|
|
1829
2386
|
const record = { handle, parent, component, path, metadata: metadata(stat2), identity: identity(stat2), stable };
|
|
1830
2387
|
this.directories.push(record);
|
|
1831
2388
|
return record;
|
|
@@ -1846,27 +2403,27 @@ var EvidenceReader = class {
|
|
|
1846
2403
|
}
|
|
1847
2404
|
async read(path, limit = MAX_PUBLIC_SOURCE_FILE_BYTES) {
|
|
1848
2405
|
canonicalPath(path, "Copied evidence path");
|
|
1849
|
-
const capturePath =
|
|
1850
|
-
|
|
2406
|
+
const capturePath = dirname2(path), captureId = basename(capturePath);
|
|
2407
|
+
check5(dirname2(capturePath) === this.root && UUID.test(captureId), "Copied evidence must be directly inside a capture directory beneath the evidence root");
|
|
1851
2408
|
const cached = this.files.get(path);
|
|
1852
2409
|
if (cached) {
|
|
1853
|
-
|
|
2410
|
+
check5(cached.bytes.length <= limit, "Evidence file exceeds its byte limit");
|
|
1854
2411
|
return cached.bytes;
|
|
1855
2412
|
}
|
|
1856
|
-
|
|
2413
|
+
check5(this.files.size < MAX_PUBLIC_SOURCE_FILES, "Evidence file-count limit exceeded");
|
|
1857
2414
|
let directory = this.captures.get(captureId);
|
|
1858
2415
|
if (!directory) {
|
|
1859
2416
|
directory = await this.directory(this.rootDirectory, captureId, capturePath, true);
|
|
1860
2417
|
this.captures.set(captureId, directory);
|
|
1861
2418
|
}
|
|
1862
2419
|
const name = basename(path);
|
|
1863
|
-
const handle = await
|
|
2420
|
+
const handle = await open4(`/proc/self/fd/${directory.handle.fd}/${name}`, FILE_FLAGS);
|
|
1864
2421
|
try {
|
|
1865
2422
|
const before = await handle.stat({ bigint: true });
|
|
1866
|
-
|
|
1867
|
-
|
|
2423
|
+
check5(before.isFile(), "Evidence must be an ordinary file");
|
|
2424
|
+
check5(before.size <= BigInt(limit), "Evidence file exceeds its byte limit");
|
|
1868
2425
|
const size = Number(before.size);
|
|
1869
|
-
|
|
2426
|
+
check5(this.totalBytes + size <= MAX_PUBLIC_SOURCE_TOTAL_BYTES, "Evidence total-byte limit exceeded");
|
|
1870
2427
|
const buffer = Buffer.alloc(size + 1);
|
|
1871
2428
|
let length = 0;
|
|
1872
2429
|
while (length < buffer.length) {
|
|
@@ -1874,8 +2431,8 @@ var EvidenceReader = class {
|
|
|
1874
2431
|
if (!result.bytesRead) break;
|
|
1875
2432
|
length += result.bytesRead;
|
|
1876
2433
|
}
|
|
1877
|
-
|
|
1878
|
-
|
|
2434
|
+
check5(length === size, "Evidence file changed size while reading");
|
|
2435
|
+
check5(metadata(await handle.stat({ bigint: true })) === metadata(before), "Evidence file metadata changed while reading");
|
|
1879
2436
|
const bytes = buffer.subarray(0, length);
|
|
1880
2437
|
this.files.set(path, { handle, directory, name, metadata: metadata(before), bytes });
|
|
1881
2438
|
this.totalBytes += length;
|
|
@@ -1888,19 +2445,19 @@ var EvidenceReader = class {
|
|
|
1888
2445
|
async stable() {
|
|
1889
2446
|
for (const directory of this.directories) {
|
|
1890
2447
|
const current = await directory.handle.stat({ bigint: true });
|
|
1891
|
-
|
|
1892
|
-
const reopened = await
|
|
2448
|
+
check5(identity(current) === directory.identity && (!directory.stable || metadata(current) === directory.metadata), "Evidence directory changed during verification");
|
|
2449
|
+
const reopened = await open4(directory.parent ? `/proc/self/fd/${directory.parent.handle.fd}/${directory.component}` : "/", DIR_FLAGS);
|
|
1893
2450
|
try {
|
|
1894
|
-
|
|
2451
|
+
check5(identity(await reopened.stat({ bigint: true })) === directory.identity, "Evidence directory path changed during verification");
|
|
1895
2452
|
} finally {
|
|
1896
2453
|
await reopened.close();
|
|
1897
2454
|
}
|
|
1898
2455
|
}
|
|
1899
2456
|
for (const file of this.files.values()) {
|
|
1900
|
-
|
|
1901
|
-
const reopened = await
|
|
2457
|
+
check5(metadata(await file.handle.stat({ bigint: true })) === file.metadata, "Evidence file changed during verification");
|
|
2458
|
+
const reopened = await open4(`/proc/self/fd/${file.directory.handle.fd}/${file.name}`, FILE_FLAGS);
|
|
1902
2459
|
try {
|
|
1903
|
-
|
|
2460
|
+
check5(metadata(await reopened.stat({ bigint: true })) === file.metadata, "Evidence file path changed during verification");
|
|
1904
2461
|
} finally {
|
|
1905
2462
|
await reopened.close();
|
|
1906
2463
|
}
|
|
@@ -1912,221 +2469,221 @@ var EvidenceReader = class {
|
|
|
1912
2469
|
}
|
|
1913
2470
|
};
|
|
1914
2471
|
function requestedRow(row, index, tool) {
|
|
1915
|
-
|
|
1916
|
-
|
|
2472
|
+
keys5(row, ["ordinal", "agent", "requestedDefinitionId", "definitionId", "executionId"], "Requested row");
|
|
2473
|
+
check5(row.ordinal === index + 1, "Requested ordinals must be a contiguous ordered sequence");
|
|
1917
2474
|
string(row.agent, "Requested agent", true);
|
|
1918
|
-
|
|
1919
|
-
|
|
2475
|
+
hash3(row.requestedDefinitionId, "Requested definition identity", true);
|
|
2476
|
+
hash3(row.definitionId, "Observed definition identity", true);
|
|
1920
2477
|
string(row.executionId, "Execution identity", tool === "delegate_describe");
|
|
1921
|
-
|
|
1922
|
-
if (tool === "delegate_describe")
|
|
2478
|
+
check5(row.definitionId === null || row.agent !== null, "An observed definition must identify its agent");
|
|
2479
|
+
if (tool === "delegate_describe") check5(row.executionId === null && row.requestedDefinitionId === null && row.agent !== null && row.definitionId !== null, "Describe has a definition but no execution/requested definition identity");
|
|
1923
2480
|
}
|
|
1924
2481
|
function finalIdentity(final) {
|
|
1925
|
-
|
|
1926
|
-
|
|
2482
|
+
keys5(final, ["state", "sessionId", "messageId", "leafId", "sha256"], "Native final");
|
|
2483
|
+
check5(final.state === "complete", "Native final must be complete");
|
|
1927
2484
|
for (const key of ["sessionId", "messageId", "leafId"]) string(final[key], "Native final " + key);
|
|
1928
|
-
|
|
2485
|
+
hash3(final.sha256, "Native final hash");
|
|
1929
2486
|
}
|
|
1930
2487
|
function workerIdentity(value, executionId) {
|
|
1931
|
-
|
|
1932
|
-
|
|
2488
|
+
keys5(value, ["revision", "executionId", "nonce", "root", "rootDevice", "rootInode", "bootId", "pidNamespace", "helperPid", "helperStartTicks", "helperSha256", "workerPid", "ownershipPath", "receiptPath"], "Cleanup identity");
|
|
2489
|
+
check5(value.revision === 1 && value.executionId === executionId, "Cleanup identity must bind the requested execution");
|
|
1933
2490
|
for (const key of ["executionId", "nonce", "root", "rootDevice", "rootInode", "bootId", "pidNamespace", "helperStartTicks", "ownershipPath", "receiptPath"]) string(value[key], "Cleanup identity " + key);
|
|
1934
2491
|
integer(value.helperPid, "Helper pid", 1);
|
|
1935
2492
|
integer(value.workerPid, "Worker pid");
|
|
1936
|
-
|
|
2493
|
+
hash3(value.helperSha256, "Helper hash");
|
|
1937
2494
|
}
|
|
1938
2495
|
function cleanup(value, executionId) {
|
|
1939
2496
|
if (value === null) return;
|
|
1940
|
-
|
|
2497
|
+
object2(value, "Cleanup");
|
|
1941
2498
|
if (value.state === "not-started" || value.state === "unknown") {
|
|
1942
2499
|
const fields = ["state", "reason"];
|
|
1943
2500
|
if (value.state === "unknown" && Object.hasOwn(value, "identity")) fields.push("identity");
|
|
1944
|
-
|
|
2501
|
+
keys5(value, fields, "Cleanup");
|
|
1945
2502
|
publicText(value.reason, "Cleanup reason");
|
|
1946
2503
|
if (Object.hasOwn(value, "identity")) workerIdentity(value.identity, executionId);
|
|
1947
2504
|
return;
|
|
1948
2505
|
}
|
|
1949
|
-
|
|
1950
|
-
|
|
2506
|
+
keys5(value, ["state", "identity", "receipt"], "Cleanup");
|
|
2507
|
+
check5(value.state === "settled", "Unsupported cleanup state");
|
|
1951
2508
|
workerIdentity(value.identity, executionId);
|
|
1952
|
-
|
|
1953
|
-
|
|
2509
|
+
keys5(value.receipt, ["state", "identity", "workerCode", "workerSignal", "reason", "reapedAll"], "Cleanup receipt");
|
|
2510
|
+
check5(value.receipt.state === "settled" && value.receipt.reapedAll === true && isDeepStrictEqual2(value.receipt.identity, value.identity), "Cleanup receipt identity or settlement mismatch");
|
|
1954
2511
|
if (value.receipt.workerCode !== null) integer(value.receipt.workerCode, "Worker exit code");
|
|
1955
2512
|
integer(value.receipt.workerSignal, "Worker signal");
|
|
1956
2513
|
enumeration(value.receipt.reason, ["worker-exit", "owner-loss", "cancelled", "helper-signal", "ownership-write-failed", "start-failed"], "Cleanup receipt reason");
|
|
1957
2514
|
}
|
|
1958
2515
|
function runtimeOutcome(outcome, index, requested) {
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
2516
|
+
keys5(outcome, ["ordinal", "ok", "work", "control", "reason", "exitCode", "timedOut", "aborted", "truncated", "spawnFailed", "final", "cleanup", "observation", "retention"], "Runtime outcome");
|
|
2517
|
+
check5(outcome.ordinal === index + 1, "Runtime ordinals must follow requested order");
|
|
2518
|
+
check5(typeof outcome.ok === "boolean", "Runtime ok must be boolean");
|
|
1962
2519
|
enumeration(outcome.work, [null, "succeeded", "failed", "unknown"], "Work state");
|
|
1963
2520
|
enumeration(outcome.control, [null, "failed"], "Control state");
|
|
1964
2521
|
if (outcome.reason !== null) publicText(outcome.reason, "Runtime reason");
|
|
1965
2522
|
if (outcome.exitCode !== null) integer(outcome.exitCode, "Runtime exit code");
|
|
1966
|
-
for (const key of ["timedOut", "aborted", "truncated", "spawnFailed"])
|
|
2523
|
+
for (const key of ["timedOut", "aborted", "truncated", "spawnFailed"]) check5(outcome[key] === null || typeof outcome[key] === "boolean", "Runtime flags must be nullable booleans");
|
|
1967
2524
|
if (outcome.final !== null) {
|
|
1968
|
-
|
|
2525
|
+
object2(outcome.final, "Runtime final");
|
|
1969
2526
|
if (outcome.final.state === "unavailable") {
|
|
1970
|
-
|
|
2527
|
+
keys5(outcome.final, ["state", "reason"], "Unavailable final");
|
|
1971
2528
|
publicText(outcome.final.reason, "Unavailable final reason");
|
|
1972
2529
|
} else finalIdentity(outcome.final);
|
|
1973
2530
|
}
|
|
1974
2531
|
cleanup(outcome.cleanup, requested.executionId);
|
|
1975
2532
|
if (outcome.observation !== null) {
|
|
1976
|
-
|
|
2533
|
+
keys5(outcome.observation, ["state", "reasons"], "Observation");
|
|
1977
2534
|
enumeration(outcome.observation.state, ["complete", "incomplete"], "Observation state");
|
|
1978
2535
|
array(outcome.observation.reasons, "Observation reasons");
|
|
1979
2536
|
for (const reason of outcome.observation.reasons) publicText(reason, "Observation reason");
|
|
1980
2537
|
}
|
|
1981
2538
|
if (outcome.retention !== null) {
|
|
1982
|
-
|
|
2539
|
+
keys5(outcome.retention, ["status"], "Retention");
|
|
1983
2540
|
enumeration(outcome.retention.status, ["disabled", "pending", "retained", "lost"], "Retention status");
|
|
1984
2541
|
}
|
|
1985
2542
|
}
|
|
1986
2543
|
function binding(value) {
|
|
1987
2544
|
if (value === null) return;
|
|
1988
|
-
|
|
1989
|
-
|
|
2545
|
+
keys5(value, ["package", "phase"], "Definition binding");
|
|
2546
|
+
check5(value.package === "principal-pi-skills", "Unsupported definition binding package");
|
|
1990
2547
|
string(value.phase, "Definition phase");
|
|
1991
2548
|
}
|
|
1992
|
-
async function capture(reader,
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
const manifestBytes = await reader.read(
|
|
1996
|
-
|
|
1997
|
-
const manifest =
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2549
|
+
async function capture(reader, reference2) {
|
|
2550
|
+
keys5(reference2, ["path", "sha256"], "Manifest reference");
|
|
2551
|
+
hash3(reference2.sha256, "Manifest hash");
|
|
2552
|
+
const manifestBytes = await reader.read(reference2.path, MAX_PUBLIC_SOURCE_MANIFEST_BYTES);
|
|
2553
|
+
check5(digest(manifestBytes) === reference2.sha256, "Manifest hash mismatch");
|
|
2554
|
+
const manifest = json3(manifestBytes, "Manifest");
|
|
2555
|
+
keys5(manifest, ["schema", "version", "captureId", "ownerId", "toolCallId", "tool", "state", "response", "requested", "runtimeEvidence", "finals", "definitions"], "Public manifest");
|
|
2556
|
+
check5(manifest.schema === "pi-daddy-public-evidence-v1" && manifest.version === 1 && manifest.state === "returned", "Unsupported public manifest version or state");
|
|
2557
|
+
check5(UUID.test(manifest.captureId) && UUID.test(manifest.ownerId), "Capture/owner identity must be a UUID");
|
|
2001
2558
|
string(manifest.toolCallId, "Tool-call identity");
|
|
2002
2559
|
enumeration(manifest.tool, TOOLS, "Tool");
|
|
2003
|
-
const directory =
|
|
2004
|
-
|
|
2005
|
-
const refs = [{ kind: "manifest", ...
|
|
2006
|
-
const seen = /* @__PURE__ */ new Set([
|
|
2560
|
+
const directory = join6(reader.root, manifest.captureId);
|
|
2561
|
+
check5(reference2.path === join6(directory, "manifest.json"), "Manifest path does not bind its capture identity");
|
|
2562
|
+
const refs = [{ kind: "manifest", ...reference2, bytes: manifestBytes.length }];
|
|
2563
|
+
const seen = /* @__PURE__ */ new Set([reference2.path]);
|
|
2007
2564
|
async function copied(ref, filename, kind) {
|
|
2008
|
-
|
|
2009
|
-
|
|
2565
|
+
keys5(ref, ["path", "sha256", "bytes"], "Copied reference");
|
|
2566
|
+
hash3(ref.sha256, "Copied hash");
|
|
2010
2567
|
integer(ref.bytes, "Copied byte count");
|
|
2011
|
-
|
|
2568
|
+
check5(ref.path === join6(directory, filename) && !seen.has(ref.path), "Copied reference path is foreign or reused");
|
|
2012
2569
|
seen.add(ref.path);
|
|
2013
|
-
|
|
2570
|
+
check5(ref.bytes <= MAX_PUBLIC_SOURCE_FILE_BYTES, "Copied reference exceeds file-byte limit");
|
|
2014
2571
|
const bytes = await reader.read(ref.path);
|
|
2015
|
-
|
|
2572
|
+
check5(bytes.length === ref.bytes && digest(bytes) === ref.sha256, "Copied reference byte count or hash mismatch");
|
|
2016
2573
|
refs.push({ kind, ...ref });
|
|
2017
2574
|
return bytes;
|
|
2018
2575
|
}
|
|
2019
2576
|
array(manifest.requested, "Requested", 8);
|
|
2020
|
-
|
|
2021
|
-
if (manifest.tool === "delegate" || manifest.tool === "delegate_describe")
|
|
2577
|
+
check5(manifest.requested.length > 0, "Capture must identify a requested row");
|
|
2578
|
+
if (manifest.tool === "delegate" || manifest.tool === "delegate_describe") check5(manifest.requested.length === 1, "Single tool requires one requested row");
|
|
2022
2579
|
manifest.requested.forEach((row, index) => requestedRow(row, index, manifest.tool));
|
|
2023
2580
|
const executionIds = manifest.requested.map((row) => row.executionId).filter((value) => value !== null);
|
|
2024
|
-
|
|
2025
|
-
const response =
|
|
2026
|
-
|
|
2027
|
-
|
|
2581
|
+
check5(new Set(executionIds).size === executionIds.length, "Requested execution identities must be unique");
|
|
2582
|
+
const response = json3(await copied(manifest.response, "response.json", "response"), "Public response");
|
|
2583
|
+
keys5(response, ["isError", "content"], "Public response");
|
|
2584
|
+
check5(typeof response.isError === "boolean", "Public isError must be boolean");
|
|
2028
2585
|
array(response.content, "Public content", 2);
|
|
2029
2586
|
for (const block of response.content) {
|
|
2030
|
-
|
|
2031
|
-
|
|
2587
|
+
keys5(block, ["type", "text"], "Public block");
|
|
2588
|
+
check5(block.type === "text" && typeof block.text === "string", "Only public text blocks are supported");
|
|
2032
2589
|
}
|
|
2033
2590
|
array(manifest.finals, "Final copies", 8);
|
|
2034
2591
|
array(manifest.definitions, "Definition copies", 8);
|
|
2035
2592
|
let outcomes = [];
|
|
2036
2593
|
if (manifest.tool === "delegate_describe") {
|
|
2037
|
-
|
|
2594
|
+
check5(manifest.runtimeEvidence === null && manifest.finals.length === 0 && response.content.length === 1 && response.isError === false, "Describe capture has no runtime/final outcome");
|
|
2038
2595
|
} else {
|
|
2039
2596
|
const runtime = manifest.runtimeEvidence;
|
|
2040
|
-
|
|
2041
|
-
|
|
2597
|
+
keys5(runtime, ["version", "tool", "requested", "outcomes"], "Runtime evidence");
|
|
2598
|
+
check5(runtime.version === 1 && runtime.tool === manifest.tool && runtime.requested === manifest.requested.length, "Runtime header does not match capture");
|
|
2042
2599
|
array(runtime.outcomes, "Runtime outcomes", 8);
|
|
2043
2600
|
outcomes = runtime.outcomes;
|
|
2044
|
-
|
|
2045
|
-
if (manifest.tool !== "delegate_chain")
|
|
2601
|
+
check5(outcomes.length > 0 && outcomes.length <= manifest.requested.length, "Runtime outcome count is invalid");
|
|
2602
|
+
if (manifest.tool !== "delegate_chain") check5(outcomes.length === manifest.requested.length, "Non-chain captures must retain every outcome");
|
|
2046
2603
|
outcomes.forEach((outcome, index) => runtimeOutcome(outcome, index, manifest.requested[index]));
|
|
2047
|
-
|
|
2048
|
-
const projection =
|
|
2049
|
-
|
|
2604
|
+
check5(response.content.length === 2 && response.content[1].text.startsWith(RUNTIME_PREFIX) && response.content[1].text.endsWith("\n```"), "Public response lacks its exact runtime evidence block");
|
|
2605
|
+
const projection = json3(Buffer.from(response.content[1].text.slice(RUNTIME_PREFIX.length, -4)), "Public runtime projection");
|
|
2606
|
+
check5(isDeepStrictEqual2(projection, runtime), "Public runtime projection differs from manifest");
|
|
2050
2607
|
const failed = outcomes.some((outcome) => !outcome.ok || outcome.control === "failed");
|
|
2051
|
-
if (manifest.tool !== "delegate_chain")
|
|
2052
|
-
else if (failed || outcomes.length < manifest.requested.length || outcomes.some((outcome) => outcome.final?.state === "unavailable"))
|
|
2053
|
-
|
|
2608
|
+
if (manifest.tool !== "delegate_chain") check5(response.isError === failed, "Public isError differs from runtime outcome");
|
|
2609
|
+
else if (failed || outcomes.length < manifest.requested.length || outcomes.some((outcome) => outcome.final?.state === "unavailable")) check5(response.isError, "Stopped chain must remain an error response");
|
|
2610
|
+
check5(manifest.finals.length === outcomes.length, "Every returned outcome must retain its final availability");
|
|
2054
2611
|
}
|
|
2055
2612
|
for (const [index, row] of manifest.finals.entries()) {
|
|
2056
|
-
|
|
2057
|
-
|
|
2613
|
+
keys5(row, ["ordinal", "executionId", "final"], "Final copy row");
|
|
2614
|
+
check5(row.ordinal === index + 1 && row.executionId === manifest.requested[index].executionId, "Final copy identity differs from requested execution");
|
|
2058
2615
|
const final = outcomes[index].final;
|
|
2059
2616
|
if (final?.state !== "complete") {
|
|
2060
|
-
|
|
2617
|
+
check5(row.final === null, "Unavailable native final cannot have a complete copy");
|
|
2061
2618
|
continue;
|
|
2062
2619
|
}
|
|
2063
|
-
|
|
2620
|
+
keys5(row.final, ["state", "sessionId", "messageId", "leafId", "sha256", "content"], "Final copy");
|
|
2064
2621
|
const { content, ...identity2 } = row.final;
|
|
2065
|
-
|
|
2622
|
+
check5(isDeepStrictEqual2(identity2, final), "Final copy identity differs from runtime final");
|
|
2066
2623
|
const bytes = await copied(content, `outcome-${row.ordinal}-final.txt`, "final");
|
|
2067
|
-
|
|
2068
|
-
|
|
2624
|
+
check5(bytes.length > 0 && digest(bytes) === final.sha256, "Final copy differs from native final hash");
|
|
2625
|
+
check5(Buffer.from(response.content[0].text, "utf8").includes(bytes), "Complete final bytes are absent from authored public content");
|
|
2069
2626
|
}
|
|
2070
2627
|
const expectedDefinitions = manifest.requested.filter((row) => row.definitionId !== null).map((row) => JSON.stringify([row.agent, row.definitionId])).sort();
|
|
2071
2628
|
const observedDefinitions = [];
|
|
2072
2629
|
for (const [index, definition] of manifest.definitions.entries()) {
|
|
2073
|
-
|
|
2630
|
+
keys5(definition, ["agent", "definitionId", "sourceHash", "bodySha256", "binding", "resources", "body"], "Definition copy");
|
|
2074
2631
|
string(definition.agent, "Definition agent");
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2632
|
+
hash3(definition.definitionId, "Definition identity");
|
|
2633
|
+
hash3(definition.sourceHash, "Definition source hash", true);
|
|
2634
|
+
hash3(definition.bodySha256, "Definition body hash");
|
|
2078
2635
|
binding(definition.binding);
|
|
2079
2636
|
observedDefinitions.push(JSON.stringify([definition.agent, definition.definitionId]));
|
|
2080
2637
|
array(definition.resources, "Definition resources");
|
|
2081
|
-
|
|
2638
|
+
check5(definition.resources.length > 0, "Definition source resources are missing");
|
|
2082
2639
|
for (const [resourceIndex, resource] of definition.resources.entries()) {
|
|
2083
|
-
|
|
2640
|
+
keys5(resource, ["kind", "path", "copy"], "Definition resource");
|
|
2084
2641
|
enumeration(resource.kind, ["selected-skill", "package", "binding-manifest", "delegated-agent"], "Source kind");
|
|
2085
2642
|
string(resource.path, "Original source identity");
|
|
2086
2643
|
await copied(resource.copy, `definition-${index}-source-${resourceIndex}.bin`, "source-copy");
|
|
2087
2644
|
}
|
|
2088
2645
|
const body = await copied(definition.body, `definition-${index}-body.txt`, "definition-body");
|
|
2089
|
-
|
|
2646
|
+
check5(digest(body) === definition.bodySha256, "Definition body copy differs from bodySha256");
|
|
2090
2647
|
}
|
|
2091
|
-
|
|
2648
|
+
check5(isDeepStrictEqual2(observedDefinitions.sort(), expectedDefinitions), "Copied definitions differ from observed requested definitions");
|
|
2092
2649
|
if (manifest.tool === "delegate_describe") {
|
|
2093
|
-
const described =
|
|
2650
|
+
const described = json3(Buffer.from(response.content[0].text), "Described definition");
|
|
2094
2651
|
const definition = manifest.definitions[0];
|
|
2095
2652
|
const expected = { version: 1, agent: definition.agent, bodySha256: definition.bodySha256, binding: definition.binding, definitionId: definition.definitionId };
|
|
2096
2653
|
if (Object.hasOwn(described, "sourceHash")) expected.sourceHash = definition.sourceHash;
|
|
2097
|
-
else
|
|
2098
|
-
|
|
2654
|
+
else check5(definition.sourceHash === null, "Described source hash is missing");
|
|
2655
|
+
check5(isDeepStrictEqual2(described, expected), "Described definition differs from manifest source/body identity");
|
|
2099
2656
|
}
|
|
2100
2657
|
return { manifest, refs, responseIsError: response.isError, outcomes };
|
|
2101
2658
|
}
|
|
2102
2659
|
async function verifySources(cases, selection, evidenceRoot) {
|
|
2103
|
-
|
|
2660
|
+
check5(process.platform === "linux", "Public source verification requires Linux held-directory-FD reads");
|
|
2104
2661
|
canonicalPath(evidenceRoot, "Evidence root");
|
|
2105
|
-
|
|
2662
|
+
check5(evidenceRoot !== "/", "Evidence root cannot be the filesystem root");
|
|
2106
2663
|
array(cases, "Cases", 100);
|
|
2107
2664
|
const raw = cases.map((item) => {
|
|
2108
|
-
|
|
2665
|
+
keys5(item, ["id", "input", "question", "provenance", "source", "visibility", "hash"], "Parsed case");
|
|
2109
2666
|
const { hash: ignored, ...original } = item;
|
|
2110
2667
|
return original;
|
|
2111
2668
|
});
|
|
2112
2669
|
const validated = parseCases({ schema: 1, cases: raw });
|
|
2113
|
-
validated.forEach((item, index) =>
|
|
2114
|
-
|
|
2115
|
-
|
|
2670
|
+
validated.forEach((item, index) => check5(item.hash === cases[index].hash, "Incoming parsed case hash is stale"));
|
|
2671
|
+
keys5(selection, ["schema", "kind", "selections"], "Source selection");
|
|
2672
|
+
check5(selection.schema === 1 && selection.kind === "decision-public-sources", "Unsupported source selection version");
|
|
2116
2673
|
array(selection.selections, "Selections", 100);
|
|
2117
|
-
|
|
2674
|
+
check5(selection.selections.length === validated.length, "Require exactly one ordered selection per case");
|
|
2118
2675
|
for (const [index, row] of selection.selections.entries()) {
|
|
2119
|
-
|
|
2676
|
+
keys5(row, ["caseId", "caseHash", "manifest", "captureId", "toolCallId", "ordinal", "agent", "definitionId", "executionId"], "Source selection row");
|
|
2120
2677
|
const item = validated[index];
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2678
|
+
check5(row.caseId === item.id && row.caseHash === item.hash, "Selections must match exact ordered case identities");
|
|
2679
|
+
keys5(row.manifest, ["path", "sha256"], "Manifest selection");
|
|
2680
|
+
hash3(row.manifest.sha256, "Selected manifest hash");
|
|
2681
|
+
check5(item.source.sha256 === row.manifest.sha256 && item.source.recordId === row.toolCallId, "Case source must bind the manifest hash and tool-call identity");
|
|
2682
|
+
check5(UUID.test(row.captureId), "Selected capture identity must be a UUID");
|
|
2126
2683
|
string(row.toolCallId, "Selected tool-call identity");
|
|
2127
2684
|
integer(row.ordinal, "Selected ordinal", 1);
|
|
2128
2685
|
string(row.agent, "Selected agent", true);
|
|
2129
|
-
|
|
2686
|
+
hash3(row.definitionId, "Selected observed definition identity", true);
|
|
2130
2687
|
string(row.executionId, "Selected execution identity", true);
|
|
2131
2688
|
}
|
|
2132
2689
|
const reader = new EvidenceReader(evidenceRoot);
|
|
@@ -2139,11 +2696,11 @@ async function verifySources(cases, selection, evidenceRoot) {
|
|
|
2139
2696
|
verified = await capture(reader, row.manifest);
|
|
2140
2697
|
captures.set(row.manifest.path, verified);
|
|
2141
2698
|
}
|
|
2142
|
-
|
|
2699
|
+
check5(verified.refs[0].sha256 === row.manifest.sha256, "Repeated manifest selections disagree on hash");
|
|
2143
2700
|
const { manifest, refs, outcomes } = verified;
|
|
2144
|
-
|
|
2701
|
+
check5(manifest.captureId === row.captureId && manifest.toolCallId === row.toolCallId, "Selected capture/tool-call identity mismatch");
|
|
2145
2702
|
const requested = manifest.requested[row.ordinal - 1];
|
|
2146
|
-
|
|
2703
|
+
check5(requested !== void 0 && ["ordinal", "agent", "definitionId", "executionId"].every((key) => requested[key] === row[key]), "Selected requested row identity mismatch");
|
|
2147
2704
|
const outcome = outcomes[row.ordinal - 1];
|
|
2148
2705
|
rows.push({
|
|
2149
2706
|
caseId: row.caseId,
|
|
@@ -2178,7 +2735,7 @@ async function verifySources(cases, selection, evidenceRoot) {
|
|
|
2178
2735
|
status: "verified-bytes-and-record-identities",
|
|
2179
2736
|
platform: "linux",
|
|
2180
2737
|
evidenceRoot,
|
|
2181
|
-
limits: { ...
|
|
2738
|
+
limits: { ...LIMITS2 },
|
|
2182
2739
|
artifactCount: reader.files.size,
|
|
2183
2740
|
totalBytes: reader.totalBytes,
|
|
2184
2741
|
captureCount: captures.size,
|
|
@@ -2212,7 +2769,9 @@ var help = `Decision research workflow (Node >=20; qualified Pi execution needs
|
|
|
2212
2769
|
preview-pi --cases FILE --experiment FILE --model openai-codex:MODEL --thinking LEVEL --split test
|
|
2213
2770
|
run-pi --cases FILE --experiment FILE --model openai-codex:MODEL --thinking LEVEL --split test --pi-package DIR --out NEW.jsonl --allow-subscription [--pi-node PATH] [--auth-path FILE] [--timeout-ms N]
|
|
2214
2771
|
compare --cases FILE --experiment FILE --labels FILE --label-evidence FILE --run FILE [--run FILE] --out NEW.json
|
|
2772
|
+
collections [--root ABSOLUTE_DIR]
|
|
2215
2773
|
import-session --cases FILE --selection FILE --consent FILE --entry FILE --out NEW.json
|
|
2774
|
+
import-workflow --cases FILE --selection FILE --consent FILE --entry FILE --out NEW.json
|
|
2216
2775
|
export-learning --cases FILE --experiment FILE --labels FILE --label-evidence FILE --consents FILE --mode fixture-demo|reviewed-data --out NEW_DIR
|
|
2217
2776
|
|
|
2218
2777
|
JEV run asks this session whether to retain selected data for later LoRA review.
|
|
@@ -2230,6 +2789,7 @@ function argumentsFor(argv) {
|
|
|
2230
2789
|
if (!command || command === "--help") return { command: "help", flags: {} };
|
|
2231
2790
|
const commands = {
|
|
2232
2791
|
...RESEARCH_OPTIONS,
|
|
2792
|
+
collections: [],
|
|
2233
2793
|
preview: ["cases", "provider", "model"],
|
|
2234
2794
|
run: ["cases", "provider", "model", "out", "allow-remote"],
|
|
2235
2795
|
score: ["cases", "labels", "run"],
|
|
@@ -2239,7 +2799,7 @@ function argumentsFor(argv) {
|
|
|
2239
2799
|
if (!Object.hasOwn(commands, command))
|
|
2240
2800
|
throw new Error("Unknown command; use --help.");
|
|
2241
2801
|
const required = commands[command];
|
|
2242
|
-
const optional = command === "run" ? ["storage", "session-id"] : RESEARCH_OPTIONAL[command] ?? [];
|
|
2802
|
+
const optional = command === "run" ? ["storage", "session-id"] : command === "collections" ? ["root"] : RESEARCH_OPTIONAL[command] ?? [];
|
|
2243
2803
|
const allowed = [...required, ...optional];
|
|
2244
2804
|
const flags = {};
|
|
2245
2805
|
for (let i = 0; i < rest.length; i++) {
|
|
@@ -2278,21 +2838,21 @@ async function textFile(path) {
|
|
|
2278
2838
|
}
|
|
2279
2839
|
}
|
|
2280
2840
|
async function jsonFile(path) {
|
|
2281
|
-
const
|
|
2841
|
+
const text4 = await textFile(path);
|
|
2282
2842
|
try {
|
|
2283
|
-
return JSON.parse(
|
|
2843
|
+
return JSON.parse(text4);
|
|
2284
2844
|
} catch {
|
|
2285
2845
|
throw new Error("Invalid JSON input file.");
|
|
2286
2846
|
}
|
|
2287
2847
|
}
|
|
2288
|
-
function
|
|
2289
|
-
return
|
|
2848
|
+
function sha2562(text4) {
|
|
2849
|
+
return createHash7("sha256").update(text4, "utf8").digest("hex");
|
|
2290
2850
|
}
|
|
2291
2851
|
function caseSetHash(cases) {
|
|
2292
|
-
return
|
|
2852
|
+
return sha2562(JSON.stringify(cases.map((c) => ({ id: c.id, hash: c.hash }))));
|
|
2293
2853
|
}
|
|
2294
2854
|
function labelSetHash(labels) {
|
|
2295
|
-
return
|
|
2855
|
+
return sha2562(
|
|
2296
2856
|
JSON.stringify(
|
|
2297
2857
|
[...labels].sort(
|
|
2298
2858
|
(a, b) => a.caseId < b.caseId ? -1 : a.caseId > b.caseId ? 1 : 0
|
|
@@ -2305,26 +2865,35 @@ function validCreatedAt(value) {
|
|
|
2305
2865
|
}
|
|
2306
2866
|
async function readRun(path, cases) {
|
|
2307
2867
|
let rows;
|
|
2308
|
-
const
|
|
2868
|
+
const text4 = await textFile(path);
|
|
2309
2869
|
try {
|
|
2310
|
-
rows =
|
|
2870
|
+
rows = text4.trimEnd().split("\n").map((line) => JSON.parse(line));
|
|
2311
2871
|
} catch {
|
|
2312
2872
|
throw new Error("Invalid run JSONL; retain damaged evidence separately.");
|
|
2313
2873
|
}
|
|
2314
2874
|
const [header, ...records] = rows;
|
|
2315
|
-
if (header?.schema
|
|
2875
|
+
if (!(header?.schema === 1 && header.kind === "decision-shadow-run" || header?.schema === 2 && header.kind === "decision-shadow-native-run") || header.caseSetHash !== caseSetHash(cases) || header.caseCount !== cases.length || header.trainingEligible !== false || !validCreatedAt(header.createdAt))
|
|
2316
2876
|
throw new Error("Run header does not match this case set.");
|
|
2317
|
-
|
|
2877
|
+
const native = header.schema === 2;
|
|
2878
|
+
if (native) {
|
|
2879
|
+
validateNativeRoute(header.route);
|
|
2880
|
+
if (header.provider !== header.route.provider || header.requestedModel !== header.route.model || header.requestFormat !== "pi-classifier-context-v1" || header.costSource !== "pi-catalog-estimate")
|
|
2881
|
+
throw new Error("Native run route does not match its header.");
|
|
2882
|
+
} else {
|
|
2883
|
+
if (!Object.hasOwn(terms, header.provider)) throw new Error("Invalid run provider.");
|
|
2884
|
+
makeRequest(header.provider, header.requestedModel, cases[0]);
|
|
2885
|
+
}
|
|
2318
2886
|
if (records.length > cases.length)
|
|
2319
2887
|
throw new Error("Too many prediction records.");
|
|
2320
2888
|
for (const [index, r] of records.entries()) {
|
|
2321
|
-
const
|
|
2889
|
+
const { kind, caseId, caseHash, requestSha256, trainingEligible, ...prediction } = r ?? {};
|
|
2890
|
+
const validState = native ? validateNativePrediction(header.route, prediction) && r.requestSha256 === nativeRequestSha256(cases[index]) && (r.status !== "error" || index === records.length - 1) : r?.status === "answered" ? isSupportedResolvedModel(r.resolvedModel) && r.error === null : r?.status === "error" && r.probability === null && providerErrors.has(r.error) && (r.resolvedModel === null || isSupportedResolvedModel(r.resolvedModel)) && index === records.length - 1;
|
|
2322
2891
|
if (!r || r.kind !== "prediction" || r.trainingEligible !== false || r.caseId !== cases[index].id || r.caseHash !== cases[index].hash || !validState || !nullableMetric(r.latencyMs) || !r.usage || !["inputTokens", "outputTokens"].every(
|
|
2323
2892
|
(key) => nullableTokenCount(r.usage[key])
|
|
2324
2893
|
) || !nullableMetric(r.usage.costUsd))
|
|
2325
2894
|
throw new Error("Invalid prediction record.");
|
|
2326
2895
|
}
|
|
2327
|
-
return { header, records, runSha256:
|
|
2896
|
+
return { header, records, runSha256: sha2562(text4) };
|
|
2328
2897
|
}
|
|
2329
2898
|
function nullableMetric(value) {
|
|
2330
2899
|
return value === null || typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
@@ -2371,7 +2940,7 @@ function measurements(records) {
|
|
|
2371
2940
|
};
|
|
2372
2941
|
}
|
|
2373
2942
|
async function writeNew(path, value) {
|
|
2374
|
-
const file = await
|
|
2943
|
+
const file = await open5(path, "wx", 384);
|
|
2375
2944
|
try {
|
|
2376
2945
|
await file.writeFile(JSON.stringify(value, null, 2) + "\n");
|
|
2377
2946
|
await file.sync();
|
|
@@ -2387,13 +2956,20 @@ async function main(argv, {
|
|
|
2387
2956
|
piRunner,
|
|
2388
2957
|
sessionConsent,
|
|
2389
2958
|
expectedCaseSetHash,
|
|
2390
|
-
beforeProviderCall
|
|
2959
|
+
beforeProviderCall,
|
|
2960
|
+
nativeRoute,
|
|
2961
|
+
nativeProviderCall,
|
|
2962
|
+
learningRoot
|
|
2391
2963
|
} = {}) {
|
|
2392
2964
|
const { command, flags } = argumentsFor(argv);
|
|
2393
2965
|
if (command === "help") {
|
|
2394
2966
|
emit(help);
|
|
2395
2967
|
return;
|
|
2396
2968
|
}
|
|
2969
|
+
if (command === "collections") {
|
|
2970
|
+
emit(JSON.stringify(await listCollections(resolveDataRoot({ env, override: flags.root })), null, 2));
|
|
2971
|
+
return;
|
|
2972
|
+
}
|
|
2397
2973
|
const cases = command === "fixtures" ? null : parseCases(await jsonFile(flags.cases));
|
|
2398
2974
|
if (Object.hasOwn(RESEARCH_OPTIONS, command))
|
|
2399
2975
|
return researchCommand(
|
|
@@ -2419,12 +2995,12 @@ async function main(argv, {
|
|
|
2419
2995
|
...verification,
|
|
2420
2996
|
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2421
2997
|
caseSetHash: caseSetHash(cases),
|
|
2422
|
-
selectionFileSha256:
|
|
2998
|
+
selectionFileSha256: sha2562(sourceText)
|
|
2423
2999
|
};
|
|
2424
3000
|
await writeNew(flags.out, receipt);
|
|
2425
3001
|
emit(
|
|
2426
3002
|
JSON.stringify({
|
|
2427
|
-
saved:
|
|
3003
|
+
saved: resolve3(flags.out),
|
|
2428
3004
|
status: receipt.status,
|
|
2429
3005
|
caseCount: cases.length,
|
|
2430
3006
|
artifactCount: receipt.artifactCount,
|
|
@@ -2437,14 +3013,22 @@ async function main(argv, {
|
|
|
2437
3013
|
if (command === "preview" || command === "run") {
|
|
2438
3014
|
if (expectedCaseSetHash !== void 0 && caseSetHash(cases) !== expectedCaseSetHash)
|
|
2439
3015
|
throw new Error("Cases changed after confirmation.");
|
|
3016
|
+
if (nativeRoute !== void 0) {
|
|
3017
|
+
validateNativeRoute(nativeRoute);
|
|
3018
|
+
if (flags.provider !== nativeRoute.provider || flags.model !== nativeRoute.model)
|
|
3019
|
+
throw new Error("Selected native provider/model changed.");
|
|
3020
|
+
if (command === "run" && typeof nativeProviderCall !== "function")
|
|
3021
|
+
throw new Error("Native classifier runner is unavailable.");
|
|
3022
|
+
}
|
|
2440
3023
|
const requests = cases.map(
|
|
2441
|
-
(c) => makeRequest(flags.provider, flags.model, c)
|
|
3024
|
+
(c) => nativeRoute ? { route: nativeRoute, context: makeNativeRequest(c), requestSha256: nativeRequestSha256(c) } : makeRequest(flags.provider, flags.model, c)
|
|
2442
3025
|
);
|
|
2443
3026
|
if (command === "preview") {
|
|
2444
3027
|
emit(
|
|
2445
3028
|
JSON.stringify(
|
|
2446
3029
|
{
|
|
2447
|
-
schema: 1,
|
|
3030
|
+
schema: nativeRoute ? 2 : 1,
|
|
3031
|
+
...nativeRoute ? { requestFormat: "pi-classifier-context-v1" } : {},
|
|
2448
3032
|
caseSetHash: caseSetHash(cases),
|
|
2449
3033
|
count: cases.length,
|
|
2450
3034
|
trainingEligible: false,
|
|
@@ -2460,18 +3044,33 @@ async function main(argv, {
|
|
|
2460
3044
|
);
|
|
2461
3045
|
return;
|
|
2462
3046
|
}
|
|
3047
|
+
const dataRoot = resolveDataRoot({ env, override: learningRoot });
|
|
2463
3048
|
const consent = sessionConsent ?? await requestSessionStorage({
|
|
2464
3049
|
sessionId: flags["session-id"],
|
|
2465
3050
|
storage: flags.storage,
|
|
2466
|
-
promptStorage
|
|
3051
|
+
promptStorage,
|
|
3052
|
+
storageRoot: dataRoot
|
|
2467
3053
|
});
|
|
2468
3054
|
const apiKey = env.OPENROUTER_API_KEY;
|
|
2469
|
-
if (!apiKey?.trim())
|
|
3055
|
+
if (!nativeRoute && !apiKey?.trim())
|
|
2470
3056
|
throw new Error("Selected provider API key is not configured.");
|
|
2471
|
-
const file = await
|
|
3057
|
+
const file = await open5(flags.out, "wx", 384);
|
|
3058
|
+
let collection, collectionFd, collectionStatus = "disabled";
|
|
2472
3059
|
const append = async (row) => {
|
|
2473
|
-
|
|
3060
|
+
const bytes = JSON.stringify(row) + "\n";
|
|
3061
|
+
await file.writeFile(bytes);
|
|
2474
3062
|
await file.sync();
|
|
3063
|
+
if (collectionFd !== void 0) {
|
|
3064
|
+
try {
|
|
3065
|
+
beforeProviderCall?.();
|
|
3066
|
+
writeFileSync3(collectionFd, bytes);
|
|
3067
|
+
fsyncSync3(collectionFd);
|
|
3068
|
+
} catch {
|
|
3069
|
+
closeSync3(collectionFd);
|
|
3070
|
+
collectionFd = void 0;
|
|
3071
|
+
collectionStatus = "incomplete";
|
|
3072
|
+
}
|
|
3073
|
+
}
|
|
2475
3074
|
};
|
|
2476
3075
|
let failures = 0;
|
|
2477
3076
|
try {
|
|
@@ -2483,26 +3082,45 @@ async function main(argv, {
|
|
|
2483
3082
|
out: flags.out + ".learning.json",
|
|
2484
3083
|
assertCurrent: beforeProviderCall
|
|
2485
3084
|
});
|
|
3085
|
+
if (consent.decision === "granted") {
|
|
3086
|
+
beforeProviderCall?.();
|
|
3087
|
+
collection = createSessionCollection(dataRoot, "jev-manual", consent.sessionId);
|
|
3088
|
+
writePrivateNew(join7(collection, "consent.json"), JSON.stringify(consent, null, 2) + "\n");
|
|
3089
|
+
await retainSelectedSessionData({
|
|
3090
|
+
cases,
|
|
3091
|
+
consent,
|
|
3092
|
+
out: join7(collection, "selected-data.json"),
|
|
3093
|
+
assertCurrent: beforeProviderCall
|
|
3094
|
+
});
|
|
3095
|
+
beforeProviderCall?.();
|
|
3096
|
+
collectionFd = openSync3(join7(collection, "results.jsonl"), "wx", 384);
|
|
3097
|
+
collectionStatus = "captured";
|
|
3098
|
+
}
|
|
2486
3099
|
await append({
|
|
2487
|
-
schema: 1,
|
|
2488
|
-
kind: "decision-shadow-run",
|
|
3100
|
+
schema: nativeRoute ? 2 : 1,
|
|
3101
|
+
kind: nativeRoute ? "decision-shadow-native-run" : "decision-shadow-run",
|
|
3102
|
+
...nativeRoute ? {
|
|
3103
|
+
route: nativeRoute,
|
|
3104
|
+
requestFormat: "pi-classifier-context-v1",
|
|
3105
|
+
costSource: "pi-catalog-estimate"
|
|
3106
|
+
} : {},
|
|
2489
3107
|
provider: flags.provider,
|
|
2490
3108
|
requestedModel: flags.model,
|
|
2491
3109
|
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2492
3110
|
caseCount: cases.length,
|
|
2493
3111
|
caseSetHash: caseSetHash(cases),
|
|
2494
3112
|
trainingEligible: false,
|
|
2495
|
-
termsUrl: terms[flags.provider]
|
|
3113
|
+
...nativeRoute ? {} : { termsUrl: terms[flags.provider] }
|
|
2496
3114
|
});
|
|
2497
3115
|
for (const c of cases) {
|
|
2498
3116
|
let result;
|
|
2499
3117
|
try {
|
|
2500
3118
|
if (beforeProviderCall) beforeProviderCall();
|
|
2501
|
-
result = await providerCall(flags.provider, flags.model, c, {
|
|
2502
|
-
|
|
2503
|
-
|
|
3119
|
+
result = nativeRoute ? await nativeProviderCall(c) : await providerCall(flags.provider, flags.model, c, { apiKey });
|
|
3120
|
+
if (nativeRoute && !validateNativePrediction(nativeRoute, result))
|
|
3121
|
+
throw new Error("Invalid native classifier result.");
|
|
2504
3122
|
} catch {
|
|
2505
|
-
result = {
|
|
3123
|
+
result = nativeRoute ? normalizeNativeResult(nativeRoute, null, null) : {
|
|
2506
3124
|
status: "error",
|
|
2507
3125
|
probability: null,
|
|
2508
3126
|
resolvedModel: null,
|
|
@@ -2516,6 +3134,7 @@ async function main(argv, {
|
|
|
2516
3134
|
caseId: c.id,
|
|
2517
3135
|
caseHash: c.hash,
|
|
2518
3136
|
...result,
|
|
3137
|
+
...nativeRoute ? { requestSha256: nativeRequestSha256(c) } : {},
|
|
2519
3138
|
trainingEligible: false
|
|
2520
3139
|
});
|
|
2521
3140
|
if (result.status === "error") {
|
|
@@ -2524,11 +3143,13 @@ async function main(argv, {
|
|
|
2524
3143
|
}
|
|
2525
3144
|
}
|
|
2526
3145
|
} finally {
|
|
3146
|
+
if (collectionFd !== void 0) closeSync3(collectionFd);
|
|
2527
3147
|
await file.close();
|
|
2528
3148
|
}
|
|
2529
3149
|
emit(
|
|
2530
3150
|
JSON.stringify({
|
|
2531
|
-
saved:
|
|
3151
|
+
saved: resolve3(flags.out),
|
|
3152
|
+
learningCollection: { status: collectionStatus, ...collection ? { path: collection } : {} },
|
|
2532
3153
|
errors: failures,
|
|
2533
3154
|
trainingEligible: false
|
|
2534
3155
|
})
|
|
@@ -2542,7 +3163,7 @@ async function main(argv, {
|
|
|
2542
3163
|
const labels = parseLabels(await jsonFile(flags.labels), cases);
|
|
2543
3164
|
if (command === "corpus") {
|
|
2544
3165
|
await writeNew(flags.out, localCorpus(cases, labels));
|
|
2545
|
-
emit(JSON.stringify({ saved:
|
|
3166
|
+
emit(JSON.stringify({ saved: resolve3(flags.out), trainingReady: false }));
|
|
2546
3167
|
return;
|
|
2547
3168
|
}
|
|
2548
3169
|
const reports = [];
|
|
@@ -2559,6 +3180,12 @@ async function main(argv, {
|
|
|
2559
3180
|
reports.push({
|
|
2560
3181
|
provider: header.provider,
|
|
2561
3182
|
requestedModel: header.requestedModel,
|
|
3183
|
+
...header.schema === 2 ? {
|
|
3184
|
+
route: header.route,
|
|
3185
|
+
requestFormat: header.requestFormat,
|
|
3186
|
+
costSource: header.costSource,
|
|
3187
|
+
modelResolution: "Pi returns the requested model identity; backend resolution is unknown."
|
|
3188
|
+
} : {},
|
|
2562
3189
|
resolvedModels,
|
|
2563
3190
|
runSha256,
|
|
2564
3191
|
createdAt: header.createdAt,
|