@openpond/evals 0.11.21 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/README.md +12 -0
  2. package/dist/connected-evidence/codex-command-observations.js +81 -0
  3. package/dist/connected-evidence/contracts.js +13 -0
  4. package/dist/connected-evidence/git-file-contracts.js +16 -0
  5. package/dist/connected-evidence/git-file-proof.js +86 -0
  6. package/dist/connected-evidence/git-starting-state.js +81 -0
  7. package/dist/connected-evidence/git.js +2 -0
  8. package/dist/connected-evidence/imports.js +2 -1
  9. package/dist/connected-evidence/index.js +1 -0
  10. package/dist/connected-evidence/normalize.js +1 -0
  11. package/dist/connected-evidence/sources/codex-metadata.js +33 -0
  12. package/dist/connected-evidence/sources/codex.js +16 -2
  13. package/dist/graders.js +10 -4
  14. package/dist/javascript-verifier.js +1 -0
  15. package/dist/native-conversations/codex-window-inventory.js +6 -7
  16. package/dist/native-conversations/git-assets.js +61 -0
  17. package/dist/native-conversations/index.js +1 -0
  18. package/dist/prepared-process-isolation.js +131 -0
  19. package/dist/rewards.js +17 -5
  20. package/dist/tasksets.js +6 -0
  21. package/dist/types/builtin-benchmarks/harness-refiner.d.ts +4 -0
  22. package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
  23. package/dist/types/conformance.d.ts +8 -0
  24. package/dist/types/conformance.d.ts.map +1 -1
  25. package/dist/types/connected-evidence/codex-command-observations.d.ts +15 -0
  26. package/dist/types/connected-evidence/codex-command-observations.d.ts.map +1 -0
  27. package/dist/types/connected-evidence/contracts.d.ts +22 -0
  28. package/dist/types/connected-evidence/contracts.d.ts.map +1 -1
  29. package/dist/types/connected-evidence/git-file-contracts.d.ts +18 -0
  30. package/dist/types/connected-evidence/git-file-contracts.d.ts.map +1 -0
  31. package/dist/types/connected-evidence/git-file-proof.d.ts +13 -0
  32. package/dist/types/connected-evidence/git-file-proof.d.ts.map +1 -0
  33. package/dist/types/connected-evidence/git-starting-state.d.ts +28 -0
  34. package/dist/types/connected-evidence/git-starting-state.d.ts.map +1 -0
  35. package/dist/types/connected-evidence/git.d.ts +3 -0
  36. package/dist/types/connected-evidence/git.d.ts.map +1 -0
  37. package/dist/types/connected-evidence/imports.d.ts.map +1 -1
  38. package/dist/types/connected-evidence/index.d.ts +1 -0
  39. package/dist/types/connected-evidence/index.d.ts.map +1 -1
  40. package/dist/types/connected-evidence/normalize.d.ts +1 -0
  41. package/dist/types/connected-evidence/normalize.d.ts.map +1 -1
  42. package/dist/types/connected-evidence/sources/claude-code.d.ts +10 -0
  43. package/dist/types/connected-evidence/sources/claude-code.d.ts.map +1 -1
  44. package/dist/types/connected-evidence/sources/codex-metadata.d.ts +10 -0
  45. package/dist/types/connected-evidence/sources/codex-metadata.d.ts.map +1 -0
  46. package/dist/types/connected-evidence/sources/codex.d.ts +10 -0
  47. package/dist/types/connected-evidence/sources/codex.d.ts.map +1 -1
  48. package/dist/types/connected-evidence/sources/grok-build.d.ts +10 -0
  49. package/dist/types/connected-evidence/sources/grok-build.d.ts.map +1 -1
  50. package/dist/types/connected-evidence/sources/history.d.ts +20 -0
  51. package/dist/types/connected-evidence/sources/history.d.ts.map +1 -1
  52. package/dist/types/connected-evidence/sources/opencode.d.ts +10 -0
  53. package/dist/types/connected-evidence/sources/opencode.d.ts.map +1 -1
  54. package/dist/types/connected-evidence/sources/pi.d.ts +10 -0
  55. package/dist/types/connected-evidence/sources/pi.d.ts.map +1 -1
  56. package/dist/types/execution-contracts.d.ts +8 -0
  57. package/dist/types/execution-contracts.d.ts.map +1 -1
  58. package/dist/types/graders.d.ts +3 -0
  59. package/dist/types/graders.d.ts.map +1 -1
  60. package/dist/types/javascript-verifier.d.ts +1 -0
  61. package/dist/types/javascript-verifier.d.ts.map +1 -1
  62. package/dist/types/learning/admission.d.ts +1 -0
  63. package/dist/types/learning/admission.d.ts.map +1 -1
  64. package/dist/types/learning/authoring-service.d.ts +2 -0
  65. package/dist/types/learning/authoring-service.d.ts.map +1 -1
  66. package/dist/types/learning/authoring.d.ts +8 -0
  67. package/dist/types/learning/authoring.d.ts.map +1 -1
  68. package/dist/types/learning/batch-service.d.ts +4 -0
  69. package/dist/types/learning/batch-service.d.ts.map +1 -1
  70. package/dist/types/learning/contracts.d.ts +17 -0
  71. package/dist/types/learning/contracts.d.ts.map +1 -1
  72. package/dist/types/learning/operations.d.ts +8 -0
  73. package/dist/types/learning/operations.d.ts.map +1 -1
  74. package/dist/types/learning/repository.d.ts +14 -0
  75. package/dist/types/learning/repository.d.ts.map +1 -1
  76. package/dist/types/learning/reward-calibration-closure.d.ts +9 -1
  77. package/dist/types/learning/reward-calibration-closure.d.ts.map +1 -1
  78. package/dist/types/learning/reward-checks.d.ts +2 -0
  79. package/dist/types/learning/reward-checks.d.ts.map +1 -1
  80. package/dist/types/learning/transport.d.ts +18 -0
  81. package/dist/types/learning/transport.d.ts.map +1 -1
  82. package/dist/types/native-conversations/codex-window-inventory.d.ts.map +1 -1
  83. package/dist/types/native-conversations/git-assets.d.ts +32 -0
  84. package/dist/types/native-conversations/git-assets.d.ts.map +1 -0
  85. package/dist/types/native-conversations/index.d.ts +1 -0
  86. package/dist/types/native-conversations/index.d.ts.map +1 -1
  87. package/dist/types/prepared-process-isolation.d.ts +5 -0
  88. package/dist/types/prepared-process-isolation.d.ts.map +1 -0
  89. package/dist/types/review-conformance.d.ts +5 -5
  90. package/dist/types/rewards.d.ts +9 -0
  91. package/dist/types/rewards.d.ts.map +1 -1
  92. package/dist/types/tasksets.d.ts +16 -0
  93. package/dist/types/tasksets.d.ts.map +1 -1
  94. package/package.json +6 -1
  95. package/schemas/learning/v1/binding.schema.json +8 -0
  96. package/schemas/learning/v1/command-request.schema.json +8 -0
  97. package/schemas/learning/v1/decision.schema.json +5 -1
  98. package/schemas/learning/v1/draft.schema.json +8 -0
  99. package/schemas/learning/v1/grade.schema.json +5 -1
  100. package/schemas/learning/v1/package.schema.json +113 -93
  101. package/schemas/learning/v1/reward_check.schema.json +2 -1
package/README.md CHANGED
@@ -163,6 +163,18 @@ overrides that state.
163
163
 
164
164
  ## Runtime adapters
165
165
 
166
+ Graders may declare `taskIds` to apply to a nonempty, unique population within
167
+ the Taskset. Grading binds the full grader-set hash while executing and
168
+ aggregating only the applicable checks. Reward binding sources carry the same
169
+ scope; scoped compositions retain `taskId` and an explicit `not_applicable`
170
+ result for each excluded source. Excluded checks cannot satisfy a required check
171
+ or change the score, and a composition without its scope is rejected.
172
+
173
+ `@openpond/evals/connected-evidence/git` verifies retained Git starting-file
174
+ receipts against exact object identities and bytes. The receipt is evidence,
175
+ not authorization to read a repository. Hosts remain responsible for owner
176
+ authorization, dependency closure, and runtime qualification.
177
+
166
178
  Implement `HarnessRuntime` for environment state and `HarnessExecutor` around
167
179
  the host's existing model/Agent loop. Do not reimplement prompting, tool dispatch,
168
180
  session persistence, cost accounting, cancellation, or cleanup in this package.
@@ -0,0 +1,81 @@
1
+ import { parse } from "acorn";
2
+ const ast = (value) => {
3
+ if (!value || typeof value !== "object" || !("type" in value))
4
+ throw new Error("connected_command_syntax_unsupported");
5
+ return value;
6
+ };
7
+ function member(value, object, property) {
8
+ const node = ast(value);
9
+ return node.type === "MemberExpression" && !node.computed && !node.optional
10
+ && ast(node.object).type === "Identifier" && ast(node.object).name === object
11
+ && ast(node.property).type === "Identifier" && ast(node.property).name === property;
12
+ }
13
+ /** Interpret recorded syntax, never execute source code. Only sequential static
14
+ * exec_command calls have an unambiguous correspondence with retained results. */
15
+ export function recordedCodexCommands(source) {
16
+ if (source.length > 128_000)
17
+ throw new Error("connected_command_syntax_limit");
18
+ const program = parse(source, { ecmaVersion: "latest", sourceType: "module" });
19
+ const commands = [];
20
+ for (const statement of program.body) {
21
+ const outer = ast(ast(statement).expression);
22
+ const args = outer.arguments;
23
+ if (statement.type !== "ExpressionStatement" || outer.type !== "CallExpression" || outer.optional
24
+ || ast(outer.callee).type !== "Identifier" || ast(outer.callee).name !== "text" || args.length !== 1)
25
+ throw new Error("connected_command_syntax_unsupported");
26
+ const awaited = ast(args[0]);
27
+ const call = ast(awaited.argument);
28
+ const innerArgs = call.arguments;
29
+ if (awaited.type !== "AwaitExpression" || call.type !== "CallExpression" || call.optional
30
+ || !member(call.callee, "tools", "exec_command") || innerArgs.length !== 1)
31
+ throw new Error("connected_command_syntax_unsupported");
32
+ const object = ast(innerArgs[0]);
33
+ if (object.type !== "ObjectExpression")
34
+ throw new Error("connected_command_syntax_unsupported");
35
+ const values = new Map();
36
+ for (const raw of object.properties) {
37
+ const property = ast(raw), key = ast(property.key), value = ast(property.value);
38
+ const name = key.type === "Identifier" ? key.name : key.type === "Literal" ? key.value : null;
39
+ if (property.type !== "Property" || property.computed || property.method || property.shorthand || property.kind !== "init"
40
+ || typeof name !== "string" || values.has(name) || value.type !== "Literal"
41
+ || !["cmd", "workdir", "max_output_tokens", "yield_time_ms"].includes(name))
42
+ throw new Error("connected_command_syntax_unsupported");
43
+ values.set(name, value.value);
44
+ }
45
+ const cmd = values.get("cmd"), budget = values.get("max_output_tokens"), cwd = values.get("workdir");
46
+ if (typeof cmd !== "string" || !cmd.length || cmd.length > 16_000
47
+ || typeof budget !== "number" || !Number.isInteger(budget) || budget <= 0 || budget > 100_000
48
+ || cwd !== undefined && typeof cwd !== "string")
49
+ throw new Error("connected_command_syntax_unsupported");
50
+ commands.push({ cmd, maxOutputTokens: budget, cwd: typeof cwd === "string" ? cwd : null });
51
+ }
52
+ if (!commands.length || commands.length > 20)
53
+ throw new Error("connected_command_syntax_unsupported");
54
+ return commands;
55
+ }
56
+ /** This deliberately bounded read-only prefix establishes that no earlier
57
+ * recorded command modified the checkout before the status observation. */
58
+ export function isRecordedReadOnlyInventory(cmd) {
59
+ return cmd === "pwd" || /^pwd && rg --files(?: -g '[a-zA-Z0-9_.*\/-]+')*$/.test(cmd);
60
+ }
61
+ export function recordedCodexCommandResults(value, count) {
62
+ if (!Array.isArray(value) || value.length !== count + 1)
63
+ throw new Error("connected_command_results_incomplete");
64
+ const blocks = value.map(block => {
65
+ if (!block || typeof block !== "object" || !("type" in block) || block.type !== "input_text"
66
+ || !("text" in block) || typeof block.text !== "string")
67
+ throw new Error("connected_command_results_incomplete");
68
+ return block.text;
69
+ });
70
+ if (!/^Script completed\nWall time [0-9.]+ seconds\nOutput:\n$/.test(blocks[0]))
71
+ throw new Error("connected_command_results_incomplete");
72
+ return blocks.slice(1).map(text => {
73
+ const result = JSON.parse(text);
74
+ if (!result || typeof result !== "object" || !("exit_code" in result) || result.exit_code !== 0
75
+ || "session_id" in result || !("output" in result) || typeof result.output !== "string"
76
+ || !("original_token_count" in result) || typeof result.original_token_count !== "number"
77
+ || !Number.isSafeInteger(result.original_token_count) || result.original_token_count < 0)
78
+ throw new Error("connected_command_results_incomplete");
79
+ return { output: result.output, originalTokenCount: result.original_token_count };
80
+ });
81
+ }
@@ -29,11 +29,24 @@ export const ConnectedBoundarySchema = z.object({
29
29
  process: z.enum(["retained", "partial", "absent"]), artifacts: z.enum(["retained", "references_only", "absent"]),
30
30
  replay: z.literal("unavailable"), gaps: z.array(z.string().max(500)).max(100) }).strict(),
31
31
  }).strict();
32
+ /** Source-reported repository identity is reconstruction evidence, not an
33
+ * admitted checkout or proof that the working tree was clean. */
34
+ export const ConnectedRepositoryContextSchema = z.object({
35
+ schemaVersion: z.literal("openpond.connectedRepositoryContext.v1"),
36
+ kind: z.literal("git"),
37
+ commitHash: z.string().regex(/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/),
38
+ branch: z.string().min(1).max(500).nullable(),
39
+ recordedAt: Time,
40
+ sourceFileHash: Hash,
41
+ metadataHash: Hash,
42
+ workingDirectoryHash: Hash.optional(),
43
+ }).strict();
32
44
  export const ConnectedSessionSchema = z.object({
33
45
  schemaVersion: z.literal(CONNECTED_EVIDENCE_VERSION), normalizerVersion: z.literal(CONNECTED_NORMALIZER_VERSION),
34
46
  origin: ConnectedSourceKindSchema, sessionId: Id, branchId: Id.nullable(), parentSessionId: Id.nullable(),
35
47
  exporterVersion: z.string().max(200).nullable(),
36
48
  acquisition: z.object({ machineId: Id, sourceInstanceId: Id }).strict().optional(),
49
+ repositoryContext: ConnectedRepositoryContextSchema.optional(),
37
50
  sourceFiles: z.array(z.object({ path: Id, contentHash: Hash, sizeBytes: z.number().int().nonnegative() }).strict()).max(CONNECTED_EVIDENCE_LIMITS.files),
38
51
  events: z.array(ConnectedEventSchema).max(CONNECTED_EVIDENCE_LIMITS.events),
39
52
  unmappedEvents: z.array(ConnectedEventSchema).max(CONNECTED_EVIDENCE_LIMITS.events),
@@ -0,0 +1,16 @@
1
+ import { z } from "zod";
2
+ const ObjectHash = z.string().regex(/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/);
3
+ const Base64 = z.string().max(3_000_000);
4
+ const ObjectBytes = z.object({ hash: ObjectHash, base64: Base64 }).strict();
5
+ const safePath = (path) => !/[\x00-\x1f\x7f]/.test(path) && !path.startsWith("/") && !path.includes("\\")
6
+ && path.split("/").length <= 32 && path.split("/").every(part => part && part !== "." && part !== ".."
7
+ && part !== ".git" && !part.startsWith(".env"));
8
+ /** A Git object proof pins bytes to recorded history. It does not establish
9
+ * that the historical working tree was clean or that these are all dependencies. */
10
+ export const ConnectedGitFileProofSchema = z.object({
11
+ schemaVersion: z.literal("openpond.connectedGitFileProof.v1"),
12
+ path: z.string().min(1).max(1_000).refine(safePath),
13
+ commit: ObjectBytes,
14
+ trees: z.array(ObjectBytes).min(1).max(32),
15
+ contentsBase64: Base64,
16
+ }).strict();
@@ -0,0 +1,86 @@
1
+ import { createHash } from "node:crypto";
2
+ import { ConnectedGitFileProofSchema } from "./git-file-contracts.js";
3
+ export * from "./git-file-contracts.js";
4
+ import { ConnectedRepositoryContextSchema } from "./contracts.js";
5
+ function bytes(base64, limit) {
6
+ const value = Buffer.from(base64, "base64");
7
+ if (value.toString("base64") !== base64 || value.byteLength > limit)
8
+ throw new Error("connected_git_object_bytes_invalid");
9
+ return value;
10
+ }
11
+ function objectHash(kind, value, width) {
12
+ return createHash(width === 40 ? "sha1" : "sha256")
13
+ .update(`${kind} ${value.byteLength}\0`).update(value).digest("hex");
14
+ }
15
+ function treeEntry(tree, name, width) {
16
+ let offset = 0;
17
+ const names = new Set();
18
+ let found = null;
19
+ const decoder = new TextDecoder("utf-8", { fatal: true });
20
+ while (offset < tree.byteLength) {
21
+ const space = tree.indexOf(32, offset), nul = tree.indexOf(0, space + 1);
22
+ const end = nul + 1 + width / 2;
23
+ if (space < offset || nul < space || end > tree.byteLength)
24
+ throw new Error("connected_git_tree_invalid");
25
+ const mode = tree.subarray(offset, space).toString("ascii");
26
+ const entryName = decoder.decode(tree.subarray(space + 1, nul));
27
+ if (!entryName || entryName.includes("/") || names.has(entryName))
28
+ throw new Error("connected_git_tree_invalid");
29
+ names.add(entryName);
30
+ if (entryName === name)
31
+ found = { mode, hash: tree.subarray(nul + 1, end).toString("hex") };
32
+ offset = end;
33
+ }
34
+ if (!found)
35
+ throw new Error("connected_git_path_missing");
36
+ return found;
37
+ }
38
+ /** The caller must supply repositoryContext from an authorized, hash-verified
39
+ * connected snapshot. No filesystem access or caller-selected Git command occurs. */
40
+ export function verifyConnectedGitFileProof(repositoryContext, raw) {
41
+ const context = ConnectedRepositoryContextSchema.parse(repositoryContext);
42
+ const proof = ConnectedGitFileProofSchema.parse(raw);
43
+ const width = context.commitHash.length;
44
+ const commit = bytes(proof.commit.base64, 256 * 1024);
45
+ if (proof.commit.hash !== context.commitHash || objectHash("commit", commit, width) !== context.commitHash)
46
+ throw new Error("connected_git_commit_mismatch");
47
+ const firstNewline = commit.indexOf(10);
48
+ if (firstNewline < 0)
49
+ throw new Error("connected_git_commit_invalid");
50
+ const firstLine = commit.subarray(0, firstNewline).toString("ascii");
51
+ if (!new RegExp(`^tree [a-f0-9]{${width}}$`).test(firstLine))
52
+ throw new Error("connected_git_commit_invalid");
53
+ const trees = new Map();
54
+ let totalBytes = commit.byteLength;
55
+ for (const object of proof.trees) {
56
+ const value = bytes(object.base64, 2 * 1024 * 1024);
57
+ totalBytes += value.byteLength;
58
+ if (totalBytes > 8 * 1024 * 1024 || trees.has(object.hash) || object.hash.length !== width
59
+ || objectHash("tree", value, width) !== object.hash)
60
+ throw new Error("connected_git_tree_mismatch");
61
+ trees.set(object.hash, value);
62
+ }
63
+ const parts = proof.path.split("/");
64
+ const visited = new Set();
65
+ let hash = firstLine.slice(5);
66
+ let mode = "";
67
+ for (const [index, part] of parts.entries()) {
68
+ const tree = trees.get(hash);
69
+ if (!tree || visited.has(hash))
70
+ throw new Error("connected_git_tree_missing");
71
+ visited.add(hash);
72
+ const entry = treeEntry(tree, part, width);
73
+ mode = entry.mode;
74
+ if (index < parts.length - 1 ? mode !== "40000" : !["100644", "100755"].includes(mode))
75
+ throw new Error("connected_git_file_mode_invalid");
76
+ hash = entry.hash;
77
+ }
78
+ if (visited.size !== trees.size)
79
+ throw new Error("connected_git_unreferenced_tree");
80
+ const content = bytes(proof.contentsBase64, 2 * 1024 * 1024);
81
+ if (objectHash("blob", content, width) !== hash)
82
+ throw new Error("connected_git_blob_mismatch");
83
+ return { path: proof.path, commitHash: context.commitHash, gitObjectHash: hash, mode,
84
+ sizeBytes: content.byteLength, sha256: createHash("sha256").update(content).digest("hex"),
85
+ contentsBase64: content.toString("base64") };
86
+ }
@@ -0,0 +1,81 @@
1
+ import { contentHash } from "@openpond/harness";
2
+ import { ConnectedSessionSchema, resolveConnectedBoundaryMetadata } from "./contracts.js";
3
+ import { recordedCodexCommands, recordedCodexCommandResults, isRecordedReadOnlyInventory } from "./codex-command-observations.js";
4
+ import { verifyConnectedGitFileProof } from "./git-file-proof.js";
5
+ function object(value) {
6
+ if (!value || typeof value !== "object" || Array.isArray(value))
7
+ throw new Error("connected_starting_state_unavailable");
8
+ return value;
9
+ }
10
+ function dirtyPaths(output) {
11
+ const paths = [];
12
+ for (const line of output.split("\n")) {
13
+ if (!line)
14
+ continue;
15
+ // Quoted/escaped paths and rename pairs need a distinct binary porcelain
16
+ // protocol. Treat them as unavailable instead of guessing path identities.
17
+ if (!/^[ MADRCU?!]{2} [^\x00-\x1f\x7f]+$/.test(line) || line.slice(3).includes('"')
18
+ || line.slice(3).includes(" -> ") || line.slice(3).includes("\\"))
19
+ throw new Error("connected_git_status_format_unsupported");
20
+ const path = line.slice(3);
21
+ if (path.startsWith("/") || path.split("/").some(part => part === "." || part === ".."))
22
+ throw new Error("connected_git_status_format_unsupported");
23
+ paths.push(path.replace(/\/$/, ""));
24
+ }
25
+ return paths;
26
+ }
27
+ /** Verify only the original first-tool boundary. Later requests need their own
28
+ * captured state. Both the repository context and observations come from the
29
+ * authorized immutable snapshot, never caller-supplied cleanliness claims. */
30
+ export function verifyConnectedGitStartingAssets(input) {
31
+ const session = ConnectedSessionSchema.parse(input.session);
32
+ const boundary = resolveConnectedBoundaryMetadata(session, input.boundaryId);
33
+ const context = session.repositoryContext;
34
+ if (session.origin !== "codex" || !context?.workingDirectoryHash || !context.recordedAt || boundary.projection !== "turn"
35
+ || !session.sourceFiles.some(file => file.contentHash === context.sourceFileHash)
36
+ || !input.proofs.length || input.proofs.length > 100)
37
+ throw new Error("connected_starting_state_unavailable");
38
+ const firstCallIndex = session.events.findIndex(event => event.kind === "tool_call");
39
+ if (firstCallIndex <= boundary.start || firstCallIndex >= boundary.end
40
+ || session.events.slice(0, firstCallIndex).some(event => event.kind === "compaction"))
41
+ throw new Error("connected_starting_state_boundary_unavailable");
42
+ const callEvent = session.events[firstCallIndex];
43
+ const requestEvent = session.events[boundary.start];
44
+ if (!callEvent.occurredAt || !requestEvent.occurredAt
45
+ || Date.parse(requestEvent.occurredAt) < Date.parse(context.recordedAt)
46
+ || Date.parse(callEvent.occurredAt) < Date.parse(requestEvent.occurredAt))
47
+ throw new Error("connected_starting_state_observation_time_unavailable");
48
+ const call = object(callEvent.content);
49
+ if (call.name !== "exec" || typeof call.input !== "string" || !callEvent.callId)
50
+ throw new Error("connected_starting_state_command_unsupported");
51
+ const commands = recordedCodexCommands(call.input);
52
+ const statusIndex = commands.findIndex(command => command.cmd === "git status --short");
53
+ if (statusIndex !== commands.length - 1 || statusIndex < 1
54
+ || commands.some(command => command.cwd !== null)
55
+ || !commands.slice(0, statusIndex).every(command => isRecordedReadOnlyInventory(command.cmd)))
56
+ throw new Error("connected_starting_state_command_unsupported");
57
+ const observations = session.events.slice(firstCallIndex + 1, boundary.end);
58
+ const resultIndex = observations.findIndex(event => event.kind === "tool_result" && event.callId === callEvent.callId);
59
+ if (resultIndex < 0 || observations.slice(0, resultIndex).some(event => ["tool_call", "tool_result", "compaction", "message"].includes(event.kind))
60
+ || observations.filter(event => event.kind === "tool_result" && event.callId === callEvent.callId).length !== 1)
61
+ throw new Error("connected_starting_state_observation_unavailable");
62
+ const resultEvent = observations[resultIndex];
63
+ const results = recordedCodexCommandResults(resultEvent.content, commands.length);
64
+ if (results.some((result, index) => result.originalTokenCount >= commands[index].maxOutputTokens
65
+ || /output truncated|truncated output|\[\.\.\./i.test(result.output)))
66
+ throw new Error("connected_starting_state_observation_truncated");
67
+ if (contentHash(results[0].output.split("\n")[0]) !== context.workingDirectoryHash)
68
+ throw new Error("connected_starting_state_directory_mismatch");
69
+ const dirty = dirtyPaths(results[statusIndex].output);
70
+ const assets = input.proofs.map(proof => verifyConnectedGitFileProof(context, proof));
71
+ if (new Set(assets.map(asset => asset.path)).size !== assets.length)
72
+ throw new Error("connected_starting_state_duplicate_path");
73
+ for (const asset of assets)
74
+ if (dirty.some(path => asset.path === path || asset.path.startsWith(path + "/")))
75
+ throw new Error("connected_starting_state_selected_file_modified");
76
+ return { schemaVersion: "openpond.connectedGitStartingAssets.v1",
77
+ sessionHash: session.contentHash, boundaryId: boundary.id, inputHash: boundary.inputHash,
78
+ revisionHash: boundary.revisionHash, commitHash: context.commitHash,
79
+ observationHash: contentHash({ call: callEvent, result: resultEvent }),
80
+ selectedPathsUnmodified: true, dependencyClosureQualified: false, assets };
81
+ }
@@ -0,0 +1,2 @@
1
+ export * from "./git-file-proof.js";
2
+ export * from "./git-starting-state.js";
@@ -10,7 +10,7 @@ import { parseClaudeSession } from "./sources/claude-code.js";
10
10
  import { parseHermesEvidence, parseOpenClawEvidence, } from "./sources/history.js";
11
11
  /** Cache identity includes source projection policy, independent of file mtimes. */
12
12
  export function agentImportNormalizerRevision(source) {
13
- return source === "codex" ? `${CONNECTED_NORMALIZER_VERSION}/${CODEX_AUTHORIZATION_PROJECTION_POLICY}` : CONNECTED_NORMALIZER_VERSION;
13
+ return source === "codex" ? `${CONNECTED_NORMALIZER_VERSION}/${CODEX_AUTHORIZATION_PROJECTION_POLICY}/fork-repository-starting-state-v1` : CONNECTED_NORMALIZER_VERSION;
14
14
  }
15
15
  /** Never admits malformed files partly or executes referenced paths/commands. */
16
16
  export function previewAgentImport(input) {
@@ -49,6 +49,7 @@ export function previewAgentImport(input) {
49
49
  warnings: session.warnings,
50
50
  contextComplete: !session.boundaries.some((boundary) => boundary.coverage.context === "unknown"),
51
51
  acquisition: input.acquisition,
52
+ ...(session.repositoryContext ? { repositoryContext: session.repositoryContext } : {}),
52
53
  })
53
54
  : session));
54
55
  }
@@ -4,3 +4,4 @@ export * from "./imports.js";
4
4
  export * from "./usage-summary.js";
5
5
  export * from "./python-correctness.js";
6
6
  export * from "./collector-coverage-contracts.js";
7
+ export * from "./git-file-contracts.js";
@@ -67,6 +67,7 @@ export function normalizeConnectedSession(input) {
67
67
  origin: input.origin, sessionId: input.sessionId, branchId: input.branchId ?? null, parentSessionId: input.parentSessionId ?? null,
68
68
  exporterVersion: input.exporterVersion ?? null, sourceFiles: input.files.map(connectedFileIdentity).sort((a, b) => a.path.localeCompare(b.path)),
69
69
  ...(input.acquisition ? { acquisition: input.acquisition } : {}),
70
+ ...(input.repositoryContext ? { repositoryContext: input.repositoryContext } : {}),
70
71
  events, unmappedEvents: (input.unmappedEvents ?? []).map((event, sequence) => ({ ...event, sequence })), boundaries, warnings: input.warnings ?? [] };
71
72
  const session = ConnectedSessionSchema.parse({ ...body, contentHash: contentHash(body) });
72
73
  if (new TextEncoder().encode(JSON.stringify(session)).length > CONNECTED_EVIDENCE_LIMITS.decodedBytes)
@@ -0,0 +1,33 @@
1
+ /** Forked Codex rollouts retain their own header followed by ancestor headers.
2
+ * Only the declared, contiguous ancestry is metadata for that source. */
3
+ export class CodexMetadataChain {
4
+ metadata = null;
5
+ parentId = null;
6
+ closed = false;
7
+ ids = new Set();
8
+ accept(row) {
9
+ if (row.type !== "session_meta") {
10
+ this.closed = true;
11
+ return true;
12
+ }
13
+ const payload = row.payload;
14
+ if (this.closed || !payload || typeof payload !== "object" || Array.isArray(payload))
15
+ return false;
16
+ const value = payload;
17
+ if (typeof value.id !== "string" || !value.id || value.id.length > 500
18
+ || this.ids.has(value.id) || this.ids.size >= 16)
19
+ return false;
20
+ if (this.metadata) {
21
+ if (value.id !== this.parentId)
22
+ return false;
23
+ for (const key of ["creator_user_id", "creator_account_id"])
24
+ if (this.metadata[key] != null && value[key] != null && this.metadata[key] !== value[key])
25
+ return false;
26
+ }
27
+ else
28
+ this.metadata = value;
29
+ this.ids.add(value.id);
30
+ this.parentId = typeof value.forked_from_id === "string" ? value.forked_from_id : null;
31
+ return true;
32
+ }
33
+ }
@@ -1,11 +1,24 @@
1
1
  import { contentHash } from "@openpond/harness";
2
- import { connectedTimestamp, event, json, normalizeConnectedSession, object } from "../normalize.js";
2
+ import { connectedFileIdentity, connectedTimestamp, event, json, normalizeConnectedSession, object } from "../normalize.js";
3
3
  import { projectCodexAuthorization } from "./codex-authorization.js";
4
4
  import { sourceUsage } from "./usage.js";
5
+ import { CodexMetadataChain } from "./codex-metadata.js";
5
6
  export function parseCodexSession(file, rows) {
6
- const metadata = object(rows.find(row => row.type === "session_meta")?.payload);
7
+ const chain = new CodexMetadataChain();
8
+ if (rows.some(row => !chain.accept(row)))
9
+ throw new Error("Codex rollout metadata does not match its declared fork ancestry.");
10
+ const metadata = chain.metadata ?? {};
7
11
  if (typeof metadata.id !== "string")
8
12
  throw new Error("This is not a supported Codex rollout: session_meta.id is missing.");
13
+ const git = object(metadata.git);
14
+ const repositoryContext = typeof git.commit_hash === "string" && /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(git.commit_hash)
15
+ ? { schemaVersion: "openpond.connectedRepositoryContext.v1", kind: "git",
16
+ commitHash: git.commit_hash, branch: typeof git.branch === "string" && git.branch.length > 0 && git.branch.length <= 500 ? git.branch : null,
17
+ recordedAt: connectedTimestamp(metadata.timestamp), sourceFileHash: connectedFileIdentity(file).contentHash,
18
+ metadataHash: contentHash(git) }
19
+ : undefined;
20
+ if (repositoryContext && typeof metadata.cwd === "string")
21
+ Object.assign(repositoryContext, { workingDirectoryHash: contentHash(metadata.cwd) });
9
22
  const events = [];
10
23
  let turnId = null;
11
24
  let authorizationOmissions = 0;
@@ -52,5 +65,6 @@ export function parseCodexSession(file, rows) {
52
65
  }
53
66
  return normalizeConnectedSession({ origin: "codex", sessionId: metadata.id, parentSessionId: typeof metadata.forked_from_id === "string" ? metadata.forked_from_id : null,
54
67
  exporterVersion: typeof metadata.cli_version === "string" ? metadata.cli_version : null, files: [file], events,
68
+ ...(repositoryContext ? { repositoryContext } : {}),
55
69
  warnings: ["Codex cumulative usage is retained as reported totals and is not summed as billed invocations.", "Referenced files are unavailable unless separately admitted.", ...(authorizationOmissions ? [`Codex native authorization state excluded by codex-native-authorization-v1 from ${authorizationOmissions} locations. Omission markers retain source-section hashes; sourceFiles retains original byte hashes. Other conversation evidence is unchanged and still requires privacy admission.`] : [])] });
56
70
  }
package/dist/graders.js CHANGED
@@ -39,9 +39,15 @@ export const GraderEvidenceContentSchema = z.object({
39
39
  modelJudgeReceipt: ModelJudgeReceiptSchema.optional(),
40
40
  }).strict();
41
41
  export const GraderEvidenceSchema = GraderEvidenceContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
42
+ /** Scope is part of the immutable grader. An absent scope applies to every
43
+ * task; an explicit scope is never inferred from answer content or metadata. */
44
+ export function gradersForTask(graders, task) {
45
+ return graders.filter(grader => !grader.taskIds || grader.taskIds.includes(task.id));
46
+ }
42
47
  export async function gradeEvidence(input) {
48
+ const graders = gradersForTask(input.graders, input.task);
43
49
  if (input.evidence.infrastructureError) {
44
- return input.graders.map((grader) => evidence(grader, {
50
+ return graders.map((grader) => evidence(grader, {
45
51
  score: null,
46
52
  passed: false,
47
53
  rewardEligible: false,
@@ -51,7 +57,7 @@ export async function gradeEvidence(input) {
51
57
  privilegedEvidenceRefs: [],
52
58
  }));
53
59
  }
54
- return Promise.all(input.graders.map(async (grader) => {
60
+ return Promise.all(graders.map(async (grader) => {
55
61
  if (grader.kind === "model_judge") {
56
62
  if (!input.modelJudge || (grader.calibrationStatus !== "passed" && input.purpose !== "fixture_calibration"))
57
63
  return evidence(grader, unavailable("Model judge is unavailable or uncalibrated."));
@@ -148,7 +154,7 @@ export async function gradeTaskEvidence(input) {
148
154
  const components = await gradeEvidence(input);
149
155
  const content = { schemaVersion: "openpond.taskGrade.v1",
150
156
  taskHash: contentHash(input.task), evidenceHash: contentHash(input.evidence), graderSetHash: contentHash(input.graders), components,
151
- ...aggregateGraderScores({ graders: input.graders, components, infrastructureError: input.evidence.infrastructureError }) };
157
+ ...aggregateGraderScores({ graders: gradersForTask(input.graders, input.task), components, infrastructureError: input.evidence.infrastructureError }) };
152
158
  return TaskGradeSchema.parse({ ...content, contentHash: contentHash(content) });
153
159
  }
154
160
  /** Verify retained results against the exact admitted inputs without rerunning
@@ -163,7 +169,7 @@ export function verifyTaskGrade(value, input) {
163
169
  if (componentHash !== contentHash(body))
164
170
  throw new Error("Grader evidence integrity failed.");
165
171
  }
166
- const summary = aggregateGraderScores({ graders: input.graders, components: result.components, infrastructureError: input.evidence.infrastructureError });
172
+ const summary = aggregateGraderScores({ graders: gradersForTask(input.graders, input.task), components: result.components, infrastructureError: input.evidence.infrastructureError });
167
173
  if (contentHash(GradeSummarySchema.parse({ score: result.score, passed: result.passed, rewardEligible: result.rewardEligible, failureClass: result.failureClass, gradingStatus: result.gradingStatus })) !== contentHash(summary))
168
174
  throw new Error("Task grade summary differs from its grader evidence.");
169
175
  return result;
@@ -1,6 +1,7 @@
1
1
  import { executeJavaScriptIsolate } from "./javascript-isolate.js";
2
2
  import { assertIsolatedVerifierRuntime, JavaScriptVerifierResultSchema } from "./javascript-verifier-contract.js";
3
3
  export { JavaScriptVerifierResultSchema } from "./javascript-verifier-contract.js";
4
+ export { PREPARED_PROCESS_ISOLATION_SOURCE } from "./prepared-process-isolation.js";
4
5
  /** A fresh interpreter with no filesystem, network, timers or module loader.
5
6
  * Hosts should use the worker entrypoint for cancellation and responsiveness. */
6
7
  export async function executeJavaScriptVerifier(input) {
@@ -1,6 +1,7 @@
1
1
  import { createHash } from "node:crypto";
2
2
  import { open } from "node:fs/promises";
3
3
  import { CONNECTED_EVIDENCE_LIMITS } from "../connected-evidence/contracts.js";
4
+ import { CodexMetadataChain } from "../connected-evidence/sources/codex-metadata.js";
4
5
  /** Streaming inventory may prove an irrelevant source without retaining it.
5
6
  * It never truncates a selected conversation into an admissible snapshot. */
6
7
  export async function inspectCodexWindow(input) {
@@ -14,7 +15,8 @@ export async function inspectCodexWindow(input) {
14
15
  const decoder = new TextDecoder("utf-8", { fatal: true });
15
16
  const from = input.since === null ? -Infinity : Date.parse(input.since);
16
17
  const to = Date.parse(input.until);
17
- let metadata = false, records = 0, bytes = 0;
18
+ const metadata = new CodexMetadataChain();
19
+ let records = 0, bytes = 0;
18
20
  let pending = [], pendingBytes = 0;
19
21
  const inspect = (line) => {
20
22
  if (++records > 1_000_000 || line.byteLength > CONNECTED_EVIDENCE_LIMITS.lineBytes)
@@ -40,11 +42,8 @@ export async function inspectCodexWindow(input) {
40
42
  const value = row;
41
43
  const payload = value.payload && typeof value.payload === "object" && !Array.isArray(value.payload)
42
44
  ? value.payload : {};
43
- if (value.type === "session_meta") {
44
- if (metadata || payload.id !== input.sessionId)
45
- return false;
46
- metadata = true;
47
- }
45
+ if (!metadata.accept(value) || (metadata.metadata && metadata.metadata.id !== input.sessionId))
46
+ return false;
48
47
  // The Codex normalizer maps other native records to unknown events;
49
48
  // they cannot create request boundaries. Missing metadata is still refused.
50
49
  const request = value.type === "response_item" && payload.type === "message" && payload.role === "user"
@@ -89,7 +88,7 @@ export async function inspectCodexWindow(input) {
89
88
  if (initial.dev !== final.dev || initial.ino !== final.ino || initial.size !== final.size
90
89
  || initial.mtimeMs !== final.mtimeMs || initial.ctimeMs !== final.ctimeMs || bytes !== final.size)
91
90
  throw new Error("Native source changed during window inventory.");
92
- return metadata ? { sha256: digest.digest("hex") } : null;
91
+ return metadata.metadata ? { sha256: digest.digest("hex") } : null;
93
92
  }
94
93
  finally {
95
94
  await handle.close();
@@ -0,0 +1,61 @@
1
+ import { execFile } from "node:child_process";
2
+ import { realpath } from "node:fs/promises";
3
+ import { isAbsolute } from "node:path";
4
+ import { promisify } from "node:util";
5
+ import { ConnectedRepositoryContextSchema } from "../connected-evidence/contracts.js";
6
+ import { ConnectedGitFileProofSchema, verifyConnectedGitFileProof } from "../connected-evidence/git-file-proof.js";
7
+ const execute = promisify(execFile);
8
+ /** Capture only recorded Git objects. A missing historical object is an explicit
9
+ * gap; this operation never reads the current worktree or fetches another repo. */
10
+ export async function captureNativeGitAssetProofs(input) {
11
+ const context = ConnectedRepositoryContextSchema.parse(input.repositoryContext);
12
+ if (!isAbsolute(input.repositoryPath) || !input.paths.length || input.paths.length > 100
13
+ || new Set(input.paths).size !== input.paths.length)
14
+ throw new Error("native_git_asset_selection_invalid");
15
+ // Validate selections before invoking Git, including paths prohibited by the
16
+ // source asset policy. The remaining fields are filled from immutable objects.
17
+ for (const path of input.paths)
18
+ ConnectedGitFileProofSchema.shape.path.parse(path);
19
+ const repository = await realpath(input.repositoryPath);
20
+ const read = async (args, maxBuffer = 2 * 1024 * 1024) => {
21
+ input.signal?.throwIfAborted();
22
+ const result = await execute("git", ["-C", repository, ...args], {
23
+ encoding: "buffer", timeout: 15_000, maxBuffer, signal: input.signal,
24
+ });
25
+ return result.stdout;
26
+ };
27
+ const objectId = async (ref) => {
28
+ const hash = (await read(["rev-parse", "--verify", ref], 1_024)).toString("ascii").trim();
29
+ if (!new RegExp(`^[a-f0-9]{${context.commitHash.length}}$`).test(hash))
30
+ throw new Error("native_git_object_identity_invalid");
31
+ return hash;
32
+ };
33
+ const commit = await read(["cat-file", "commit", context.commitHash], 256 * 1024);
34
+ const cache = new Map();
35
+ let totalBytes = 0;
36
+ const proofs = [];
37
+ for (const path of input.paths) {
38
+ const directories = ["", ...path.split("/").slice(0, -1).map((_, index, parts) => parts.slice(0, index + 1).join("/"))];
39
+ const trees = [];
40
+ for (const directory of directories) {
41
+ const ref = directory ? `${context.commitHash}:${directory}` : `${context.commitHash}^{tree}`;
42
+ let tree = cache.get(ref);
43
+ if (!tree) {
44
+ const hash = await objectId(ref);
45
+ tree = { hash, base64: (await read(["cat-file", "tree", hash])).toString("base64") };
46
+ cache.set(ref, tree);
47
+ }
48
+ trees.push(tree);
49
+ }
50
+ const blob = await objectId(`${context.commitHash}:${path}`);
51
+ const proof = ConnectedGitFileProofSchema.parse({ schemaVersion: "openpond.connectedGitFileProof.v1", path,
52
+ commit: { hash: context.commitHash, base64: commit.toString("base64") }, trees,
53
+ contentsBase64: (await read(["cat-file", "blob", blob])).toString("base64") });
54
+ const verified = verifyConnectedGitFileProof(context, proof);
55
+ totalBytes += Buffer.byteLength(JSON.stringify(proof));
56
+ if (totalBytes > 16 * 1024 * 1024)
57
+ throw new Error("native_git_asset_capture_limit");
58
+ proofs.push({ proof, verified });
59
+ }
60
+ return proofs;
61
+ }
@@ -10,3 +10,4 @@ export * from "./location.js";
10
10
  export * from "./collector-branches.js";
11
11
  export * from "./collector-destinations.js";
12
12
  export * from "./collector-schedule.js";
13
+ export { captureNativeGitAssetProofs } from "./git-assets.js";