@openpond/evals 0.11.20 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/README.md +12 -0
  2. package/dist/connected-evidence/codex-command-observations.js +81 -0
  3. package/dist/connected-evidence/collector-coverage-contracts.js +160 -0
  4. package/dist/connected-evidence/contracts.js +13 -0
  5. package/dist/connected-evidence/git-file-contracts.js +16 -0
  6. package/dist/connected-evidence/git-file-proof.js +86 -0
  7. package/dist/connected-evidence/git-starting-state.js +81 -0
  8. package/dist/connected-evidence/git.js +2 -0
  9. package/dist/connected-evidence/imports.js +2 -1
  10. package/dist/connected-evidence/index.js +2 -0
  11. package/dist/connected-evidence/normalize.js +1 -0
  12. package/dist/connected-evidence/sources/codex-metadata.js +33 -0
  13. package/dist/connected-evidence/sources/codex.js +16 -2
  14. package/dist/execution-contracts.js +1 -1
  15. package/dist/graders.js +10 -4
  16. package/dist/javascript-isolate-node.js +2 -1
  17. package/dist/javascript-isolate-process-source.js +1 -1
  18. package/dist/javascript-isolate-process.js +2 -1
  19. package/dist/javascript-isolate.js +2 -1
  20. package/dist/javascript-verifier-worker-source.js +1 -1
  21. package/dist/javascript-verifier.js +1 -0
  22. package/dist/native-conversations/codex-window-inventory.js +96 -0
  23. package/dist/native-conversations/collector-acquisition.js +149 -0
  24. package/dist/native-conversations/collector-branches.js +1 -0
  25. package/dist/native-conversations/collector-calendar-gate.js +19 -0
  26. package/dist/native-conversations/collector-contracts.js +4 -3
  27. package/dist/native-conversations/collector-coverage.js +100 -0
  28. package/dist/native-conversations/collector-cron.js +43 -0
  29. package/dist/native-conversations/collector-schedule.js +61 -0
  30. package/dist/native-conversations/collector-service-files.js +51 -0
  31. package/dist/native-conversations/collector-snapshots.js +70 -0
  32. package/dist/native-conversations/collector-source-cache.js +41 -0
  33. package/dist/native-conversations/collector-store.js +43 -8
  34. package/dist/native-conversations/collector.js +189 -261
  35. package/dist/native-conversations/git-assets.js +61 -0
  36. package/dist/native-conversations/history.js +9 -0
  37. package/dist/native-conversations/index.js +2 -0
  38. package/dist/native-conversations/location.js +4 -2
  39. package/dist/native-conversations/supervisor.js +131 -81
  40. package/dist/prepared-process-isolation.js +131 -0
  41. package/dist/rewards.js +17 -5
  42. package/dist/sql-execution-node.js +2 -1
  43. package/dist/tasksets.js +7 -1
  44. package/dist/types/builtin-benchmarks/harness-refiner.d.ts +4 -0
  45. package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
  46. package/dist/types/conformance.d.ts +8 -0
  47. package/dist/types/conformance.d.ts.map +1 -1
  48. package/dist/types/connected-evidence/codex-command-observations.d.ts +15 -0
  49. package/dist/types/connected-evidence/codex-command-observations.d.ts.map +1 -0
  50. package/dist/types/connected-evidence/collector-coverage-contracts.d.ts +98 -0
  51. package/dist/types/connected-evidence/collector-coverage-contracts.d.ts.map +1 -0
  52. package/dist/types/connected-evidence/contracts.d.ts +22 -0
  53. package/dist/types/connected-evidence/contracts.d.ts.map +1 -1
  54. package/dist/types/connected-evidence/git-file-contracts.d.ts +18 -0
  55. package/dist/types/connected-evidence/git-file-contracts.d.ts.map +1 -0
  56. package/dist/types/connected-evidence/git-file-proof.d.ts +13 -0
  57. package/dist/types/connected-evidence/git-file-proof.d.ts.map +1 -0
  58. package/dist/types/connected-evidence/git-starting-state.d.ts +28 -0
  59. package/dist/types/connected-evidence/git-starting-state.d.ts.map +1 -0
  60. package/dist/types/connected-evidence/git.d.ts +3 -0
  61. package/dist/types/connected-evidence/git.d.ts.map +1 -0
  62. package/dist/types/connected-evidence/imports.d.ts.map +1 -1
  63. package/dist/types/connected-evidence/index.d.ts +2 -0
  64. package/dist/types/connected-evidence/index.d.ts.map +1 -1
  65. package/dist/types/connected-evidence/normalize.d.ts +1 -0
  66. package/dist/types/connected-evidence/normalize.d.ts.map +1 -1
  67. package/dist/types/connected-evidence/sources/claude-code.d.ts +10 -0
  68. package/dist/types/connected-evidence/sources/claude-code.d.ts.map +1 -1
  69. package/dist/types/connected-evidence/sources/codex-metadata.d.ts +10 -0
  70. package/dist/types/connected-evidence/sources/codex-metadata.d.ts.map +1 -0
  71. package/dist/types/connected-evidence/sources/codex.d.ts +10 -0
  72. package/dist/types/connected-evidence/sources/codex.d.ts.map +1 -1
  73. package/dist/types/connected-evidence/sources/grok-build.d.ts +10 -0
  74. package/dist/types/connected-evidence/sources/grok-build.d.ts.map +1 -1
  75. package/dist/types/connected-evidence/sources/history.d.ts +20 -0
  76. package/dist/types/connected-evidence/sources/history.d.ts.map +1 -1
  77. package/dist/types/connected-evidence/sources/opencode.d.ts +10 -0
  78. package/dist/types/connected-evidence/sources/opencode.d.ts.map +1 -1
  79. package/dist/types/connected-evidence/sources/pi.d.ts +10 -0
  80. package/dist/types/connected-evidence/sources/pi.d.ts.map +1 -1
  81. package/dist/types/execution-contracts.d.ts +8 -0
  82. package/dist/types/execution-contracts.d.ts.map +1 -1
  83. package/dist/types/graders.d.ts +3 -0
  84. package/dist/types/graders.d.ts.map +1 -1
  85. package/dist/types/javascript-isolate-node.d.ts.map +1 -1
  86. package/dist/types/javascript-isolate-process.d.ts.map +1 -1
  87. package/dist/types/javascript-isolate.d.ts.map +1 -1
  88. package/dist/types/javascript-verifier.d.ts +1 -0
  89. package/dist/types/javascript-verifier.d.ts.map +1 -1
  90. package/dist/types/learning/admission.d.ts +1 -0
  91. package/dist/types/learning/admission.d.ts.map +1 -1
  92. package/dist/types/learning/authoring-service.d.ts +2 -0
  93. package/dist/types/learning/authoring-service.d.ts.map +1 -1
  94. package/dist/types/learning/authoring.d.ts +8 -0
  95. package/dist/types/learning/authoring.d.ts.map +1 -1
  96. package/dist/types/learning/batch-service.d.ts +4 -0
  97. package/dist/types/learning/batch-service.d.ts.map +1 -1
  98. package/dist/types/learning/contracts.d.ts +17 -0
  99. package/dist/types/learning/contracts.d.ts.map +1 -1
  100. package/dist/types/learning/operations.d.ts +8 -0
  101. package/dist/types/learning/operations.d.ts.map +1 -1
  102. package/dist/types/learning/repository.d.ts +14 -0
  103. package/dist/types/learning/repository.d.ts.map +1 -1
  104. package/dist/types/learning/reward-calibration-closure.d.ts +9 -1
  105. package/dist/types/learning/reward-calibration-closure.d.ts.map +1 -1
  106. package/dist/types/learning/reward-checks.d.ts +2 -0
  107. package/dist/types/learning/reward-checks.d.ts.map +1 -1
  108. package/dist/types/learning/transport.d.ts +18 -0
  109. package/dist/types/learning/transport.d.ts.map +1 -1
  110. package/dist/types/native-conversations/codex-window-inventory.d.ts +12 -0
  111. package/dist/types/native-conversations/codex-window-inventory.d.ts.map +1 -0
  112. package/dist/types/native-conversations/collector-acquisition.d.ts +17 -0
  113. package/dist/types/native-conversations/collector-acquisition.d.ts.map +1 -0
  114. package/dist/types/native-conversations/collector-branches.d.ts.map +1 -1
  115. package/dist/types/native-conversations/collector-calendar-gate.d.ts +9 -0
  116. package/dist/types/native-conversations/collector-calendar-gate.d.ts.map +1 -0
  117. package/dist/types/native-conversations/collector-contracts.d.ts +41 -5
  118. package/dist/types/native-conversations/collector-contracts.d.ts.map +1 -1
  119. package/dist/types/native-conversations/collector-coverage.d.ts +62 -0
  120. package/dist/types/native-conversations/collector-coverage.d.ts.map +1 -0
  121. package/dist/types/native-conversations/collector-cron.d.ts +20 -0
  122. package/dist/types/native-conversations/collector-cron.d.ts.map +1 -0
  123. package/dist/types/native-conversations/collector-schedule.d.ts +13 -0
  124. package/dist/types/native-conversations/collector-schedule.d.ts.map +1 -0
  125. package/dist/types/native-conversations/collector-service-files.d.ts +20 -0
  126. package/dist/types/native-conversations/collector-service-files.d.ts.map +1 -0
  127. package/dist/types/native-conversations/collector-snapshots.d.ts +25 -0
  128. package/dist/types/native-conversations/collector-snapshots.d.ts.map +1 -0
  129. package/dist/types/native-conversations/collector-source-cache.d.ts +31 -0
  130. package/dist/types/native-conversations/collector-source-cache.d.ts.map +1 -0
  131. package/dist/types/native-conversations/collector-store.d.ts +11 -2
  132. package/dist/types/native-conversations/collector-store.d.ts.map +1 -1
  133. package/dist/types/native-conversations/collector.d.ts +5 -1
  134. package/dist/types/native-conversations/collector.d.ts.map +1 -1
  135. package/dist/types/native-conversations/git-assets.d.ts +32 -0
  136. package/dist/types/native-conversations/git-assets.d.ts.map +1 -0
  137. package/dist/types/native-conversations/history.d.ts +7 -0
  138. package/dist/types/native-conversations/history.d.ts.map +1 -1
  139. package/dist/types/native-conversations/index.d.ts +2 -0
  140. package/dist/types/native-conversations/index.d.ts.map +1 -1
  141. package/dist/types/native-conversations/location.d.ts.map +1 -1
  142. package/dist/types/native-conversations/supervisor.d.ts +4 -3
  143. package/dist/types/native-conversations/supervisor.d.ts.map +1 -1
  144. package/dist/types/prepared-process-isolation.d.ts +5 -0
  145. package/dist/types/prepared-process-isolation.d.ts.map +1 -0
  146. package/dist/types/review-conformance.d.ts +5 -5
  147. package/dist/types/rewards.d.ts +9 -0
  148. package/dist/types/rewards.d.ts.map +1 -1
  149. package/dist/types/sql-execution-node.d.ts.map +1 -1
  150. package/dist/types/tasksets.d.ts +16 -0
  151. package/dist/types/tasksets.d.ts.map +1 -1
  152. package/package.json +8 -2
  153. package/schemas/learning/v1/binding.schema.json +8 -0
  154. package/schemas/learning/v1/command-request.schema.json +8 -0
  155. package/schemas/learning/v1/decision.schema.json +5 -1
  156. package/schemas/learning/v1/draft.schema.json +8 -0
  157. package/schemas/learning/v1/grade.schema.json +5 -1
  158. package/schemas/learning/v1/package.schema.json +113 -94
  159. package/schemas/learning/v1/reward_check.schema.json +2 -1
package/README.md CHANGED
@@ -163,6 +163,18 @@ overrides that state.
163
163
 
164
164
  ## Runtime adapters
165
165
 
166
+ Graders may declare `taskIds` to apply to a nonempty, unique population within
167
+ the Taskset. Grading binds the full grader-set hash while executing and
168
+ aggregating only the applicable checks. Reward binding sources carry the same
169
+ scope; scoped compositions retain `taskId` and an explicit `not_applicable`
170
+ result for each excluded source. Excluded checks cannot satisfy a required check
171
+ or change the score, and a composition without its scope is rejected.
172
+
173
+ `@openpond/evals/connected-evidence/git` verifies retained Git starting-file
174
+ receipts against exact object identities and bytes. The receipt is evidence,
175
+ not authorization to read a repository. Hosts remain responsible for owner
176
+ authorization, dependency closure, and runtime qualification.
177
+
166
178
  Implement `HarnessRuntime` for environment state and `HarnessExecutor` around
167
179
  the host's existing model/Agent loop. Do not reimplement prompting, tool dispatch,
168
180
  session persistence, cost accounting, cancellation, or cleanup in this package.
@@ -0,0 +1,81 @@
1
+ import { parse } from "acorn";
2
+ const ast = (value) => {
3
+ if (!value || typeof value !== "object" || !("type" in value))
4
+ throw new Error("connected_command_syntax_unsupported");
5
+ return value;
6
+ };
7
+ function member(value, object, property) {
8
+ const node = ast(value);
9
+ return node.type === "MemberExpression" && !node.computed && !node.optional
10
+ && ast(node.object).type === "Identifier" && ast(node.object).name === object
11
+ && ast(node.property).type === "Identifier" && ast(node.property).name === property;
12
+ }
13
+ /** Interpret recorded syntax, never execute source code. Only sequential static
14
+ * exec_command calls have an unambiguous correspondence with retained results. */
15
+ export function recordedCodexCommands(source) {
16
+ if (source.length > 128_000)
17
+ throw new Error("connected_command_syntax_limit");
18
+ const program = parse(source, { ecmaVersion: "latest", sourceType: "module" });
19
+ const commands = [];
20
+ for (const statement of program.body) {
21
+ const outer = ast(ast(statement).expression);
22
+ const args = outer.arguments;
23
+ if (statement.type !== "ExpressionStatement" || outer.type !== "CallExpression" || outer.optional
24
+ || ast(outer.callee).type !== "Identifier" || ast(outer.callee).name !== "text" || args.length !== 1)
25
+ throw new Error("connected_command_syntax_unsupported");
26
+ const awaited = ast(args[0]);
27
+ const call = ast(awaited.argument);
28
+ const innerArgs = call.arguments;
29
+ if (awaited.type !== "AwaitExpression" || call.type !== "CallExpression" || call.optional
30
+ || !member(call.callee, "tools", "exec_command") || innerArgs.length !== 1)
31
+ throw new Error("connected_command_syntax_unsupported");
32
+ const object = ast(innerArgs[0]);
33
+ if (object.type !== "ObjectExpression")
34
+ throw new Error("connected_command_syntax_unsupported");
35
+ const values = new Map();
36
+ for (const raw of object.properties) {
37
+ const property = ast(raw), key = ast(property.key), value = ast(property.value);
38
+ const name = key.type === "Identifier" ? key.name : key.type === "Literal" ? key.value : null;
39
+ if (property.type !== "Property" || property.computed || property.method || property.shorthand || property.kind !== "init"
40
+ || typeof name !== "string" || values.has(name) || value.type !== "Literal"
41
+ || !["cmd", "workdir", "max_output_tokens", "yield_time_ms"].includes(name))
42
+ throw new Error("connected_command_syntax_unsupported");
43
+ values.set(name, value.value);
44
+ }
45
+ const cmd = values.get("cmd"), budget = values.get("max_output_tokens"), cwd = values.get("workdir");
46
+ if (typeof cmd !== "string" || !cmd.length || cmd.length > 16_000
47
+ || typeof budget !== "number" || !Number.isInteger(budget) || budget <= 0 || budget > 100_000
48
+ || cwd !== undefined && typeof cwd !== "string")
49
+ throw new Error("connected_command_syntax_unsupported");
50
+ commands.push({ cmd, maxOutputTokens: budget, cwd: typeof cwd === "string" ? cwd : null });
51
+ }
52
+ if (!commands.length || commands.length > 20)
53
+ throw new Error("connected_command_syntax_unsupported");
54
+ return commands;
55
+ }
56
+ /** This deliberately bounded read-only prefix establishes that no earlier
57
+ * recorded command modified the checkout before the status observation. */
58
+ export function isRecordedReadOnlyInventory(cmd) {
59
+ return cmd === "pwd" || /^pwd && rg --files(?: -g '[a-zA-Z0-9_.*\/-]+')*$/.test(cmd);
60
+ }
61
+ export function recordedCodexCommandResults(value, count) {
62
+ if (!Array.isArray(value) || value.length !== count + 1)
63
+ throw new Error("connected_command_results_incomplete");
64
+ const blocks = value.map(block => {
65
+ if (!block || typeof block !== "object" || !("type" in block) || block.type !== "input_text"
66
+ || !("text" in block) || typeof block.text !== "string")
67
+ throw new Error("connected_command_results_incomplete");
68
+ return block.text;
69
+ });
70
+ if (!/^Script completed\nWall time [0-9.]+ seconds\nOutput:\n$/.test(blocks[0]))
71
+ throw new Error("connected_command_results_incomplete");
72
+ return blocks.slice(1).map(text => {
73
+ const result = JSON.parse(text);
74
+ if (!result || typeof result !== "object" || !("exit_code" in result) || result.exit_code !== 0
75
+ || "session_id" in result || !("output" in result) || typeof result.output !== "string"
76
+ || !("original_token_count" in result) || typeof result.original_token_count !== "number"
77
+ || !Number.isSafeInteger(result.original_token_count) || result.original_token_count < 0)
78
+ throw new Error("connected_command_results_incomplete");
79
+ return { output: result.output, originalTokenCount: result.original_token_count };
80
+ });
81
+ }
@@ -0,0 +1,160 @@
1
+ import { z } from "zod";
2
+ import { contentHash } from "@openpond/harness";
3
+ const Id = z.string().trim().min(1).max(500);
4
+ const Hash = z.string().regex(/^[a-f0-9]{64}$/);
5
+ const Timestamp = z.iso.datetime();
6
+ const Count = z.number().int().nonnegative().max(1_000_000);
7
+ export const CollectorCoverageSessionSchema = z
8
+ .object({
9
+ sourceKeyHash: Hash,
10
+ nativeRevisionHash: Hash.nullable(),
11
+ state: z.enum(["reading", "admitted", "skipped", "failed"]),
12
+ reason: z
13
+ .enum([
14
+ "no_eligible_boundaries",
15
+ "native_session_not_ready",
16
+ "source_read_failed",
17
+ "source_revision_unavailable",
18
+ ])
19
+ .nullable(),
20
+ unknownTimeBoundaries: Count,
21
+ normalized: z
22
+ .array(z
23
+ .object({
24
+ sessionHash: Hash,
25
+ boundaries: z
26
+ .array(z
27
+ .object({ id: Id, inputHash: Hash, revisionHash: Hash, occurredAt: Timestamp })
28
+ .strict())
29
+ .max(5_000),
30
+ })
31
+ .strict())
32
+ .max(1_000),
33
+ })
34
+ .strict();
35
+ export const CollectorCoverageManifestSchema = z
36
+ .object({
37
+ schemaVersion: z.literal("openpond.collectorCoverageManifest.v1"),
38
+ jobId: Id,
39
+ connectionId: Id,
40
+ connectionRevision: z.number().int().positive(),
41
+ machineId: Id,
42
+ sourceInstanceId: Id,
43
+ from: Timestamp.nullable(),
44
+ to: Timestamp,
45
+ startedAt: Timestamp,
46
+ finishedAt: Timestamp.nullable(),
47
+ listingComplete: z.boolean(),
48
+ pendingOperations: Count,
49
+ sessions: z.array(CollectorCoverageSessionSchema).max(10_000),
50
+ })
51
+ .strict()
52
+ .superRefine((value, ctx) => {
53
+ if (value.from !== null && Date.parse(value.from) >= Date.parse(value.to))
54
+ ctx.addIssue({
55
+ code: "custom",
56
+ message: "Collector source window is empty.",
57
+ });
58
+ if (value.finishedAt !== null &&
59
+ Date.parse(value.finishedAt) < Date.parse(value.startedAt))
60
+ ctx.addIssue({
61
+ code: "custom",
62
+ message: "Collector finish time precedes its start.",
63
+ });
64
+ if (new Set(value.sessions.map((session) => session.sourceKeyHash)).size !==
65
+ value.sessions.length)
66
+ ctx.addIssue({
67
+ code: "custom",
68
+ message: "Collector source inventory is duplicated.",
69
+ });
70
+ let boundaries = 0;
71
+ for (const session of value.sessions)
72
+ for (const normalized of session.normalized) {
73
+ boundaries += normalized.boundaries.length;
74
+ if (new Set(normalized.boundaries.map((boundary) => boundary.id)).size !==
75
+ normalized.boundaries.length)
76
+ ctx.addIssue({
77
+ code: "custom",
78
+ message: "Normalized source boundaries are duplicated.",
79
+ });
80
+ for (const boundary of normalized.boundaries)
81
+ if (Date.parse(boundary.occurredAt) >= Date.parse(value.to) ||
82
+ (value.from !== null &&
83
+ Date.parse(boundary.occurredAt) < Date.parse(value.from)))
84
+ ctx.addIssue({
85
+ code: "custom",
86
+ message: "Eligible boundary is outside the declared event-time window.",
87
+ });
88
+ }
89
+ if (boundaries > 20_000)
90
+ ctx.addIssue({
91
+ code: "custom",
92
+ message: "Collector coverage exceeds its bounded boundary inventory.",
93
+ });
94
+ });
95
+ export const CollectorCoverageSummarySchema = z
96
+ .object({
97
+ jobId: Id,
98
+ manifestHash: Hash,
99
+ sourceInventoryHash: Hash,
100
+ eligibleBoundaryHash: Hash,
101
+ from: Timestamp.nullable(),
102
+ to: Timestamp,
103
+ finishedAt: Timestamp.nullable(),
104
+ conversations: Count,
105
+ eligibleTurns: Count,
106
+ admittedSessions: Count,
107
+ skippedSessions: Count,
108
+ failedSessions: Count,
109
+ unknownTimeBoundaries: Count,
110
+ pendingOperations: Count,
111
+ listingComplete: z.boolean(),
112
+ complete: z.boolean(),
113
+ })
114
+ .strict();
115
+ export function hashCollectorEligibleBoundaries(input) {
116
+ const boundaries = new Map(input.map((boundary) => [
117
+ JSON.stringify([boundary.id, boundary.inputHash, boundary.revisionHash]),
118
+ { id: boundary.id, inputHash: boundary.inputHash, revisionHash: boundary.revisionHash },
119
+ ]));
120
+ const ordered = [...boundaries]
121
+ .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
122
+ .map(([, boundary]) => boundary);
123
+ return { hash: contentHash(ordered), count: ordered.length };
124
+ }
125
+ export function summarizeCollectorCoverage(raw) {
126
+ const manifest = CollectorCoverageManifestSchema.parse(raw);
127
+ const boundaries = hashCollectorEligibleBoundaries(manifest.sessions.flatMap((session) => session.normalized.flatMap((normalized) => normalized.boundaries)));
128
+ const unknownTimeBoundaries = manifest.sessions.reduce((sum, session) => sum + session.unknownTimeBoundaries, 0);
129
+ return CollectorCoverageSummarySchema.parse({
130
+ jobId: manifest.jobId,
131
+ manifestHash: contentHash(manifest),
132
+ sourceInventoryHash: contentHash(manifest.sessions
133
+ .map((session) => ({
134
+ sourceKeyHash: session.sourceKeyHash,
135
+ nativeRevisionHash: session.nativeRevisionHash,
136
+ snapshots: session.normalized.map((normalized) => normalized.sessionHash),
137
+ }))
138
+ .sort((a, b) => a.sourceKeyHash.localeCompare(b.sourceKeyHash))),
139
+ eligibleBoundaryHash: boundaries.hash,
140
+ from: manifest.from,
141
+ to: manifest.to,
142
+ finishedAt: manifest.finishedAt,
143
+ conversations: manifest.sessions.length,
144
+ eligibleTurns: boundaries.count,
145
+ admittedSessions: manifest.sessions.filter((session) => session.state === "admitted").length,
146
+ skippedSessions: manifest.sessions.filter((session) => session.state === "skipped").length,
147
+ failedSessions: manifest.sessions.filter((session) => session.state === "failed").length,
148
+ unknownTimeBoundaries,
149
+ pendingOperations: manifest.pendingOperations,
150
+ listingComplete: manifest.listingComplete,
151
+ complete: manifest.finishedAt !== null &&
152
+ manifest.listingComplete &&
153
+ manifest.pendingOperations === 0 &&
154
+ unknownTimeBoundaries === 0 &&
155
+ manifest.sessions.every((session) => session.nativeRevisionHash !== null &&
156
+ (session.state === "admitted" ||
157
+ (session.state === "skipped" &&
158
+ session.reason === "no_eligible_boundaries"))),
159
+ });
160
+ }
@@ -29,11 +29,24 @@ export const ConnectedBoundarySchema = z.object({
29
29
  process: z.enum(["retained", "partial", "absent"]), artifacts: z.enum(["retained", "references_only", "absent"]),
30
30
  replay: z.literal("unavailable"), gaps: z.array(z.string().max(500)).max(100) }).strict(),
31
31
  }).strict();
32
+ /** Source-reported repository identity is reconstruction evidence, not an
33
+ * admitted checkout or proof that the working tree was clean. */
34
+ export const ConnectedRepositoryContextSchema = z.object({
35
+ schemaVersion: z.literal("openpond.connectedRepositoryContext.v1"),
36
+ kind: z.literal("git"),
37
+ commitHash: z.string().regex(/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/),
38
+ branch: z.string().min(1).max(500).nullable(),
39
+ recordedAt: Time,
40
+ sourceFileHash: Hash,
41
+ metadataHash: Hash,
42
+ workingDirectoryHash: Hash.optional(),
43
+ }).strict();
32
44
  export const ConnectedSessionSchema = z.object({
33
45
  schemaVersion: z.literal(CONNECTED_EVIDENCE_VERSION), normalizerVersion: z.literal(CONNECTED_NORMALIZER_VERSION),
34
46
  origin: ConnectedSourceKindSchema, sessionId: Id, branchId: Id.nullable(), parentSessionId: Id.nullable(),
35
47
  exporterVersion: z.string().max(200).nullable(),
36
48
  acquisition: z.object({ machineId: Id, sourceInstanceId: Id }).strict().optional(),
49
+ repositoryContext: ConnectedRepositoryContextSchema.optional(),
37
50
  sourceFiles: z.array(z.object({ path: Id, contentHash: Hash, sizeBytes: z.number().int().nonnegative() }).strict()).max(CONNECTED_EVIDENCE_LIMITS.files),
38
51
  events: z.array(ConnectedEventSchema).max(CONNECTED_EVIDENCE_LIMITS.events),
39
52
  unmappedEvents: z.array(ConnectedEventSchema).max(CONNECTED_EVIDENCE_LIMITS.events),
@@ -0,0 +1,16 @@
1
+ import { z } from "zod";
2
+ const ObjectHash = z.string().regex(/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/);
3
+ const Base64 = z.string().max(3_000_000);
4
+ const ObjectBytes = z.object({ hash: ObjectHash, base64: Base64 }).strict();
5
+ const safePath = (path) => !/[\x00-\x1f\x7f]/.test(path) && !path.startsWith("/") && !path.includes("\\")
6
+ && path.split("/").length <= 32 && path.split("/").every(part => part && part !== "." && part !== ".."
7
+ && part !== ".git" && !part.startsWith(".env"));
8
+ /** A Git object proof pins bytes to recorded history. It does not establish
9
+ * that the historical working tree was clean or that these are all dependencies. */
10
+ export const ConnectedGitFileProofSchema = z.object({
11
+ schemaVersion: z.literal("openpond.connectedGitFileProof.v1"),
12
+ path: z.string().min(1).max(1_000).refine(safePath),
13
+ commit: ObjectBytes,
14
+ trees: z.array(ObjectBytes).min(1).max(32),
15
+ contentsBase64: Base64,
16
+ }).strict();
@@ -0,0 +1,86 @@
1
+ import { createHash } from "node:crypto";
2
+ import { ConnectedGitFileProofSchema } from "./git-file-contracts.js";
3
+ export * from "./git-file-contracts.js";
4
+ import { ConnectedRepositoryContextSchema } from "./contracts.js";
5
+ function bytes(base64, limit) {
6
+ const value = Buffer.from(base64, "base64");
7
+ if (value.toString("base64") !== base64 || value.byteLength > limit)
8
+ throw new Error("connected_git_object_bytes_invalid");
9
+ return value;
10
+ }
11
+ function objectHash(kind, value, width) {
12
+ return createHash(width === 40 ? "sha1" : "sha256")
13
+ .update(`${kind} ${value.byteLength}\0`).update(value).digest("hex");
14
+ }
15
+ function treeEntry(tree, name, width) {
16
+ let offset = 0;
17
+ const names = new Set();
18
+ let found = null;
19
+ const decoder = new TextDecoder("utf-8", { fatal: true });
20
+ while (offset < tree.byteLength) {
21
+ const space = tree.indexOf(32, offset), nul = tree.indexOf(0, space + 1);
22
+ const end = nul + 1 + width / 2;
23
+ if (space < offset || nul < space || end > tree.byteLength)
24
+ throw new Error("connected_git_tree_invalid");
25
+ const mode = tree.subarray(offset, space).toString("ascii");
26
+ const entryName = decoder.decode(tree.subarray(space + 1, nul));
27
+ if (!entryName || entryName.includes("/") || names.has(entryName))
28
+ throw new Error("connected_git_tree_invalid");
29
+ names.add(entryName);
30
+ if (entryName === name)
31
+ found = { mode, hash: tree.subarray(nul + 1, end).toString("hex") };
32
+ offset = end;
33
+ }
34
+ if (!found)
35
+ throw new Error("connected_git_path_missing");
36
+ return found;
37
+ }
38
+ /** The caller must supply repositoryContext from an authorized, hash-verified
39
+ * connected snapshot. No filesystem access or caller-selected Git command occurs. */
40
+ export function verifyConnectedGitFileProof(repositoryContext, raw) {
41
+ const context = ConnectedRepositoryContextSchema.parse(repositoryContext);
42
+ const proof = ConnectedGitFileProofSchema.parse(raw);
43
+ const width = context.commitHash.length;
44
+ const commit = bytes(proof.commit.base64, 256 * 1024);
45
+ if (proof.commit.hash !== context.commitHash || objectHash("commit", commit, width) !== context.commitHash)
46
+ throw new Error("connected_git_commit_mismatch");
47
+ const firstNewline = commit.indexOf(10);
48
+ if (firstNewline < 0)
49
+ throw new Error("connected_git_commit_invalid");
50
+ const firstLine = commit.subarray(0, firstNewline).toString("ascii");
51
+ if (!new RegExp(`^tree [a-f0-9]{${width}}$`).test(firstLine))
52
+ throw new Error("connected_git_commit_invalid");
53
+ const trees = new Map();
54
+ let totalBytes = commit.byteLength;
55
+ for (const object of proof.trees) {
56
+ const value = bytes(object.base64, 2 * 1024 * 1024);
57
+ totalBytes += value.byteLength;
58
+ if (totalBytes > 8 * 1024 * 1024 || trees.has(object.hash) || object.hash.length !== width
59
+ || objectHash("tree", value, width) !== object.hash)
60
+ throw new Error("connected_git_tree_mismatch");
61
+ trees.set(object.hash, value);
62
+ }
63
+ const parts = proof.path.split("/");
64
+ const visited = new Set();
65
+ let hash = firstLine.slice(5);
66
+ let mode = "";
67
+ for (const [index, part] of parts.entries()) {
68
+ const tree = trees.get(hash);
69
+ if (!tree || visited.has(hash))
70
+ throw new Error("connected_git_tree_missing");
71
+ visited.add(hash);
72
+ const entry = treeEntry(tree, part, width);
73
+ mode = entry.mode;
74
+ if (index < parts.length - 1 ? mode !== "40000" : !["100644", "100755"].includes(mode))
75
+ throw new Error("connected_git_file_mode_invalid");
76
+ hash = entry.hash;
77
+ }
78
+ if (visited.size !== trees.size)
79
+ throw new Error("connected_git_unreferenced_tree");
80
+ const content = bytes(proof.contentsBase64, 2 * 1024 * 1024);
81
+ if (objectHash("blob", content, width) !== hash)
82
+ throw new Error("connected_git_blob_mismatch");
83
+ return { path: proof.path, commitHash: context.commitHash, gitObjectHash: hash, mode,
84
+ sizeBytes: content.byteLength, sha256: createHash("sha256").update(content).digest("hex"),
85
+ contentsBase64: content.toString("base64") };
86
+ }
@@ -0,0 +1,81 @@
1
+ import { contentHash } from "@openpond/harness";
2
+ import { ConnectedSessionSchema, resolveConnectedBoundaryMetadata } from "./contracts.js";
3
+ import { recordedCodexCommands, recordedCodexCommandResults, isRecordedReadOnlyInventory } from "./codex-command-observations.js";
4
+ import { verifyConnectedGitFileProof } from "./git-file-proof.js";
5
+ function object(value) {
6
+ if (!value || typeof value !== "object" || Array.isArray(value))
7
+ throw new Error("connected_starting_state_unavailable");
8
+ return value;
9
+ }
10
+ function dirtyPaths(output) {
11
+ const paths = [];
12
+ for (const line of output.split("\n")) {
13
+ if (!line)
14
+ continue;
15
+ // Quoted/escaped paths and rename pairs need a distinct binary porcelain
16
+ // protocol. Treat them as unavailable instead of guessing path identities.
17
+ if (!/^[ MADRCU?!]{2} [^\x00-\x1f\x7f]+$/.test(line) || line.slice(3).includes('"')
18
+ || line.slice(3).includes(" -> ") || line.slice(3).includes("\\"))
19
+ throw new Error("connected_git_status_format_unsupported");
20
+ const path = line.slice(3);
21
+ if (path.startsWith("/") || path.split("/").some(part => part === "." || part === ".."))
22
+ throw new Error("connected_git_status_format_unsupported");
23
+ paths.push(path.replace(/\/$/, ""));
24
+ }
25
+ return paths;
26
+ }
27
+ /** Verify only the original first-tool boundary. Later requests need their own
28
+ * captured state. Both the repository context and observations come from the
29
+ * authorized immutable snapshot, never caller-supplied cleanliness claims. */
30
+ export function verifyConnectedGitStartingAssets(input) {
31
+ const session = ConnectedSessionSchema.parse(input.session);
32
+ const boundary = resolveConnectedBoundaryMetadata(session, input.boundaryId);
33
+ const context = session.repositoryContext;
34
+ if (session.origin !== "codex" || !context?.workingDirectoryHash || !context.recordedAt || boundary.projection !== "turn"
35
+ || !session.sourceFiles.some(file => file.contentHash === context.sourceFileHash)
36
+ || !input.proofs.length || input.proofs.length > 100)
37
+ throw new Error("connected_starting_state_unavailable");
38
+ const firstCallIndex = session.events.findIndex(event => event.kind === "tool_call");
39
+ if (firstCallIndex <= boundary.start || firstCallIndex >= boundary.end
40
+ || session.events.slice(0, firstCallIndex).some(event => event.kind === "compaction"))
41
+ throw new Error("connected_starting_state_boundary_unavailable");
42
+ const callEvent = session.events[firstCallIndex];
43
+ const requestEvent = session.events[boundary.start];
44
+ if (!callEvent.occurredAt || !requestEvent.occurredAt
45
+ || Date.parse(requestEvent.occurredAt) < Date.parse(context.recordedAt)
46
+ || Date.parse(callEvent.occurredAt) < Date.parse(requestEvent.occurredAt))
47
+ throw new Error("connected_starting_state_observation_time_unavailable");
48
+ const call = object(callEvent.content);
49
+ if (call.name !== "exec" || typeof call.input !== "string" || !callEvent.callId)
50
+ throw new Error("connected_starting_state_command_unsupported");
51
+ const commands = recordedCodexCommands(call.input);
52
+ const statusIndex = commands.findIndex(command => command.cmd === "git status --short");
53
+ if (statusIndex !== commands.length - 1 || statusIndex < 1
54
+ || commands.some(command => command.cwd !== null)
55
+ || !commands.slice(0, statusIndex).every(command => isRecordedReadOnlyInventory(command.cmd)))
56
+ throw new Error("connected_starting_state_command_unsupported");
57
+ const observations = session.events.slice(firstCallIndex + 1, boundary.end);
58
+ const resultIndex = observations.findIndex(event => event.kind === "tool_result" && event.callId === callEvent.callId);
59
+ if (resultIndex < 0 || observations.slice(0, resultIndex).some(event => ["tool_call", "tool_result", "compaction", "message"].includes(event.kind))
60
+ || observations.filter(event => event.kind === "tool_result" && event.callId === callEvent.callId).length !== 1)
61
+ throw new Error("connected_starting_state_observation_unavailable");
62
+ const resultEvent = observations[resultIndex];
63
+ const results = recordedCodexCommandResults(resultEvent.content, commands.length);
64
+ if (results.some((result, index) => result.originalTokenCount >= commands[index].maxOutputTokens
65
+ || /output truncated|truncated output|\[\.\.\./i.test(result.output)))
66
+ throw new Error("connected_starting_state_observation_truncated");
67
+ if (contentHash(results[0].output.split("\n")[0]) !== context.workingDirectoryHash)
68
+ throw new Error("connected_starting_state_directory_mismatch");
69
+ const dirty = dirtyPaths(results[statusIndex].output);
70
+ const assets = input.proofs.map(proof => verifyConnectedGitFileProof(context, proof));
71
+ if (new Set(assets.map(asset => asset.path)).size !== assets.length)
72
+ throw new Error("connected_starting_state_duplicate_path");
73
+ for (const asset of assets)
74
+ if (dirty.some(path => asset.path === path || asset.path.startsWith(path + "/")))
75
+ throw new Error("connected_starting_state_selected_file_modified");
76
+ return { schemaVersion: "openpond.connectedGitStartingAssets.v1",
77
+ sessionHash: session.contentHash, boundaryId: boundary.id, inputHash: boundary.inputHash,
78
+ revisionHash: boundary.revisionHash, commitHash: context.commitHash,
79
+ observationHash: contentHash({ call: callEvent, result: resultEvent }),
80
+ selectedPathsUnmodified: true, dependencyClosureQualified: false, assets };
81
+ }
@@ -0,0 +1,2 @@
1
+ export * from "./git-file-proof.js";
2
+ export * from "./git-starting-state.js";
@@ -10,7 +10,7 @@ import { parseClaudeSession } from "./sources/claude-code.js";
10
10
  import { parseHermesEvidence, parseOpenClawEvidence, } from "./sources/history.js";
11
11
  /** Cache identity includes source projection policy, independent of file mtimes. */
12
12
  export function agentImportNormalizerRevision(source) {
13
- return source === "codex" ? `${CONNECTED_NORMALIZER_VERSION}/${CODEX_AUTHORIZATION_PROJECTION_POLICY}` : CONNECTED_NORMALIZER_VERSION;
13
+ return source === "codex" ? `${CONNECTED_NORMALIZER_VERSION}/${CODEX_AUTHORIZATION_PROJECTION_POLICY}/fork-repository-starting-state-v1` : CONNECTED_NORMALIZER_VERSION;
14
14
  }
15
15
  /** Never admits malformed files partly or executes referenced paths/commands. */
16
16
  export function previewAgentImport(input) {
@@ -49,6 +49,7 @@ export function previewAgentImport(input) {
49
49
  warnings: session.warnings,
50
50
  contextComplete: !session.boundaries.some((boundary) => boundary.coverage.context === "unknown"),
51
51
  acquisition: input.acquisition,
52
+ ...(session.repositoryContext ? { repositoryContext: session.repositoryContext } : {}),
52
53
  })
53
54
  : session));
54
55
  }
@@ -3,3 +3,5 @@ export * from "./normalize.js";
3
3
  export * from "./imports.js";
4
4
  export * from "./usage-summary.js";
5
5
  export * from "./python-correctness.js";
6
+ export * from "./collector-coverage-contracts.js";
7
+ export * from "./git-file-contracts.js";
@@ -67,6 +67,7 @@ export function normalizeConnectedSession(input) {
67
67
  origin: input.origin, sessionId: input.sessionId, branchId: input.branchId ?? null, parentSessionId: input.parentSessionId ?? null,
68
68
  exporterVersion: input.exporterVersion ?? null, sourceFiles: input.files.map(connectedFileIdentity).sort((a, b) => a.path.localeCompare(b.path)),
69
69
  ...(input.acquisition ? { acquisition: input.acquisition } : {}),
70
+ ...(input.repositoryContext ? { repositoryContext: input.repositoryContext } : {}),
70
71
  events, unmappedEvents: (input.unmappedEvents ?? []).map((event, sequence) => ({ ...event, sequence })), boundaries, warnings: input.warnings ?? [] };
71
72
  const session = ConnectedSessionSchema.parse({ ...body, contentHash: contentHash(body) });
72
73
  if (new TextEncoder().encode(JSON.stringify(session)).length > CONNECTED_EVIDENCE_LIMITS.decodedBytes)
@@ -0,0 +1,33 @@
1
+ /** Forked Codex rollouts retain their own header followed by ancestor headers.
2
+ * Only the declared, contiguous ancestry is metadata for that source. */
3
+ export class CodexMetadataChain {
4
+ metadata = null;
5
+ parentId = null;
6
+ closed = false;
7
+ ids = new Set();
8
+ accept(row) {
9
+ if (row.type !== "session_meta") {
10
+ this.closed = true;
11
+ return true;
12
+ }
13
+ const payload = row.payload;
14
+ if (this.closed || !payload || typeof payload !== "object" || Array.isArray(payload))
15
+ return false;
16
+ const value = payload;
17
+ if (typeof value.id !== "string" || !value.id || value.id.length > 500
18
+ || this.ids.has(value.id) || this.ids.size >= 16)
19
+ return false;
20
+ if (this.metadata) {
21
+ if (value.id !== this.parentId)
22
+ return false;
23
+ for (const key of ["creator_user_id", "creator_account_id"])
24
+ if (this.metadata[key] != null && value[key] != null && this.metadata[key] !== value[key])
25
+ return false;
26
+ }
27
+ else
28
+ this.metadata = value;
29
+ this.ids.add(value.id);
30
+ this.parentId = typeof value.forked_from_id === "string" ? value.forked_from_id : null;
31
+ return true;
32
+ }
33
+ }
@@ -1,11 +1,24 @@
1
1
  import { contentHash } from "@openpond/harness";
2
- import { connectedTimestamp, event, json, normalizeConnectedSession, object } from "../normalize.js";
2
+ import { connectedFileIdentity, connectedTimestamp, event, json, normalizeConnectedSession, object } from "../normalize.js";
3
3
  import { projectCodexAuthorization } from "./codex-authorization.js";
4
4
  import { sourceUsage } from "./usage.js";
5
+ import { CodexMetadataChain } from "./codex-metadata.js";
5
6
  export function parseCodexSession(file, rows) {
6
- const metadata = object(rows.find(row => row.type === "session_meta")?.payload);
7
+ const chain = new CodexMetadataChain();
8
+ if (rows.some(row => !chain.accept(row)))
9
+ throw new Error("Codex rollout metadata does not match its declared fork ancestry.");
10
+ const metadata = chain.metadata ?? {};
7
11
  if (typeof metadata.id !== "string")
8
12
  throw new Error("This is not a supported Codex rollout: session_meta.id is missing.");
13
+ const git = object(metadata.git);
14
+ const repositoryContext = typeof git.commit_hash === "string" && /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(git.commit_hash)
15
+ ? { schemaVersion: "openpond.connectedRepositoryContext.v1", kind: "git",
16
+ commitHash: git.commit_hash, branch: typeof git.branch === "string" && git.branch.length > 0 && git.branch.length <= 500 ? git.branch : null,
17
+ recordedAt: connectedTimestamp(metadata.timestamp), sourceFileHash: connectedFileIdentity(file).contentHash,
18
+ metadataHash: contentHash(git) }
19
+ : undefined;
20
+ if (repositoryContext && typeof metadata.cwd === "string")
21
+ Object.assign(repositoryContext, { workingDirectoryHash: contentHash(metadata.cwd) });
9
22
  const events = [];
10
23
  let turnId = null;
11
24
  let authorizationOmissions = 0;
@@ -52,5 +65,6 @@ export function parseCodexSession(file, rows) {
52
65
  }
53
66
  return normalizeConnectedSession({ origin: "codex", sessionId: metadata.id, parentSessionId: typeof metadata.forked_from_id === "string" ? metadata.forked_from_id : null,
54
67
  exporterVersion: typeof metadata.cli_version === "string" ? metadata.cli_version : null, files: [file], events,
68
+ ...(repositoryContext ? { repositoryContext } : {}),
55
69
  warnings: ["Codex cumulative usage is retained as reported totals and is not summed as billed invocations.", "Referenced files are unavailable unless separately admitted.", ...(authorizationOmissions ? [`Codex native authorization state excluded by codex-native-authorization-v1 from ${authorizationOmissions} locations. Omission markers retain source-section hashes; sourceFiles retains original byte hashes. Other conversation evidence is unchanged and still requires privacy admission.`] : [])] });
56
70
  }
@@ -83,7 +83,7 @@ export const VerifierSetReleaseContentSchema = z.object({
83
83
  schemaVersion: z.literal("openpond.verifierSetRelease.v1"),
84
84
  id: ReleaseIdSchema,
85
85
  revision: z.number().int().positive(),
86
- graders: z.array(GraderSpecSchema).min(1).max(1_000),
86
+ graders: z.array(GraderSpecSchema).max(1_000),
87
87
  isolation: z.object({
88
88
  processBoundary: z.enum(["same_process", "isolated_process", "container"]),
89
89
  networkPolicy: z.literal("none"),
package/dist/graders.js CHANGED
@@ -39,9 +39,15 @@ export const GraderEvidenceContentSchema = z.object({
39
39
  modelJudgeReceipt: ModelJudgeReceiptSchema.optional(),
40
40
  }).strict();
41
41
  export const GraderEvidenceSchema = GraderEvidenceContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
42
+ /** Scope is part of the immutable grader. An absent scope applies to every
43
+ * task; an explicit scope is never inferred from answer content or metadata. */
44
+ export function gradersForTask(graders, task) {
45
+ return graders.filter(grader => !grader.taskIds || grader.taskIds.includes(task.id));
46
+ }
42
47
  export async function gradeEvidence(input) {
48
+ const graders = gradersForTask(input.graders, input.task);
43
49
  if (input.evidence.infrastructureError) {
44
- return input.graders.map((grader) => evidence(grader, {
50
+ return graders.map((grader) => evidence(grader, {
45
51
  score: null,
46
52
  passed: false,
47
53
  rewardEligible: false,
@@ -51,7 +57,7 @@ export async function gradeEvidence(input) {
51
57
  privilegedEvidenceRefs: [],
52
58
  }));
53
59
  }
54
- return Promise.all(input.graders.map(async (grader) => {
60
+ return Promise.all(graders.map(async (grader) => {
55
61
  if (grader.kind === "model_judge") {
56
62
  if (!input.modelJudge || (grader.calibrationStatus !== "passed" && input.purpose !== "fixture_calibration"))
57
63
  return evidence(grader, unavailable("Model judge is unavailable or uncalibrated."));
@@ -148,7 +154,7 @@ export async function gradeTaskEvidence(input) {
148
154
  const components = await gradeEvidence(input);
149
155
  const content = { schemaVersion: "openpond.taskGrade.v1",
150
156
  taskHash: contentHash(input.task), evidenceHash: contentHash(input.evidence), graderSetHash: contentHash(input.graders), components,
151
- ...aggregateGraderScores({ graders: input.graders, components, infrastructureError: input.evidence.infrastructureError }) };
157
+ ...aggregateGraderScores({ graders: gradersForTask(input.graders, input.task), components, infrastructureError: input.evidence.infrastructureError }) };
152
158
  return TaskGradeSchema.parse({ ...content, contentHash: contentHash(content) });
153
159
  }
154
160
  /** Verify retained results against the exact admitted inputs without rerunning
@@ -163,7 +169,7 @@ export function verifyTaskGrade(value, input) {
163
169
  if (componentHash !== contentHash(body))
164
170
  throw new Error("Grader evidence integrity failed.");
165
171
  }
166
- const summary = aggregateGraderScores({ graders: input.graders, components: result.components, infrastructureError: input.evidence.infrastructureError });
172
+ const summary = aggregateGraderScores({ graders: gradersForTask(input.graders, input.task), components: result.components, infrastructureError: input.evidence.infrastructureError });
167
173
  if (contentHash(GradeSummarySchema.parse({ score: result.score, passed: result.passed, rewardEligible: result.rewardEligible, failureClass: result.failureClass, gradingStatus: result.gradingStatus })) !== contentHash(summary))
168
174
  throw new Error("Task grade summary differs from its grader evidence.");
169
175
  return result;