@openpond/evals 0.11.20 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/dist/connected-evidence/codex-command-observations.js +81 -0
- package/dist/connected-evidence/collector-coverage-contracts.js +160 -0
- package/dist/connected-evidence/contracts.js +13 -0
- package/dist/connected-evidence/git-file-contracts.js +16 -0
- package/dist/connected-evidence/git-file-proof.js +86 -0
- package/dist/connected-evidence/git-starting-state.js +81 -0
- package/dist/connected-evidence/git.js +2 -0
- package/dist/connected-evidence/imports.js +2 -1
- package/dist/connected-evidence/index.js +2 -0
- package/dist/connected-evidence/normalize.js +1 -0
- package/dist/connected-evidence/sources/codex-metadata.js +33 -0
- package/dist/connected-evidence/sources/codex.js +16 -2
- package/dist/execution-contracts.js +1 -1
- package/dist/graders.js +10 -4
- package/dist/javascript-isolate-node.js +2 -1
- package/dist/javascript-isolate-process-source.js +1 -1
- package/dist/javascript-isolate-process.js +2 -1
- package/dist/javascript-isolate.js +2 -1
- package/dist/javascript-verifier-worker-source.js +1 -1
- package/dist/javascript-verifier.js +1 -0
- package/dist/native-conversations/codex-window-inventory.js +96 -0
- package/dist/native-conversations/collector-acquisition.js +149 -0
- package/dist/native-conversations/collector-branches.js +1 -0
- package/dist/native-conversations/collector-calendar-gate.js +19 -0
- package/dist/native-conversations/collector-contracts.js +4 -3
- package/dist/native-conversations/collector-coverage.js +100 -0
- package/dist/native-conversations/collector-cron.js +43 -0
- package/dist/native-conversations/collector-schedule.js +61 -0
- package/dist/native-conversations/collector-service-files.js +51 -0
- package/dist/native-conversations/collector-snapshots.js +70 -0
- package/dist/native-conversations/collector-source-cache.js +41 -0
- package/dist/native-conversations/collector-store.js +43 -8
- package/dist/native-conversations/collector.js +189 -261
- package/dist/native-conversations/git-assets.js +61 -0
- package/dist/native-conversations/history.js +9 -0
- package/dist/native-conversations/index.js +2 -0
- package/dist/native-conversations/location.js +4 -2
- package/dist/native-conversations/supervisor.js +131 -81
- package/dist/prepared-process-isolation.js +131 -0
- package/dist/rewards.js +17 -5
- package/dist/sql-execution-node.js +2 -1
- package/dist/tasksets.js +7 -1
- package/dist/types/builtin-benchmarks/harness-refiner.d.ts +4 -0
- package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map +1 -1
- package/dist/types/conformance.d.ts +8 -0
- package/dist/types/conformance.d.ts.map +1 -1
- package/dist/types/connected-evidence/codex-command-observations.d.ts +15 -0
- package/dist/types/connected-evidence/codex-command-observations.d.ts.map +1 -0
- package/dist/types/connected-evidence/collector-coverage-contracts.d.ts +98 -0
- package/dist/types/connected-evidence/collector-coverage-contracts.d.ts.map +1 -0
- package/dist/types/connected-evidence/contracts.d.ts +22 -0
- package/dist/types/connected-evidence/contracts.d.ts.map +1 -1
- package/dist/types/connected-evidence/git-file-contracts.d.ts +18 -0
- package/dist/types/connected-evidence/git-file-contracts.d.ts.map +1 -0
- package/dist/types/connected-evidence/git-file-proof.d.ts +13 -0
- package/dist/types/connected-evidence/git-file-proof.d.ts.map +1 -0
- package/dist/types/connected-evidence/git-starting-state.d.ts +28 -0
- package/dist/types/connected-evidence/git-starting-state.d.ts.map +1 -0
- package/dist/types/connected-evidence/git.d.ts +3 -0
- package/dist/types/connected-evidence/git.d.ts.map +1 -0
- package/dist/types/connected-evidence/imports.d.ts.map +1 -1
- package/dist/types/connected-evidence/index.d.ts +2 -0
- package/dist/types/connected-evidence/index.d.ts.map +1 -1
- package/dist/types/connected-evidence/normalize.d.ts +1 -0
- package/dist/types/connected-evidence/normalize.d.ts.map +1 -1
- package/dist/types/connected-evidence/sources/claude-code.d.ts +10 -0
- package/dist/types/connected-evidence/sources/claude-code.d.ts.map +1 -1
- package/dist/types/connected-evidence/sources/codex-metadata.d.ts +10 -0
- package/dist/types/connected-evidence/sources/codex-metadata.d.ts.map +1 -0
- package/dist/types/connected-evidence/sources/codex.d.ts +10 -0
- package/dist/types/connected-evidence/sources/codex.d.ts.map +1 -1
- package/dist/types/connected-evidence/sources/grok-build.d.ts +10 -0
- package/dist/types/connected-evidence/sources/grok-build.d.ts.map +1 -1
- package/dist/types/connected-evidence/sources/history.d.ts +20 -0
- package/dist/types/connected-evidence/sources/history.d.ts.map +1 -1
- package/dist/types/connected-evidence/sources/opencode.d.ts +10 -0
- package/dist/types/connected-evidence/sources/opencode.d.ts.map +1 -1
- package/dist/types/connected-evidence/sources/pi.d.ts +10 -0
- package/dist/types/connected-evidence/sources/pi.d.ts.map +1 -1
- package/dist/types/execution-contracts.d.ts +8 -0
- package/dist/types/execution-contracts.d.ts.map +1 -1
- package/dist/types/graders.d.ts +3 -0
- package/dist/types/graders.d.ts.map +1 -1
- package/dist/types/javascript-isolate-node.d.ts.map +1 -1
- package/dist/types/javascript-isolate-process.d.ts.map +1 -1
- package/dist/types/javascript-isolate.d.ts.map +1 -1
- package/dist/types/javascript-verifier.d.ts +1 -0
- package/dist/types/javascript-verifier.d.ts.map +1 -1
- package/dist/types/learning/admission.d.ts +1 -0
- package/dist/types/learning/admission.d.ts.map +1 -1
- package/dist/types/learning/authoring-service.d.ts +2 -0
- package/dist/types/learning/authoring-service.d.ts.map +1 -1
- package/dist/types/learning/authoring.d.ts +8 -0
- package/dist/types/learning/authoring.d.ts.map +1 -1
- package/dist/types/learning/batch-service.d.ts +4 -0
- package/dist/types/learning/batch-service.d.ts.map +1 -1
- package/dist/types/learning/contracts.d.ts +17 -0
- package/dist/types/learning/contracts.d.ts.map +1 -1
- package/dist/types/learning/operations.d.ts +8 -0
- package/dist/types/learning/operations.d.ts.map +1 -1
- package/dist/types/learning/repository.d.ts +14 -0
- package/dist/types/learning/repository.d.ts.map +1 -1
- package/dist/types/learning/reward-calibration-closure.d.ts +9 -1
- package/dist/types/learning/reward-calibration-closure.d.ts.map +1 -1
- package/dist/types/learning/reward-checks.d.ts +2 -0
- package/dist/types/learning/reward-checks.d.ts.map +1 -1
- package/dist/types/learning/transport.d.ts +18 -0
- package/dist/types/learning/transport.d.ts.map +1 -1
- package/dist/types/native-conversations/codex-window-inventory.d.ts +12 -0
- package/dist/types/native-conversations/codex-window-inventory.d.ts.map +1 -0
- package/dist/types/native-conversations/collector-acquisition.d.ts +17 -0
- package/dist/types/native-conversations/collector-acquisition.d.ts.map +1 -0
- package/dist/types/native-conversations/collector-branches.d.ts.map +1 -1
- package/dist/types/native-conversations/collector-calendar-gate.d.ts +9 -0
- package/dist/types/native-conversations/collector-calendar-gate.d.ts.map +1 -0
- package/dist/types/native-conversations/collector-contracts.d.ts +41 -5
- package/dist/types/native-conversations/collector-contracts.d.ts.map +1 -1
- package/dist/types/native-conversations/collector-coverage.d.ts +62 -0
- package/dist/types/native-conversations/collector-coverage.d.ts.map +1 -0
- package/dist/types/native-conversations/collector-cron.d.ts +20 -0
- package/dist/types/native-conversations/collector-cron.d.ts.map +1 -0
- package/dist/types/native-conversations/collector-schedule.d.ts +13 -0
- package/dist/types/native-conversations/collector-schedule.d.ts.map +1 -0
- package/dist/types/native-conversations/collector-service-files.d.ts +20 -0
- package/dist/types/native-conversations/collector-service-files.d.ts.map +1 -0
- package/dist/types/native-conversations/collector-snapshots.d.ts +25 -0
- package/dist/types/native-conversations/collector-snapshots.d.ts.map +1 -0
- package/dist/types/native-conversations/collector-source-cache.d.ts +31 -0
- package/dist/types/native-conversations/collector-source-cache.d.ts.map +1 -0
- package/dist/types/native-conversations/collector-store.d.ts +11 -2
- package/dist/types/native-conversations/collector-store.d.ts.map +1 -1
- package/dist/types/native-conversations/collector.d.ts +5 -1
- package/dist/types/native-conversations/collector.d.ts.map +1 -1
- package/dist/types/native-conversations/git-assets.d.ts +32 -0
- package/dist/types/native-conversations/git-assets.d.ts.map +1 -0
- package/dist/types/native-conversations/history.d.ts +7 -0
- package/dist/types/native-conversations/history.d.ts.map +1 -1
- package/dist/types/native-conversations/index.d.ts +2 -0
- package/dist/types/native-conversations/index.d.ts.map +1 -1
- package/dist/types/native-conversations/location.d.ts.map +1 -1
- package/dist/types/native-conversations/supervisor.d.ts +4 -3
- package/dist/types/native-conversations/supervisor.d.ts.map +1 -1
- package/dist/types/prepared-process-isolation.d.ts +5 -0
- package/dist/types/prepared-process-isolation.d.ts.map +1 -0
- package/dist/types/review-conformance.d.ts +5 -5
- package/dist/types/rewards.d.ts +9 -0
- package/dist/types/rewards.d.ts.map +1 -1
- package/dist/types/sql-execution-node.d.ts.map +1 -1
- package/dist/types/tasksets.d.ts +16 -0
- package/dist/types/tasksets.d.ts.map +1 -1
- package/package.json +8 -2
- package/schemas/learning/v1/binding.schema.json +8 -0
- package/schemas/learning/v1/command-request.schema.json +8 -0
- package/schemas/learning/v1/decision.schema.json +5 -1
- package/schemas/learning/v1/draft.schema.json +8 -0
- package/schemas/learning/v1/grade.schema.json +5 -1
- package/schemas/learning/v1/package.schema.json +113 -94
- package/schemas/learning/v1/reward_check.schema.json +2 -1
package/README.md
CHANGED
|
@@ -163,6 +163,18 @@ overrides that state.
|
|
|
163
163
|
|
|
164
164
|
## Runtime adapters
|
|
165
165
|
|
|
166
|
+
Graders may declare `taskIds` to apply to a nonempty, unique population within
|
|
167
|
+
the Taskset. Grading binds the full grader-set hash while executing and
|
|
168
|
+
aggregating only the applicable checks. Reward binding sources carry the same
|
|
169
|
+
scope; scoped compositions retain `taskId` and an explicit `not_applicable`
|
|
170
|
+
result for each excluded source. Excluded checks cannot satisfy a required check
|
|
171
|
+
or change the score, and a composition without its scope is rejected.
|
|
172
|
+
|
|
173
|
+
`@openpond/evals/connected-evidence/git` verifies retained Git starting-file
|
|
174
|
+
receipts against exact object identities and bytes. The receipt is evidence,
|
|
175
|
+
not authorization to read a repository. Hosts remain responsible for owner
|
|
176
|
+
authorization, dependency closure, and runtime qualification.
|
|
177
|
+
|
|
166
178
|
Implement `HarnessRuntime` for environment state and `HarnessExecutor` around
|
|
167
179
|
the host's existing model/Agent loop. Do not reimplement prompting, tool dispatch,
|
|
168
180
|
session persistence, cost accounting, cancellation, or cleanup in this package.
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { parse } from "acorn";
|
|
2
|
+
const ast = (value) => {
|
|
3
|
+
if (!value || typeof value !== "object" || !("type" in value))
|
|
4
|
+
throw new Error("connected_command_syntax_unsupported");
|
|
5
|
+
return value;
|
|
6
|
+
};
|
|
7
|
+
function member(value, object, property) {
|
|
8
|
+
const node = ast(value);
|
|
9
|
+
return node.type === "MemberExpression" && !node.computed && !node.optional
|
|
10
|
+
&& ast(node.object).type === "Identifier" && ast(node.object).name === object
|
|
11
|
+
&& ast(node.property).type === "Identifier" && ast(node.property).name === property;
|
|
12
|
+
}
|
|
13
|
+
/** Interpret recorded syntax, never execute source code. Only sequential static
|
|
14
|
+
* exec_command calls have an unambiguous correspondence with retained results. */
|
|
15
|
+
export function recordedCodexCommands(source) {
|
|
16
|
+
if (source.length > 128_000)
|
|
17
|
+
throw new Error("connected_command_syntax_limit");
|
|
18
|
+
const program = parse(source, { ecmaVersion: "latest", sourceType: "module" });
|
|
19
|
+
const commands = [];
|
|
20
|
+
for (const statement of program.body) {
|
|
21
|
+
const outer = ast(ast(statement).expression);
|
|
22
|
+
const args = outer.arguments;
|
|
23
|
+
if (statement.type !== "ExpressionStatement" || outer.type !== "CallExpression" || outer.optional
|
|
24
|
+
|| ast(outer.callee).type !== "Identifier" || ast(outer.callee).name !== "text" || args.length !== 1)
|
|
25
|
+
throw new Error("connected_command_syntax_unsupported");
|
|
26
|
+
const awaited = ast(args[0]);
|
|
27
|
+
const call = ast(awaited.argument);
|
|
28
|
+
const innerArgs = call.arguments;
|
|
29
|
+
if (awaited.type !== "AwaitExpression" || call.type !== "CallExpression" || call.optional
|
|
30
|
+
|| !member(call.callee, "tools", "exec_command") || innerArgs.length !== 1)
|
|
31
|
+
throw new Error("connected_command_syntax_unsupported");
|
|
32
|
+
const object = ast(innerArgs[0]);
|
|
33
|
+
if (object.type !== "ObjectExpression")
|
|
34
|
+
throw new Error("connected_command_syntax_unsupported");
|
|
35
|
+
const values = new Map();
|
|
36
|
+
for (const raw of object.properties) {
|
|
37
|
+
const property = ast(raw), key = ast(property.key), value = ast(property.value);
|
|
38
|
+
const name = key.type === "Identifier" ? key.name : key.type === "Literal" ? key.value : null;
|
|
39
|
+
if (property.type !== "Property" || property.computed || property.method || property.shorthand || property.kind !== "init"
|
|
40
|
+
|| typeof name !== "string" || values.has(name) || value.type !== "Literal"
|
|
41
|
+
|| !["cmd", "workdir", "max_output_tokens", "yield_time_ms"].includes(name))
|
|
42
|
+
throw new Error("connected_command_syntax_unsupported");
|
|
43
|
+
values.set(name, value.value);
|
|
44
|
+
}
|
|
45
|
+
const cmd = values.get("cmd"), budget = values.get("max_output_tokens"), cwd = values.get("workdir");
|
|
46
|
+
if (typeof cmd !== "string" || !cmd.length || cmd.length > 16_000
|
|
47
|
+
|| typeof budget !== "number" || !Number.isInteger(budget) || budget <= 0 || budget > 100_000
|
|
48
|
+
|| cwd !== undefined && typeof cwd !== "string")
|
|
49
|
+
throw new Error("connected_command_syntax_unsupported");
|
|
50
|
+
commands.push({ cmd, maxOutputTokens: budget, cwd: typeof cwd === "string" ? cwd : null });
|
|
51
|
+
}
|
|
52
|
+
if (!commands.length || commands.length > 20)
|
|
53
|
+
throw new Error("connected_command_syntax_unsupported");
|
|
54
|
+
return commands;
|
|
55
|
+
}
|
|
56
|
+
/** This deliberately bounded read-only prefix establishes that no earlier
|
|
57
|
+
* recorded command modified the checkout before the status observation. */
|
|
58
|
+
export function isRecordedReadOnlyInventory(cmd) {
|
|
59
|
+
return cmd === "pwd" || /^pwd && rg --files(?: -g '[a-zA-Z0-9_.*\/-]+')*$/.test(cmd);
|
|
60
|
+
}
|
|
61
|
+
export function recordedCodexCommandResults(value, count) {
|
|
62
|
+
if (!Array.isArray(value) || value.length !== count + 1)
|
|
63
|
+
throw new Error("connected_command_results_incomplete");
|
|
64
|
+
const blocks = value.map(block => {
|
|
65
|
+
if (!block || typeof block !== "object" || !("type" in block) || block.type !== "input_text"
|
|
66
|
+
|| !("text" in block) || typeof block.text !== "string")
|
|
67
|
+
throw new Error("connected_command_results_incomplete");
|
|
68
|
+
return block.text;
|
|
69
|
+
});
|
|
70
|
+
if (!/^Script completed\nWall time [0-9.]+ seconds\nOutput:\n$/.test(blocks[0]))
|
|
71
|
+
throw new Error("connected_command_results_incomplete");
|
|
72
|
+
return blocks.slice(1).map(text => {
|
|
73
|
+
const result = JSON.parse(text);
|
|
74
|
+
if (!result || typeof result !== "object" || !("exit_code" in result) || result.exit_code !== 0
|
|
75
|
+
|| "session_id" in result || !("output" in result) || typeof result.output !== "string"
|
|
76
|
+
|| !("original_token_count" in result) || typeof result.original_token_count !== "number"
|
|
77
|
+
|| !Number.isSafeInteger(result.original_token_count) || result.original_token_count < 0)
|
|
78
|
+
throw new Error("connected_command_results_incomplete");
|
|
79
|
+
return { output: result.output, originalTokenCount: result.original_token_count };
|
|
80
|
+
});
|
|
81
|
+
}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { contentHash } from "@openpond/harness";
|
|
3
|
+
const Id = z.string().trim().min(1).max(500);
|
|
4
|
+
const Hash = z.string().regex(/^[a-f0-9]{64}$/);
|
|
5
|
+
const Timestamp = z.iso.datetime();
|
|
6
|
+
const Count = z.number().int().nonnegative().max(1_000_000);
|
|
7
|
+
export const CollectorCoverageSessionSchema = z
|
|
8
|
+
.object({
|
|
9
|
+
sourceKeyHash: Hash,
|
|
10
|
+
nativeRevisionHash: Hash.nullable(),
|
|
11
|
+
state: z.enum(["reading", "admitted", "skipped", "failed"]),
|
|
12
|
+
reason: z
|
|
13
|
+
.enum([
|
|
14
|
+
"no_eligible_boundaries",
|
|
15
|
+
"native_session_not_ready",
|
|
16
|
+
"source_read_failed",
|
|
17
|
+
"source_revision_unavailable",
|
|
18
|
+
])
|
|
19
|
+
.nullable(),
|
|
20
|
+
unknownTimeBoundaries: Count,
|
|
21
|
+
normalized: z
|
|
22
|
+
.array(z
|
|
23
|
+
.object({
|
|
24
|
+
sessionHash: Hash,
|
|
25
|
+
boundaries: z
|
|
26
|
+
.array(z
|
|
27
|
+
.object({ id: Id, inputHash: Hash, revisionHash: Hash, occurredAt: Timestamp })
|
|
28
|
+
.strict())
|
|
29
|
+
.max(5_000),
|
|
30
|
+
})
|
|
31
|
+
.strict())
|
|
32
|
+
.max(1_000),
|
|
33
|
+
})
|
|
34
|
+
.strict();
|
|
35
|
+
export const CollectorCoverageManifestSchema = z
|
|
36
|
+
.object({
|
|
37
|
+
schemaVersion: z.literal("openpond.collectorCoverageManifest.v1"),
|
|
38
|
+
jobId: Id,
|
|
39
|
+
connectionId: Id,
|
|
40
|
+
connectionRevision: z.number().int().positive(),
|
|
41
|
+
machineId: Id,
|
|
42
|
+
sourceInstanceId: Id,
|
|
43
|
+
from: Timestamp.nullable(),
|
|
44
|
+
to: Timestamp,
|
|
45
|
+
startedAt: Timestamp,
|
|
46
|
+
finishedAt: Timestamp.nullable(),
|
|
47
|
+
listingComplete: z.boolean(),
|
|
48
|
+
pendingOperations: Count,
|
|
49
|
+
sessions: z.array(CollectorCoverageSessionSchema).max(10_000),
|
|
50
|
+
})
|
|
51
|
+
.strict()
|
|
52
|
+
.superRefine((value, ctx) => {
|
|
53
|
+
if (value.from !== null && Date.parse(value.from) >= Date.parse(value.to))
|
|
54
|
+
ctx.addIssue({
|
|
55
|
+
code: "custom",
|
|
56
|
+
message: "Collector source window is empty.",
|
|
57
|
+
});
|
|
58
|
+
if (value.finishedAt !== null &&
|
|
59
|
+
Date.parse(value.finishedAt) < Date.parse(value.startedAt))
|
|
60
|
+
ctx.addIssue({
|
|
61
|
+
code: "custom",
|
|
62
|
+
message: "Collector finish time precedes its start.",
|
|
63
|
+
});
|
|
64
|
+
if (new Set(value.sessions.map((session) => session.sourceKeyHash)).size !==
|
|
65
|
+
value.sessions.length)
|
|
66
|
+
ctx.addIssue({
|
|
67
|
+
code: "custom",
|
|
68
|
+
message: "Collector source inventory is duplicated.",
|
|
69
|
+
});
|
|
70
|
+
let boundaries = 0;
|
|
71
|
+
for (const session of value.sessions)
|
|
72
|
+
for (const normalized of session.normalized) {
|
|
73
|
+
boundaries += normalized.boundaries.length;
|
|
74
|
+
if (new Set(normalized.boundaries.map((boundary) => boundary.id)).size !==
|
|
75
|
+
normalized.boundaries.length)
|
|
76
|
+
ctx.addIssue({
|
|
77
|
+
code: "custom",
|
|
78
|
+
message: "Normalized source boundaries are duplicated.",
|
|
79
|
+
});
|
|
80
|
+
for (const boundary of normalized.boundaries)
|
|
81
|
+
if (Date.parse(boundary.occurredAt) >= Date.parse(value.to) ||
|
|
82
|
+
(value.from !== null &&
|
|
83
|
+
Date.parse(boundary.occurredAt) < Date.parse(value.from)))
|
|
84
|
+
ctx.addIssue({
|
|
85
|
+
code: "custom",
|
|
86
|
+
message: "Eligible boundary is outside the declared event-time window.",
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
if (boundaries > 20_000)
|
|
90
|
+
ctx.addIssue({
|
|
91
|
+
code: "custom",
|
|
92
|
+
message: "Collector coverage exceeds its bounded boundary inventory.",
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
export const CollectorCoverageSummarySchema = z
|
|
96
|
+
.object({
|
|
97
|
+
jobId: Id,
|
|
98
|
+
manifestHash: Hash,
|
|
99
|
+
sourceInventoryHash: Hash,
|
|
100
|
+
eligibleBoundaryHash: Hash,
|
|
101
|
+
from: Timestamp.nullable(),
|
|
102
|
+
to: Timestamp,
|
|
103
|
+
finishedAt: Timestamp.nullable(),
|
|
104
|
+
conversations: Count,
|
|
105
|
+
eligibleTurns: Count,
|
|
106
|
+
admittedSessions: Count,
|
|
107
|
+
skippedSessions: Count,
|
|
108
|
+
failedSessions: Count,
|
|
109
|
+
unknownTimeBoundaries: Count,
|
|
110
|
+
pendingOperations: Count,
|
|
111
|
+
listingComplete: z.boolean(),
|
|
112
|
+
complete: z.boolean(),
|
|
113
|
+
})
|
|
114
|
+
.strict();
|
|
115
|
+
export function hashCollectorEligibleBoundaries(input) {
|
|
116
|
+
const boundaries = new Map(input.map((boundary) => [
|
|
117
|
+
JSON.stringify([boundary.id, boundary.inputHash, boundary.revisionHash]),
|
|
118
|
+
{ id: boundary.id, inputHash: boundary.inputHash, revisionHash: boundary.revisionHash },
|
|
119
|
+
]));
|
|
120
|
+
const ordered = [...boundaries]
|
|
121
|
+
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
|
|
122
|
+
.map(([, boundary]) => boundary);
|
|
123
|
+
return { hash: contentHash(ordered), count: ordered.length };
|
|
124
|
+
}
|
|
125
|
+
export function summarizeCollectorCoverage(raw) {
|
|
126
|
+
const manifest = CollectorCoverageManifestSchema.parse(raw);
|
|
127
|
+
const boundaries = hashCollectorEligibleBoundaries(manifest.sessions.flatMap((session) => session.normalized.flatMap((normalized) => normalized.boundaries)));
|
|
128
|
+
const unknownTimeBoundaries = manifest.sessions.reduce((sum, session) => sum + session.unknownTimeBoundaries, 0);
|
|
129
|
+
return CollectorCoverageSummarySchema.parse({
|
|
130
|
+
jobId: manifest.jobId,
|
|
131
|
+
manifestHash: contentHash(manifest),
|
|
132
|
+
sourceInventoryHash: contentHash(manifest.sessions
|
|
133
|
+
.map((session) => ({
|
|
134
|
+
sourceKeyHash: session.sourceKeyHash,
|
|
135
|
+
nativeRevisionHash: session.nativeRevisionHash,
|
|
136
|
+
snapshots: session.normalized.map((normalized) => normalized.sessionHash),
|
|
137
|
+
}))
|
|
138
|
+
.sort((a, b) => a.sourceKeyHash.localeCompare(b.sourceKeyHash))),
|
|
139
|
+
eligibleBoundaryHash: boundaries.hash,
|
|
140
|
+
from: manifest.from,
|
|
141
|
+
to: manifest.to,
|
|
142
|
+
finishedAt: manifest.finishedAt,
|
|
143
|
+
conversations: manifest.sessions.length,
|
|
144
|
+
eligibleTurns: boundaries.count,
|
|
145
|
+
admittedSessions: manifest.sessions.filter((session) => session.state === "admitted").length,
|
|
146
|
+
skippedSessions: manifest.sessions.filter((session) => session.state === "skipped").length,
|
|
147
|
+
failedSessions: manifest.sessions.filter((session) => session.state === "failed").length,
|
|
148
|
+
unknownTimeBoundaries,
|
|
149
|
+
pendingOperations: manifest.pendingOperations,
|
|
150
|
+
listingComplete: manifest.listingComplete,
|
|
151
|
+
complete: manifest.finishedAt !== null &&
|
|
152
|
+
manifest.listingComplete &&
|
|
153
|
+
manifest.pendingOperations === 0 &&
|
|
154
|
+
unknownTimeBoundaries === 0 &&
|
|
155
|
+
manifest.sessions.every((session) => session.nativeRevisionHash !== null &&
|
|
156
|
+
(session.state === "admitted" ||
|
|
157
|
+
(session.state === "skipped" &&
|
|
158
|
+
session.reason === "no_eligible_boundaries"))),
|
|
159
|
+
});
|
|
160
|
+
}
|
|
@@ -29,11 +29,24 @@ export const ConnectedBoundarySchema = z.object({
|
|
|
29
29
|
process: z.enum(["retained", "partial", "absent"]), artifacts: z.enum(["retained", "references_only", "absent"]),
|
|
30
30
|
replay: z.literal("unavailable"), gaps: z.array(z.string().max(500)).max(100) }).strict(),
|
|
31
31
|
}).strict();
|
|
32
|
+
/** Source-reported repository identity is reconstruction evidence, not an
|
|
33
|
+
* admitted checkout or proof that the working tree was clean. */
|
|
34
|
+
export const ConnectedRepositoryContextSchema = z.object({
|
|
35
|
+
schemaVersion: z.literal("openpond.connectedRepositoryContext.v1"),
|
|
36
|
+
kind: z.literal("git"),
|
|
37
|
+
commitHash: z.string().regex(/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/),
|
|
38
|
+
branch: z.string().min(1).max(500).nullable(),
|
|
39
|
+
recordedAt: Time,
|
|
40
|
+
sourceFileHash: Hash,
|
|
41
|
+
metadataHash: Hash,
|
|
42
|
+
workingDirectoryHash: Hash.optional(),
|
|
43
|
+
}).strict();
|
|
32
44
|
export const ConnectedSessionSchema = z.object({
|
|
33
45
|
schemaVersion: z.literal(CONNECTED_EVIDENCE_VERSION), normalizerVersion: z.literal(CONNECTED_NORMALIZER_VERSION),
|
|
34
46
|
origin: ConnectedSourceKindSchema, sessionId: Id, branchId: Id.nullable(), parentSessionId: Id.nullable(),
|
|
35
47
|
exporterVersion: z.string().max(200).nullable(),
|
|
36
48
|
acquisition: z.object({ machineId: Id, sourceInstanceId: Id }).strict().optional(),
|
|
49
|
+
repositoryContext: ConnectedRepositoryContextSchema.optional(),
|
|
37
50
|
sourceFiles: z.array(z.object({ path: Id, contentHash: Hash, sizeBytes: z.number().int().nonnegative() }).strict()).max(CONNECTED_EVIDENCE_LIMITS.files),
|
|
38
51
|
events: z.array(ConnectedEventSchema).max(CONNECTED_EVIDENCE_LIMITS.events),
|
|
39
52
|
unmappedEvents: z.array(ConnectedEventSchema).max(CONNECTED_EVIDENCE_LIMITS.events),
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
const ObjectHash = z.string().regex(/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/);
|
|
3
|
+
const Base64 = z.string().max(3_000_000);
|
|
4
|
+
const ObjectBytes = z.object({ hash: ObjectHash, base64: Base64 }).strict();
|
|
5
|
+
const safePath = (path) => !/[\x00-\x1f\x7f]/.test(path) && !path.startsWith("/") && !path.includes("\\")
|
|
6
|
+
&& path.split("/").length <= 32 && path.split("/").every(part => part && part !== "." && part !== ".."
|
|
7
|
+
&& part !== ".git" && !part.startsWith(".env"));
|
|
8
|
+
/** A Git object proof pins bytes to recorded history. It does not establish
|
|
9
|
+
* that the historical working tree was clean or that these are all dependencies. */
|
|
10
|
+
export const ConnectedGitFileProofSchema = z.object({
|
|
11
|
+
schemaVersion: z.literal("openpond.connectedGitFileProof.v1"),
|
|
12
|
+
path: z.string().min(1).max(1_000).refine(safePath),
|
|
13
|
+
commit: ObjectBytes,
|
|
14
|
+
trees: z.array(ObjectBytes).min(1).max(32),
|
|
15
|
+
contentsBase64: Base64,
|
|
16
|
+
}).strict();
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { ConnectedGitFileProofSchema } from "./git-file-contracts.js";
|
|
3
|
+
export * from "./git-file-contracts.js";
|
|
4
|
+
import { ConnectedRepositoryContextSchema } from "./contracts.js";
|
|
5
|
+
function bytes(base64, limit) {
|
|
6
|
+
const value = Buffer.from(base64, "base64");
|
|
7
|
+
if (value.toString("base64") !== base64 || value.byteLength > limit)
|
|
8
|
+
throw new Error("connected_git_object_bytes_invalid");
|
|
9
|
+
return value;
|
|
10
|
+
}
|
|
11
|
+
function objectHash(kind, value, width) {
|
|
12
|
+
return createHash(width === 40 ? "sha1" : "sha256")
|
|
13
|
+
.update(`${kind} ${value.byteLength}\0`).update(value).digest("hex");
|
|
14
|
+
}
|
|
15
|
+
function treeEntry(tree, name, width) {
|
|
16
|
+
let offset = 0;
|
|
17
|
+
const names = new Set();
|
|
18
|
+
let found = null;
|
|
19
|
+
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
20
|
+
while (offset < tree.byteLength) {
|
|
21
|
+
const space = tree.indexOf(32, offset), nul = tree.indexOf(0, space + 1);
|
|
22
|
+
const end = nul + 1 + width / 2;
|
|
23
|
+
if (space < offset || nul < space || end > tree.byteLength)
|
|
24
|
+
throw new Error("connected_git_tree_invalid");
|
|
25
|
+
const mode = tree.subarray(offset, space).toString("ascii");
|
|
26
|
+
const entryName = decoder.decode(tree.subarray(space + 1, nul));
|
|
27
|
+
if (!entryName || entryName.includes("/") || names.has(entryName))
|
|
28
|
+
throw new Error("connected_git_tree_invalid");
|
|
29
|
+
names.add(entryName);
|
|
30
|
+
if (entryName === name)
|
|
31
|
+
found = { mode, hash: tree.subarray(nul + 1, end).toString("hex") };
|
|
32
|
+
offset = end;
|
|
33
|
+
}
|
|
34
|
+
if (!found)
|
|
35
|
+
throw new Error("connected_git_path_missing");
|
|
36
|
+
return found;
|
|
37
|
+
}
|
|
38
|
+
/** The caller must supply repositoryContext from an authorized, hash-verified
|
|
39
|
+
* connected snapshot. No filesystem access or caller-selected Git command occurs. */
|
|
40
|
+
export function verifyConnectedGitFileProof(repositoryContext, raw) {
|
|
41
|
+
const context = ConnectedRepositoryContextSchema.parse(repositoryContext);
|
|
42
|
+
const proof = ConnectedGitFileProofSchema.parse(raw);
|
|
43
|
+
const width = context.commitHash.length;
|
|
44
|
+
const commit = bytes(proof.commit.base64, 256 * 1024);
|
|
45
|
+
if (proof.commit.hash !== context.commitHash || objectHash("commit", commit, width) !== context.commitHash)
|
|
46
|
+
throw new Error("connected_git_commit_mismatch");
|
|
47
|
+
const firstNewline = commit.indexOf(10);
|
|
48
|
+
if (firstNewline < 0)
|
|
49
|
+
throw new Error("connected_git_commit_invalid");
|
|
50
|
+
const firstLine = commit.subarray(0, firstNewline).toString("ascii");
|
|
51
|
+
if (!new RegExp(`^tree [a-f0-9]{${width}}$`).test(firstLine))
|
|
52
|
+
throw new Error("connected_git_commit_invalid");
|
|
53
|
+
const trees = new Map();
|
|
54
|
+
let totalBytes = commit.byteLength;
|
|
55
|
+
for (const object of proof.trees) {
|
|
56
|
+
const value = bytes(object.base64, 2 * 1024 * 1024);
|
|
57
|
+
totalBytes += value.byteLength;
|
|
58
|
+
if (totalBytes > 8 * 1024 * 1024 || trees.has(object.hash) || object.hash.length !== width
|
|
59
|
+
|| objectHash("tree", value, width) !== object.hash)
|
|
60
|
+
throw new Error("connected_git_tree_mismatch");
|
|
61
|
+
trees.set(object.hash, value);
|
|
62
|
+
}
|
|
63
|
+
const parts = proof.path.split("/");
|
|
64
|
+
const visited = new Set();
|
|
65
|
+
let hash = firstLine.slice(5);
|
|
66
|
+
let mode = "";
|
|
67
|
+
for (const [index, part] of parts.entries()) {
|
|
68
|
+
const tree = trees.get(hash);
|
|
69
|
+
if (!tree || visited.has(hash))
|
|
70
|
+
throw new Error("connected_git_tree_missing");
|
|
71
|
+
visited.add(hash);
|
|
72
|
+
const entry = treeEntry(tree, part, width);
|
|
73
|
+
mode = entry.mode;
|
|
74
|
+
if (index < parts.length - 1 ? mode !== "40000" : !["100644", "100755"].includes(mode))
|
|
75
|
+
throw new Error("connected_git_file_mode_invalid");
|
|
76
|
+
hash = entry.hash;
|
|
77
|
+
}
|
|
78
|
+
if (visited.size !== trees.size)
|
|
79
|
+
throw new Error("connected_git_unreferenced_tree");
|
|
80
|
+
const content = bytes(proof.contentsBase64, 2 * 1024 * 1024);
|
|
81
|
+
if (objectHash("blob", content, width) !== hash)
|
|
82
|
+
throw new Error("connected_git_blob_mismatch");
|
|
83
|
+
return { path: proof.path, commitHash: context.commitHash, gitObjectHash: hash, mode,
|
|
84
|
+
sizeBytes: content.byteLength, sha256: createHash("sha256").update(content).digest("hex"),
|
|
85
|
+
contentsBase64: content.toString("base64") };
|
|
86
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { contentHash } from "@openpond/harness";
|
|
2
|
+
import { ConnectedSessionSchema, resolveConnectedBoundaryMetadata } from "./contracts.js";
|
|
3
|
+
import { recordedCodexCommands, recordedCodexCommandResults, isRecordedReadOnlyInventory } from "./codex-command-observations.js";
|
|
4
|
+
import { verifyConnectedGitFileProof } from "./git-file-proof.js";
|
|
5
|
+
function object(value) {
|
|
6
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
7
|
+
throw new Error("connected_starting_state_unavailable");
|
|
8
|
+
return value;
|
|
9
|
+
}
|
|
10
|
+
function dirtyPaths(output) {
|
|
11
|
+
const paths = [];
|
|
12
|
+
for (const line of output.split("\n")) {
|
|
13
|
+
if (!line)
|
|
14
|
+
continue;
|
|
15
|
+
// Quoted/escaped paths and rename pairs need a distinct binary porcelain
|
|
16
|
+
// protocol. Treat them as unavailable instead of guessing path identities.
|
|
17
|
+
if (!/^[ MADRCU?!]{2} [^\x00-\x1f\x7f]+$/.test(line) || line.slice(3).includes('"')
|
|
18
|
+
|| line.slice(3).includes(" -> ") || line.slice(3).includes("\\"))
|
|
19
|
+
throw new Error("connected_git_status_format_unsupported");
|
|
20
|
+
const path = line.slice(3);
|
|
21
|
+
if (path.startsWith("/") || path.split("/").some(part => part === "." || part === ".."))
|
|
22
|
+
throw new Error("connected_git_status_format_unsupported");
|
|
23
|
+
paths.push(path.replace(/\/$/, ""));
|
|
24
|
+
}
|
|
25
|
+
return paths;
|
|
26
|
+
}
|
|
27
|
+
/** Verify only the original first-tool boundary. Later requests need their own
|
|
28
|
+
* captured state. Both the repository context and observations come from the
|
|
29
|
+
* authorized immutable snapshot, never caller-supplied cleanliness claims. */
|
|
30
|
+
export function verifyConnectedGitStartingAssets(input) {
|
|
31
|
+
const session = ConnectedSessionSchema.parse(input.session);
|
|
32
|
+
const boundary = resolveConnectedBoundaryMetadata(session, input.boundaryId);
|
|
33
|
+
const context = session.repositoryContext;
|
|
34
|
+
if (session.origin !== "codex" || !context?.workingDirectoryHash || !context.recordedAt || boundary.projection !== "turn"
|
|
35
|
+
|| !session.sourceFiles.some(file => file.contentHash === context.sourceFileHash)
|
|
36
|
+
|| !input.proofs.length || input.proofs.length > 100)
|
|
37
|
+
throw new Error("connected_starting_state_unavailable");
|
|
38
|
+
const firstCallIndex = session.events.findIndex(event => event.kind === "tool_call");
|
|
39
|
+
if (firstCallIndex <= boundary.start || firstCallIndex >= boundary.end
|
|
40
|
+
|| session.events.slice(0, firstCallIndex).some(event => event.kind === "compaction"))
|
|
41
|
+
throw new Error("connected_starting_state_boundary_unavailable");
|
|
42
|
+
const callEvent = session.events[firstCallIndex];
|
|
43
|
+
const requestEvent = session.events[boundary.start];
|
|
44
|
+
if (!callEvent.occurredAt || !requestEvent.occurredAt
|
|
45
|
+
|| Date.parse(requestEvent.occurredAt) < Date.parse(context.recordedAt)
|
|
46
|
+
|| Date.parse(callEvent.occurredAt) < Date.parse(requestEvent.occurredAt))
|
|
47
|
+
throw new Error("connected_starting_state_observation_time_unavailable");
|
|
48
|
+
const call = object(callEvent.content);
|
|
49
|
+
if (call.name !== "exec" || typeof call.input !== "string" || !callEvent.callId)
|
|
50
|
+
throw new Error("connected_starting_state_command_unsupported");
|
|
51
|
+
const commands = recordedCodexCommands(call.input);
|
|
52
|
+
const statusIndex = commands.findIndex(command => command.cmd === "git status --short");
|
|
53
|
+
if (statusIndex !== commands.length - 1 || statusIndex < 1
|
|
54
|
+
|| commands.some(command => command.cwd !== null)
|
|
55
|
+
|| !commands.slice(0, statusIndex).every(command => isRecordedReadOnlyInventory(command.cmd)))
|
|
56
|
+
throw new Error("connected_starting_state_command_unsupported");
|
|
57
|
+
const observations = session.events.slice(firstCallIndex + 1, boundary.end);
|
|
58
|
+
const resultIndex = observations.findIndex(event => event.kind === "tool_result" && event.callId === callEvent.callId);
|
|
59
|
+
if (resultIndex < 0 || observations.slice(0, resultIndex).some(event => ["tool_call", "tool_result", "compaction", "message"].includes(event.kind))
|
|
60
|
+
|| observations.filter(event => event.kind === "tool_result" && event.callId === callEvent.callId).length !== 1)
|
|
61
|
+
throw new Error("connected_starting_state_observation_unavailable");
|
|
62
|
+
const resultEvent = observations[resultIndex];
|
|
63
|
+
const results = recordedCodexCommandResults(resultEvent.content, commands.length);
|
|
64
|
+
if (results.some((result, index) => result.originalTokenCount >= commands[index].maxOutputTokens
|
|
65
|
+
|| /output truncated|truncated output|\[\.\.\./i.test(result.output)))
|
|
66
|
+
throw new Error("connected_starting_state_observation_truncated");
|
|
67
|
+
if (contentHash(results[0].output.split("\n")[0]) !== context.workingDirectoryHash)
|
|
68
|
+
throw new Error("connected_starting_state_directory_mismatch");
|
|
69
|
+
const dirty = dirtyPaths(results[statusIndex].output);
|
|
70
|
+
const assets = input.proofs.map(proof => verifyConnectedGitFileProof(context, proof));
|
|
71
|
+
if (new Set(assets.map(asset => asset.path)).size !== assets.length)
|
|
72
|
+
throw new Error("connected_starting_state_duplicate_path");
|
|
73
|
+
for (const asset of assets)
|
|
74
|
+
if (dirty.some(path => asset.path === path || asset.path.startsWith(path + "/")))
|
|
75
|
+
throw new Error("connected_starting_state_selected_file_modified");
|
|
76
|
+
return { schemaVersion: "openpond.connectedGitStartingAssets.v1",
|
|
77
|
+
sessionHash: session.contentHash, boundaryId: boundary.id, inputHash: boundary.inputHash,
|
|
78
|
+
revisionHash: boundary.revisionHash, commitHash: context.commitHash,
|
|
79
|
+
observationHash: contentHash({ call: callEvent, result: resultEvent }),
|
|
80
|
+
selectedPathsUnmodified: true, dependencyClosureQualified: false, assets };
|
|
81
|
+
}
|
|
@@ -10,7 +10,7 @@ import { parseClaudeSession } from "./sources/claude-code.js";
|
|
|
10
10
|
import { parseHermesEvidence, parseOpenClawEvidence, } from "./sources/history.js";
|
|
11
11
|
/** Cache identity includes source projection policy, independent of file mtimes. */
|
|
12
12
|
export function agentImportNormalizerRevision(source) {
|
|
13
|
-
return source === "codex" ? `${CONNECTED_NORMALIZER_VERSION}/${CODEX_AUTHORIZATION_PROJECTION_POLICY}` : CONNECTED_NORMALIZER_VERSION;
|
|
13
|
+
return source === "codex" ? `${CONNECTED_NORMALIZER_VERSION}/${CODEX_AUTHORIZATION_PROJECTION_POLICY}/fork-repository-starting-state-v1` : CONNECTED_NORMALIZER_VERSION;
|
|
14
14
|
}
|
|
15
15
|
/** Never admits malformed files partly or executes referenced paths/commands. */
|
|
16
16
|
export function previewAgentImport(input) {
|
|
@@ -49,6 +49,7 @@ export function previewAgentImport(input) {
|
|
|
49
49
|
warnings: session.warnings,
|
|
50
50
|
contextComplete: !session.boundaries.some((boundary) => boundary.coverage.context === "unknown"),
|
|
51
51
|
acquisition: input.acquisition,
|
|
52
|
+
...(session.repositoryContext ? { repositoryContext: session.repositoryContext } : {}),
|
|
52
53
|
})
|
|
53
54
|
: session));
|
|
54
55
|
}
|
|
@@ -67,6 +67,7 @@ export function normalizeConnectedSession(input) {
|
|
|
67
67
|
origin: input.origin, sessionId: input.sessionId, branchId: input.branchId ?? null, parentSessionId: input.parentSessionId ?? null,
|
|
68
68
|
exporterVersion: input.exporterVersion ?? null, sourceFiles: input.files.map(connectedFileIdentity).sort((a, b) => a.path.localeCompare(b.path)),
|
|
69
69
|
...(input.acquisition ? { acquisition: input.acquisition } : {}),
|
|
70
|
+
...(input.repositoryContext ? { repositoryContext: input.repositoryContext } : {}),
|
|
70
71
|
events, unmappedEvents: (input.unmappedEvents ?? []).map((event, sequence) => ({ ...event, sequence })), boundaries, warnings: input.warnings ?? [] };
|
|
71
72
|
const session = ConnectedSessionSchema.parse({ ...body, contentHash: contentHash(body) });
|
|
72
73
|
if (new TextEncoder().encode(JSON.stringify(session)).length > CONNECTED_EVIDENCE_LIMITS.decodedBytes)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/** Forked Codex rollouts retain their own header followed by ancestor headers.
|
|
2
|
+
* Only the declared, contiguous ancestry is metadata for that source. */
|
|
3
|
+
export class CodexMetadataChain {
|
|
4
|
+
metadata = null;
|
|
5
|
+
parentId = null;
|
|
6
|
+
closed = false;
|
|
7
|
+
ids = new Set();
|
|
8
|
+
accept(row) {
|
|
9
|
+
if (row.type !== "session_meta") {
|
|
10
|
+
this.closed = true;
|
|
11
|
+
return true;
|
|
12
|
+
}
|
|
13
|
+
const payload = row.payload;
|
|
14
|
+
if (this.closed || !payload || typeof payload !== "object" || Array.isArray(payload))
|
|
15
|
+
return false;
|
|
16
|
+
const value = payload;
|
|
17
|
+
if (typeof value.id !== "string" || !value.id || value.id.length > 500
|
|
18
|
+
|| this.ids.has(value.id) || this.ids.size >= 16)
|
|
19
|
+
return false;
|
|
20
|
+
if (this.metadata) {
|
|
21
|
+
if (value.id !== this.parentId)
|
|
22
|
+
return false;
|
|
23
|
+
for (const key of ["creator_user_id", "creator_account_id"])
|
|
24
|
+
if (this.metadata[key] != null && value[key] != null && this.metadata[key] !== value[key])
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
else
|
|
28
|
+
this.metadata = value;
|
|
29
|
+
this.ids.add(value.id);
|
|
30
|
+
this.parentId = typeof value.forked_from_id === "string" ? value.forked_from_id : null;
|
|
31
|
+
return true;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
@@ -1,11 +1,24 @@
|
|
|
1
1
|
import { contentHash } from "@openpond/harness";
|
|
2
|
-
import { connectedTimestamp, event, json, normalizeConnectedSession, object } from "../normalize.js";
|
|
2
|
+
import { connectedFileIdentity, connectedTimestamp, event, json, normalizeConnectedSession, object } from "../normalize.js";
|
|
3
3
|
import { projectCodexAuthorization } from "./codex-authorization.js";
|
|
4
4
|
import { sourceUsage } from "./usage.js";
|
|
5
|
+
import { CodexMetadataChain } from "./codex-metadata.js";
|
|
5
6
|
export function parseCodexSession(file, rows) {
|
|
6
|
-
const
|
|
7
|
+
const chain = new CodexMetadataChain();
|
|
8
|
+
if (rows.some(row => !chain.accept(row)))
|
|
9
|
+
throw new Error("Codex rollout metadata does not match its declared fork ancestry.");
|
|
10
|
+
const metadata = chain.metadata ?? {};
|
|
7
11
|
if (typeof metadata.id !== "string")
|
|
8
12
|
throw new Error("This is not a supported Codex rollout: session_meta.id is missing.");
|
|
13
|
+
const git = object(metadata.git);
|
|
14
|
+
const repositoryContext = typeof git.commit_hash === "string" && /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(git.commit_hash)
|
|
15
|
+
? { schemaVersion: "openpond.connectedRepositoryContext.v1", kind: "git",
|
|
16
|
+
commitHash: git.commit_hash, branch: typeof git.branch === "string" && git.branch.length > 0 && git.branch.length <= 500 ? git.branch : null,
|
|
17
|
+
recordedAt: connectedTimestamp(metadata.timestamp), sourceFileHash: connectedFileIdentity(file).contentHash,
|
|
18
|
+
metadataHash: contentHash(git) }
|
|
19
|
+
: undefined;
|
|
20
|
+
if (repositoryContext && typeof metadata.cwd === "string")
|
|
21
|
+
Object.assign(repositoryContext, { workingDirectoryHash: contentHash(metadata.cwd) });
|
|
9
22
|
const events = [];
|
|
10
23
|
let turnId = null;
|
|
11
24
|
let authorizationOmissions = 0;
|
|
@@ -52,5 +65,6 @@ export function parseCodexSession(file, rows) {
|
|
|
52
65
|
}
|
|
53
66
|
return normalizeConnectedSession({ origin: "codex", sessionId: metadata.id, parentSessionId: typeof metadata.forked_from_id === "string" ? metadata.forked_from_id : null,
|
|
54
67
|
exporterVersion: typeof metadata.cli_version === "string" ? metadata.cli_version : null, files: [file], events,
|
|
68
|
+
...(repositoryContext ? { repositoryContext } : {}),
|
|
55
69
|
warnings: ["Codex cumulative usage is retained as reported totals and is not summed as billed invocations.", "Referenced files are unavailable unless separately admitted.", ...(authorizationOmissions ? [`Codex native authorization state excluded by codex-native-authorization-v1 from ${authorizationOmissions} locations. Omission markers retain source-section hashes; sourceFiles retains original byte hashes. Other conversation evidence is unchanged and still requires privacy admission.`] : [])] });
|
|
56
70
|
}
|
|
@@ -83,7 +83,7 @@ export const VerifierSetReleaseContentSchema = z.object({
|
|
|
83
83
|
schemaVersion: z.literal("openpond.verifierSetRelease.v1"),
|
|
84
84
|
id: ReleaseIdSchema,
|
|
85
85
|
revision: z.number().int().positive(),
|
|
86
|
-
graders: z.array(GraderSpecSchema).
|
|
86
|
+
graders: z.array(GraderSpecSchema).max(1_000),
|
|
87
87
|
isolation: z.object({
|
|
88
88
|
processBoundary: z.enum(["same_process", "isolated_process", "container"]),
|
|
89
89
|
networkPolicy: z.literal("none"),
|
package/dist/graders.js
CHANGED
|
@@ -39,9 +39,15 @@ export const GraderEvidenceContentSchema = z.object({
|
|
|
39
39
|
modelJudgeReceipt: ModelJudgeReceiptSchema.optional(),
|
|
40
40
|
}).strict();
|
|
41
41
|
export const GraderEvidenceSchema = GraderEvidenceContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
|
|
42
|
+
/** Scope is part of the immutable grader. An absent scope applies to every
|
|
43
|
+
* task; an explicit scope is never inferred from answer content or metadata. */
|
|
44
|
+
export function gradersForTask(graders, task) {
|
|
45
|
+
return graders.filter(grader => !grader.taskIds || grader.taskIds.includes(task.id));
|
|
46
|
+
}
|
|
42
47
|
export async function gradeEvidence(input) {
|
|
48
|
+
const graders = gradersForTask(input.graders, input.task);
|
|
43
49
|
if (input.evidence.infrastructureError) {
|
|
44
|
-
return
|
|
50
|
+
return graders.map((grader) => evidence(grader, {
|
|
45
51
|
score: null,
|
|
46
52
|
passed: false,
|
|
47
53
|
rewardEligible: false,
|
|
@@ -51,7 +57,7 @@ export async function gradeEvidence(input) {
|
|
|
51
57
|
privilegedEvidenceRefs: [],
|
|
52
58
|
}));
|
|
53
59
|
}
|
|
54
|
-
return Promise.all(
|
|
60
|
+
return Promise.all(graders.map(async (grader) => {
|
|
55
61
|
if (grader.kind === "model_judge") {
|
|
56
62
|
if (!input.modelJudge || (grader.calibrationStatus !== "passed" && input.purpose !== "fixture_calibration"))
|
|
57
63
|
return evidence(grader, unavailable("Model judge is unavailable or uncalibrated."));
|
|
@@ -148,7 +154,7 @@ export async function gradeTaskEvidence(input) {
|
|
|
148
154
|
const components = await gradeEvidence(input);
|
|
149
155
|
const content = { schemaVersion: "openpond.taskGrade.v1",
|
|
150
156
|
taskHash: contentHash(input.task), evidenceHash: contentHash(input.evidence), graderSetHash: contentHash(input.graders), components,
|
|
151
|
-
...aggregateGraderScores({ graders: input.graders, components, infrastructureError: input.evidence.infrastructureError }) };
|
|
157
|
+
...aggregateGraderScores({ graders: gradersForTask(input.graders, input.task), components, infrastructureError: input.evidence.infrastructureError }) };
|
|
152
158
|
return TaskGradeSchema.parse({ ...content, contentHash: contentHash(content) });
|
|
153
159
|
}
|
|
154
160
|
/** Verify retained results against the exact admitted inputs without rerunning
|
|
@@ -163,7 +169,7 @@ export function verifyTaskGrade(value, input) {
|
|
|
163
169
|
if (componentHash !== contentHash(body))
|
|
164
170
|
throw new Error("Grader evidence integrity failed.");
|
|
165
171
|
}
|
|
166
|
-
const summary = aggregateGraderScores({ graders: input.graders, components: result.components, infrastructureError: input.evidence.infrastructureError });
|
|
172
|
+
const summary = aggregateGraderScores({ graders: gradersForTask(input.graders, input.task), components: result.components, infrastructureError: input.evidence.infrastructureError });
|
|
167
173
|
if (contentHash(GradeSummarySchema.parse({ score: result.score, passed: result.passed, rewardEligible: result.rewardEligible, failureClass: result.failureClass, gradingStatus: result.gradingStatus })) !== contentHash(summary))
|
|
168
174
|
throw new Error("Task grade summary differs from its grader evidence.");
|
|
169
175
|
return result;
|