@springbrand/eval-harness 0.0.0-stage → 0.1.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -2
- package/dist/index.d.ts +180 -0
- package/dist/index.js +196 -0
- package/package.json +31 -3
package/README.md
CHANGED
|
@@ -1,3 +1,77 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @springbrand/eval-harness
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
SP's service contract for Eval-managed ordinary Chats. This package owns only
|
|
4
|
+
wire validation, stable request facts and service route names. Inbox owns Chat
|
|
5
|
+
creation and routing, Severn owns Submission and canonical Session history,
|
|
6
|
+
Workspace owns files, and API owns billing settlements. Eval Project, Dataset,
|
|
7
|
+
Sampling and scoring remain in `eval-platform`.
|
|
8
|
+
|
|
9
|
+
The Eval Run Worker authenticates with a dedicated service token. Its requests
|
|
10
|
+
include the owning user ID because an SP Inbox is addressed by user. The Chat
|
|
11
|
+
creation facts and each Submission operation key are immutable. Initialization
|
|
12
|
+
receives Eval-provided bundle/resource IDs and their content: material files go
|
|
13
|
+
to the target Chat Workspace; canonical JSON or JSONL context may be imported into
|
|
14
|
+
the Chat Session before the first Submission and need not be copied to Workspace.
|
|
15
|
+
|
|
16
|
+
The service interface is implemented under `apps/agent/src/eval/`. The Agent
|
|
17
|
+
Worker requires the separately provisioned `EVAL_SP_SERVICE_TOKEN` secret and
|
|
18
|
+
the Run Worker sends it as `X-Eval-Service-Token`. No browser Origin is accepted.
|
|
19
|
+
The Eval Worker uses the independently deployed Eval SP origin. Its user ID is
|
|
20
|
+
part of the Agent Worker path because an SP Inbox is addressed by user.
|
|
21
|
+
That Agent Worker's `API_ORIGIN` and SpaceDO binding must point to the same
|
|
22
|
+
independent Eval SP deployment; the shared SP dev API and database are not used.
|
|
23
|
+
|
|
24
|
+
| Operation | Agent Worker route |
|
|
25
|
+
| --- | --- |
|
|
26
|
+
| Ensure the ordinary Chat | `PUT /internal/v1/eval-harness/users/{userId}/chats/{chatId}` |
|
|
27
|
+
| Initialize one Data sample's optional Session and materials | `POST /internal/v1/eval-harness/users/{userId}/chats/{chatId}/initialization` |
|
|
28
|
+
| Submit one Prompt | `POST /internal/v1/eval-harness/users/{userId}/chats/{chatId}/submissions` |
|
|
29
|
+
| Recover a Submission by operation key | `GET /internal/v1/eval-harness/users/{userId}/chats/{chatId}/submissions/by-key/{key}` |
|
|
30
|
+
| Stop every unfinished Submission in this Eval Chat | `POST /internal/v1/eval-harness/users/{userId}/chats/{chatId}/submissions/stop` |
|
|
31
|
+
| Observe a Submission | `WS /internal/v1/eval-harness/users/{userId}/chats/{chatId}/submissions/{submissionId}/events` |
|
|
32
|
+
| Collect the complete artifact list and trajectory reference | `GET /internal/v1/eval-harness/users/{userId}/chats/{chatId}/output` |
|
|
33
|
+
| Collect raw resource facts | `GET /internal/v1/eval-harness/users/{userId}/chats/{chatId}/resources/by-key/{key}` |
|
|
34
|
+
| Export: list produced artifacts with size and media type | `GET /internal/v1/eval-harness/users/{userId}/chats/{chatId}/evidence/artifacts` |
|
|
35
|
+
| Export: read one produced file's bytes | `GET /internal/v1/eval-harness/users/{userId}/chats/{chatId}/evidence/files?path=` |
|
|
36
|
+
| Export: page the complete trajectory, oldest first | `GET /internal/v1/eval-harness/users/{userId}/chats/{chatId}/evidence/trajectory?afterSeq=&limit=` |
|
|
37
|
+
| Import: write one evidence file into an Evaluator's own Space | `PUT /internal/v1/eval-harness/users/{userId}/chats/{chatId}/evidence/files?path=` |
|
|
38
|
+
|
|
39
|
+
The initialization body includes the Data sample and, only when referenced by
|
|
40
|
+
that sample, `initialSession: {bundleId, sessionId, context?: {format, content}}`
|
|
41
|
+
and/or `initialMaterials: {bundleId, materialId, files}`. `content` is canonical
|
|
42
|
+
Session JSON or JSONL; each material file supplies its path, media type and
|
|
43
|
+
base64 bytes. Supplied Session context is imported into the Chat before the Prompt and only
|
|
44
|
+
material files are written to SpaceDO. A sample may specify either source, both,
|
|
45
|
+
or neither. There is no Dataset-level source binding.
|
|
46
|
+
|
|
47
|
+
An Evaluator is an ordinary Eval Chat with its own Session, Submission and
|
|
48
|
+
Space; its creation facts add only `evaluationId`. It never reads or writes the
|
|
49
|
+
Subject's Space. Eval Platform exports a finished Subject's evidence through the
|
|
50
|
+
read-only export routes and freezes it as a content-addressed bundle, then imports
|
|
51
|
+
that bundle into the Evaluator's own Space under `/evidence` before the first
|
|
52
|
+
Submission. Export routes answer only after the Chat's last Submission ended and
|
|
53
|
+
its whole trajectory was delivered (409 before that). The artifact list covers
|
|
54
|
+
files the Agent produced in its own Space, excluding `/reference`,
|
|
55
|
+
`/initial-materials`, `/userspace`, `/evidence` and uploads; a single file crosses
|
|
56
|
+
one Durable Object RPC, so files over 25 MB are refused with 413. Trajectory pages
|
|
57
|
+
come straight from `trajectory_events` with a forward `afterSeq` cursor
|
|
58
|
+
(`nextAfterSeq` is null after the last page). An import `PUT` carries the raw
|
|
59
|
+
bytes, their `Content-Type` and `X-Eval-Content-Sha256: <hex>`; SP verifies the
|
|
60
|
+
hash, writes `/evidence/<path>` and returns `{path, size, sha256}`. Only an
|
|
61
|
+
Evaluator imports; an identical repeat is a no-op, different content is a 409,
|
|
62
|
+
and imports stop with the Chat's stop latch.
|
|
63
|
+
|
|
64
|
+
Eval Platform writes the Evaluator instructions into the Prompt; SP submits it
|
|
65
|
+
unchanged. The Evaluator's `output` adds `structuredOutput: {path, content,
|
|
66
|
+
contentHash}` with the raw UTF-8 text of `/output/result.json` in its own Space
|
|
67
|
+
(at most 1 MB, `contentHash` is `sha256:<hex>`); it is omitted when the Evaluator
|
|
68
|
+
wrote no result file.
|
|
69
|
+
The Chat observer WebSocket carries status only. Browser Chat input and writes
|
|
70
|
+
are rejected. SP API provides its account, Agent Config, and billing facts.
|
|
71
|
+
The service-only stop route delegates to the ordinary Chat Submission lifecycle:
|
|
72
|
+
it persists an Eval Chat stop latch, waits for already accepted submit calls,
|
|
73
|
+
interrupts active work, and returns after terminal status is recorded. New
|
|
74
|
+
initialization and Submission requests are rejected after the latch is saved,
|
|
75
|
+
including after a DO restart.
|
|
76
|
+
It refuses a Chat without an Eval binding. Repeating stop after all Submissions
|
|
77
|
+
finish returns `{ "ok": false }` without changing history.
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export declare const EVAL_SERVICE_HEADER = "X-Eval-Service-Token";
|
|
3
|
+
export declare const EVAL_HARNESS_PATH = "/internal/v1/eval-harness";
|
|
4
|
+
export declare const evalEnsureChatSchema: z.ZodObject<{
|
|
5
|
+
chatId: z.ZodUUID;
|
|
6
|
+
role: z.ZodEnum<{
|
|
7
|
+
evaluator: "evaluator";
|
|
8
|
+
subject: "subject";
|
|
9
|
+
}>;
|
|
10
|
+
framework: z.ZodLiteral<"sp">;
|
|
11
|
+
samplingId: z.ZodString;
|
|
12
|
+
projectId: z.ZodString;
|
|
13
|
+
agentConfig: z.ZodObject<{
|
|
14
|
+
ownerUserId: z.ZodUUID;
|
|
15
|
+
userAgentId: z.ZodUUID;
|
|
16
|
+
}, z.core.$strict>;
|
|
17
|
+
evaluationId: z.ZodOptional<z.ZodString>;
|
|
18
|
+
runtime: z.ZodOptional<z.ZodObject<{
|
|
19
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
20
|
+
}, z.core.$strict>>;
|
|
21
|
+
}, z.core.$strict>;
|
|
22
|
+
export declare const evalMaterialFileSchema: z.ZodObject<{
|
|
23
|
+
path: z.ZodString;
|
|
24
|
+
mediaType: z.ZodString;
|
|
25
|
+
contentBase64: z.ZodBase64;
|
|
26
|
+
}, z.core.$strict>;
|
|
27
|
+
export declare const evalInitialContextSchema: z.ZodObject<{
|
|
28
|
+
format: z.ZodEnum<{
|
|
29
|
+
json: "json";
|
|
30
|
+
jsonl: "jsonl";
|
|
31
|
+
}>;
|
|
32
|
+
content: z.ZodString;
|
|
33
|
+
}, z.core.$strict>;
|
|
34
|
+
export declare const evalInitializationSchema: z.ZodObject<{
|
|
35
|
+
idempotencyKey: z.ZodString;
|
|
36
|
+
data: z.ZodIntersection<z.ZodRecord<z.ZodString, z.ZodUnknown>, z.ZodObject<{
|
|
37
|
+
userPrompt: z.ZodString;
|
|
38
|
+
}, z.core.$strip>>;
|
|
39
|
+
initialSession: z.ZodOptional<z.ZodObject<{
|
|
40
|
+
bundleId: z.ZodString;
|
|
41
|
+
sessionId: z.ZodString;
|
|
42
|
+
context: z.ZodOptional<z.ZodObject<{
|
|
43
|
+
format: z.ZodEnum<{
|
|
44
|
+
json: "json";
|
|
45
|
+
jsonl: "jsonl";
|
|
46
|
+
}>;
|
|
47
|
+
content: z.ZodString;
|
|
48
|
+
}, z.core.$strict>>;
|
|
49
|
+
}, z.core.$strict>>;
|
|
50
|
+
initialMaterials: z.ZodOptional<z.ZodObject<{
|
|
51
|
+
bundleId: z.ZodString;
|
|
52
|
+
materialId: z.ZodString;
|
|
53
|
+
files: z.ZodArray<z.ZodObject<{
|
|
54
|
+
path: z.ZodString;
|
|
55
|
+
mediaType: z.ZodString;
|
|
56
|
+
contentBase64: z.ZodBase64;
|
|
57
|
+
}, z.core.$strict>>;
|
|
58
|
+
}, z.core.$strict>>;
|
|
59
|
+
}, z.core.$strict>;
|
|
60
|
+
export declare const evalSubmitSchema: z.ZodObject<{
|
|
61
|
+
idempotencyKey: z.ZodString;
|
|
62
|
+
prompt: z.ZodString;
|
|
63
|
+
unattended: z.ZodLiteral<true>;
|
|
64
|
+
runtime: z.ZodOptional<z.ZodObject<{
|
|
65
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
66
|
+
}, z.core.$strict>>;
|
|
67
|
+
}, z.core.$strict>;
|
|
68
|
+
export type EvalEnsureChat = z.output<typeof evalEnsureChatSchema>;
|
|
69
|
+
export type EvalInitialization = z.output<typeof evalInitializationSchema>;
|
|
70
|
+
export type EvalSubmit = z.output<typeof evalSubmitSchema>;
|
|
71
|
+
export type EvalSubmissionReceipt = {
|
|
72
|
+
readonly submissionId: string;
|
|
73
|
+
readonly status: string;
|
|
74
|
+
readonly normalizedStatus: "queued" | "running" | "completed" | "failed" | "interrupted" | "cancelled";
|
|
75
|
+
readonly accepted: boolean;
|
|
76
|
+
readonly error?: string;
|
|
77
|
+
readonly retryable?: boolean;
|
|
78
|
+
readonly createdAt: number;
|
|
79
|
+
readonly completedAt?: number;
|
|
80
|
+
};
|
|
81
|
+
export declare const evalBillingResourcesSchema: z.ZodObject<{
|
|
82
|
+
usageEvents: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
83
|
+
billingSettlements: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
84
|
+
}, z.core.$strict>;
|
|
85
|
+
export type EvalBillingResources = z.output<typeof evalBillingResourcesSchema>;
|
|
86
|
+
export declare function projectEvalSubmission(input: {
|
|
87
|
+
readonly submissionId: string;
|
|
88
|
+
readonly status: string;
|
|
89
|
+
readonly accepted: boolean;
|
|
90
|
+
readonly error?: string;
|
|
91
|
+
readonly retryable?: boolean;
|
|
92
|
+
readonly createdAt: number;
|
|
93
|
+
readonly completedAt?: number;
|
|
94
|
+
}): EvalSubmissionReceipt;
|
|
95
|
+
export type EvalCanonicalContext = {
|
|
96
|
+
readonly entries: readonly {
|
|
97
|
+
readonly id: string;
|
|
98
|
+
readonly message: Readonly<Record<string, unknown>>;
|
|
99
|
+
readonly createdAt: number;
|
|
100
|
+
}[];
|
|
101
|
+
};
|
|
102
|
+
/** JSON stores one canonical snapshot; JSONL stores one canonical entry per line. */
|
|
103
|
+
export declare function readEvalCanonicalContext(input: z.output<typeof evalInitialContextSchema>): EvalCanonicalContext;
|
|
104
|
+
/** The one file an Evaluator writes its scored result to, in its own Space. */
|
|
105
|
+
export declare const EVAL_RESULT_FILE = "result.json";
|
|
106
|
+
export declare const EVAL_RESULT_PATH = "/output/result.json";
|
|
107
|
+
export declare const EVAL_RESULT_MAX_BYTES = 1000000;
|
|
108
|
+
/** The Evaluator result file exactly as written; Eval Platform parses and validates it. */
|
|
109
|
+
export type EvalStructuredOutput = {
|
|
110
|
+
readonly path: string;
|
|
111
|
+
readonly content: string;
|
|
112
|
+
readonly contentHash: string;
|
|
113
|
+
};
|
|
114
|
+
/** Reads the result file as raw UTF-8 text; any byte order mark and formatting are kept. */
|
|
115
|
+
export declare function readEvalStructuredOutput(path: string, bytes: Uint8Array): Promise<EvalStructuredOutput>;
|
|
116
|
+
/** The `output` route body; an Evaluator's `structuredOutput` is omitted when it wrote no result file. */
|
|
117
|
+
export type EvalEvidence = {
|
|
118
|
+
readonly status: "succeeded" | "failed";
|
|
119
|
+
readonly samplingId: string;
|
|
120
|
+
readonly evaluationId?: string;
|
|
121
|
+
readonly artifacts: readonly {
|
|
122
|
+
readonly artifactId: string;
|
|
123
|
+
readonly kind: string;
|
|
124
|
+
readonly path: string;
|
|
125
|
+
}[];
|
|
126
|
+
readonly trajectory: {
|
|
127
|
+
readonly trajectoryId: string;
|
|
128
|
+
readonly source: "subject" | "evaluator";
|
|
129
|
+
readonly path: string;
|
|
130
|
+
};
|
|
131
|
+
readonly structuredOutput?: EvalStructuredOutput;
|
|
132
|
+
readonly error?: string;
|
|
133
|
+
};
|
|
134
|
+
export declare function evalMaterialRoot(bundleId: string, materialId: string): string;
|
|
135
|
+
export declare function evalMaterialPath(root: string, path: string): string;
|
|
136
|
+
/** Where imported evidence lives in the Evaluator's own Space. */
|
|
137
|
+
export declare const EVAL_EVIDENCE_ROOT = "/evidence";
|
|
138
|
+
/** One exported or imported file crosses a single Durable Object RPC as bytes. */
|
|
139
|
+
export declare const EVAL_EVIDENCE_FILE_MAX_BYTES = 25000000;
|
|
140
|
+
export declare const EVAL_EVIDENCE_TRAJECTORY_PAGE_MAX = 200;
|
|
141
|
+
export declare const EVAL_EVIDENCE_SHA256_HEADER = "X-Eval-Content-Sha256";
|
|
142
|
+
/** One Subject artifact; its bytes are read separately so the inventory stays small. */
|
|
143
|
+
export declare const evalEvidenceArtifactSchema: z.ZodObject<{
|
|
144
|
+
path: z.ZodString;
|
|
145
|
+
size: z.ZodNumber;
|
|
146
|
+
mediaType: z.ZodString;
|
|
147
|
+
}, z.core.$strict>;
|
|
148
|
+
export type EvalEvidenceArtifact = z.output<typeof evalEvidenceArtifactSchema>;
|
|
149
|
+
export declare const evalEvidenceArtifactsSchema: z.ZodObject<{
|
|
150
|
+
artifacts: z.ZodArray<z.ZodObject<{
|
|
151
|
+
path: z.ZodString;
|
|
152
|
+
size: z.ZodNumber;
|
|
153
|
+
mediaType: z.ZodString;
|
|
154
|
+
}, z.core.$strict>>;
|
|
155
|
+
}, z.core.$strict>;
|
|
156
|
+
export type EvalEvidenceArtifacts = z.output<typeof evalEvidenceArtifactsSchema>;
|
|
157
|
+
/** Forward pages of the Chat's complete `trajectory_events`, oldest first. */
|
|
158
|
+
export type EvalEvidenceTrajectoryPage = {
|
|
159
|
+
readonly events: readonly {
|
|
160
|
+
readonly ingestSeq: number;
|
|
161
|
+
readonly event: object;
|
|
162
|
+
}[];
|
|
163
|
+
/** Pass as `afterSeq` for the next page; null once the last event was returned. */
|
|
164
|
+
readonly nextAfterSeq: number | null;
|
|
165
|
+
};
|
|
166
|
+
export declare function readEvalTrajectoryPageQuery(params: URLSearchParams): {
|
|
167
|
+
afterSeq: number;
|
|
168
|
+
limit: number;
|
|
169
|
+
};
|
|
170
|
+
/** A Space path for one evidence file; `relative` keeps the bundle's own layout below `/evidence`. */
|
|
171
|
+
export declare function evalEvidencePath(relative: string): string;
|
|
172
|
+
/** The receipt for one imported file; identical repeats return the same receipt. */
|
|
173
|
+
export type EvalEvidenceFileReceipt = {
|
|
174
|
+
readonly path: string;
|
|
175
|
+
readonly size: number;
|
|
176
|
+
readonly sha256: string;
|
|
177
|
+
};
|
|
178
|
+
export declare function isEvalSha256(value: string | null): value is string;
|
|
179
|
+
/** Compare creation and operation facts without depending on JSON property order. */
|
|
180
|
+
export declare function stableEvalFacts(value: unknown): string;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export const EVAL_SERVICE_HEADER = "X-Eval-Service-Token";
|
|
3
|
+
export const EVAL_HARNESS_PATH = "/internal/v1/eval-harness";
|
|
4
|
+
const id = z.string().trim().min(1);
|
|
5
|
+
const uuid = z.uuid();
|
|
6
|
+
export const evalEnsureChatSchema = z.strictObject({
|
|
7
|
+
chatId: uuid,
|
|
8
|
+
role: z.enum(["subject", "evaluator"]),
|
|
9
|
+
framework: z.literal("sp"),
|
|
10
|
+
samplingId: id,
|
|
11
|
+
projectId: id,
|
|
12
|
+
agentConfig: z.strictObject({ ownerUserId: uuid, userAgentId: uuid }),
|
|
13
|
+
/** Only an Evaluator carries it; the Evaluator still owns an ordinary Chat and its own Space. */
|
|
14
|
+
evaluationId: id.optional(),
|
|
15
|
+
runtime: z.strictObject({ timeoutMs: z.number().finite().positive().optional() }).optional(),
|
|
16
|
+
}).superRefine((value, context) => {
|
|
17
|
+
if ((value.role === "evaluator") !== (value.evaluationId !== undefined)) {
|
|
18
|
+
context.addIssue({ code: "custom", path: ["evaluationId"], message: "Evaluation ID must match role" });
|
|
19
|
+
}
|
|
20
|
+
});
|
|
21
|
+
export const evalMaterialFileSchema = z.strictObject({
|
|
22
|
+
path: z.string().startsWith("/").min(2),
|
|
23
|
+
mediaType: id,
|
|
24
|
+
contentBase64: z.base64(),
|
|
25
|
+
});
|
|
26
|
+
export const evalInitialContextSchema = z.strictObject({
|
|
27
|
+
format: z.enum(["json", "jsonl"]),
|
|
28
|
+
content: z.string().min(1),
|
|
29
|
+
});
|
|
30
|
+
export const evalInitializationSchema = z.strictObject({
|
|
31
|
+
idempotencyKey: id,
|
|
32
|
+
data: z.record(z.string(), z.unknown()).and(z.object({ userPrompt: z.string().trim().min(1) })),
|
|
33
|
+
initialSession: z.strictObject({
|
|
34
|
+
bundleId: id,
|
|
35
|
+
sessionId: id,
|
|
36
|
+
context: evalInitialContextSchema.optional(),
|
|
37
|
+
}).optional(),
|
|
38
|
+
initialMaterials: z.strictObject({
|
|
39
|
+
bundleId: id,
|
|
40
|
+
materialId: id,
|
|
41
|
+
files: z.array(evalMaterialFileSchema).max(500),
|
|
42
|
+
}).optional(),
|
|
43
|
+
}).superRefine((value, context) => {
|
|
44
|
+
const session = value.data.initialSession;
|
|
45
|
+
const materials = value.data.initialMaterials;
|
|
46
|
+
if (session ? !value.initialSession || session.bundleId !== value.initialSession.bundleId ||
|
|
47
|
+
session.sessionId !== value.initialSession.sessionId : value.initialSession !== undefined) {
|
|
48
|
+
context.addIssue({ code: "custom", path: ["initialSession"], message: "Session content must match the Data reference" });
|
|
49
|
+
}
|
|
50
|
+
if (materials ? !value.initialMaterials || materials.bundleId !== value.initialMaterials.bundleId ||
|
|
51
|
+
materials.materialId !== value.initialMaterials.materialId : value.initialMaterials !== undefined) {
|
|
52
|
+
context.addIssue({ code: "custom", path: ["initialMaterials"], message: "Material content must match the Data reference" });
|
|
53
|
+
}
|
|
54
|
+
});
|
|
55
|
+
export const evalSubmitSchema = z.strictObject({
|
|
56
|
+
idempotencyKey: id,
|
|
57
|
+
prompt: z.string().trim().min(1),
|
|
58
|
+
unattended: z.literal(true),
|
|
59
|
+
runtime: z.strictObject({ timeoutMs: z.number().finite().positive().optional() }).optional(),
|
|
60
|
+
});
|
|
61
|
+
export const evalBillingResourcesSchema = z.strictObject({
|
|
62
|
+
usageEvents: z.array(z.record(z.string(), z.unknown())),
|
|
63
|
+
billingSettlements: z.array(z.record(z.string(), z.unknown())),
|
|
64
|
+
});
|
|
65
|
+
export function projectEvalSubmission(input) {
|
|
66
|
+
const normalizedStatus = {
|
|
67
|
+
pending: "queued",
|
|
68
|
+
running: "running",
|
|
69
|
+
completed: "completed",
|
|
70
|
+
error: "failed",
|
|
71
|
+
skipped: "failed",
|
|
72
|
+
aborted: "interrupted",
|
|
73
|
+
}[input.status];
|
|
74
|
+
if (!normalizedStatus)
|
|
75
|
+
throw new Error("Unknown SP Submission status");
|
|
76
|
+
return { ...input, normalizedStatus: normalizedStatus };
|
|
77
|
+
}
|
|
78
|
+
/** JSON stores one canonical snapshot; JSONL stores one canonical entry per line. */
|
|
79
|
+
export function readEvalCanonicalContext(input) {
|
|
80
|
+
if (input.content.length > 2_000_000)
|
|
81
|
+
throw new Error("Initial context is too large");
|
|
82
|
+
const parsed = input.format === "json"
|
|
83
|
+
? JSON.parse(input.content)
|
|
84
|
+
: { entries: input.content.split(/\r?\n/u).filter((line) => line.trim()).map((line) => JSON.parse(line)) };
|
|
85
|
+
if (!parsed || typeof parsed !== "object" || !Array.isArray(parsed.entries)) {
|
|
86
|
+
throw new Error("Initial context must contain canonical entries");
|
|
87
|
+
}
|
|
88
|
+
const entries = parsed.entries;
|
|
89
|
+
if (entries.length > 10_000)
|
|
90
|
+
throw new Error("Initial context has too many entries");
|
|
91
|
+
const seen = new Set();
|
|
92
|
+
for (const entry of entries) {
|
|
93
|
+
if (!entry || typeof entry !== "object")
|
|
94
|
+
throw new Error("Invalid canonical context entry");
|
|
95
|
+
const row = entry;
|
|
96
|
+
if (typeof row.id !== "string" || !row.id || seen.has(row.id) ||
|
|
97
|
+
!Number.isSafeInteger(row.createdAt) || Number(row.createdAt) < 0 ||
|
|
98
|
+
!row.message || typeof row.message !== "object" || Array.isArray(row.message) ||
|
|
99
|
+
!["user", "assistant", "toolResult"].includes(row.message.role ?? "")) {
|
|
100
|
+
throw new Error("Invalid or duplicate canonical context entry");
|
|
101
|
+
}
|
|
102
|
+
seen.add(row.id);
|
|
103
|
+
}
|
|
104
|
+
if (entries.length > 0 && entries[0].message.role !== "user") {
|
|
105
|
+
throw new Error("Initial context must begin with a user message");
|
|
106
|
+
}
|
|
107
|
+
return { entries: entries };
|
|
108
|
+
}
|
|
109
|
+
/** The one file an Evaluator writes its scored result to, in its own Space. */
|
|
110
|
+
export const EVAL_RESULT_FILE = "result.json";
|
|
111
|
+
export const EVAL_RESULT_PATH = `/output/${EVAL_RESULT_FILE}`;
|
|
112
|
+
export const EVAL_RESULT_MAX_BYTES = 1_000_000;
|
|
113
|
+
/** Reads the result file as raw UTF-8 text; any byte order mark and formatting are kept. */
|
|
114
|
+
export async function readEvalStructuredOutput(path, bytes) {
|
|
115
|
+
if (bytes.byteLength > EVAL_RESULT_MAX_BYTES)
|
|
116
|
+
throw new Error(`${EVAL_RESULT_FILE} exceeds ${EVAL_RESULT_MAX_BYTES} bytes`);
|
|
117
|
+
let content;
|
|
118
|
+
try {
|
|
119
|
+
content = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
|
|
120
|
+
}
|
|
121
|
+
catch {
|
|
122
|
+
throw new Error(`${EVAL_RESULT_FILE} is not valid UTF-8`);
|
|
123
|
+
}
|
|
124
|
+
const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", new Uint8Array(bytes)));
|
|
125
|
+
const contentHash = `sha256:${[...digest].map((byte) => byte.toString(16).padStart(2, "0")).join("")}`;
|
|
126
|
+
return { path, content, contentHash };
|
|
127
|
+
}
|
|
128
|
+
export function evalMaterialRoot(bundleId, materialId) {
|
|
129
|
+
return `/initial-materials/${encodeURIComponent(bundleId)}/${encodeURIComponent(materialId)}`;
|
|
130
|
+
}
|
|
131
|
+
/** Absolute paths below a root; `.`/`..`, `.git`, backslashes and NUL never reach the Space. */
|
|
132
|
+
function evalPathParts(path) {
|
|
133
|
+
const parts = path.split("/").slice(1);
|
|
134
|
+
if (!path.startsWith("/") || parts.length === 0 || parts.some((part) => !part || part === "." || part === ".." || part === ".git" || part.includes("\\") || part.includes("\0"))) {
|
|
135
|
+
return null;
|
|
136
|
+
}
|
|
137
|
+
return parts;
|
|
138
|
+
}
|
|
139
|
+
export function evalMaterialPath(root, path) {
|
|
140
|
+
const parts = evalPathParts(path);
|
|
141
|
+
if (!parts)
|
|
142
|
+
throw new Error("Invalid initial material path");
|
|
143
|
+
return `${root}/${parts.join("/")}`;
|
|
144
|
+
}
|
|
145
|
+
/*
|
|
146
|
+
* Evidence: a Subject's artifacts and complete trajectory are exported once, then imported into
|
|
147
|
+
* the Evaluator's own Space under `/evidence`. Neither Chat ever reads the other's Space.
|
|
148
|
+
*/
|
|
149
|
+
/** Where imported evidence lives in the Evaluator's own Space. */
|
|
150
|
+
export const EVAL_EVIDENCE_ROOT = "/evidence";
|
|
151
|
+
/** One exported or imported file crosses a single Durable Object RPC as bytes. */
|
|
152
|
+
export const EVAL_EVIDENCE_FILE_MAX_BYTES = 25_000_000;
|
|
153
|
+
export const EVAL_EVIDENCE_TRAJECTORY_PAGE_MAX = 200;
|
|
154
|
+
export const EVAL_EVIDENCE_SHA256_HEADER = "X-Eval-Content-Sha256";
|
|
155
|
+
/** One Subject artifact; its bytes are read separately so the inventory stays small. */
|
|
156
|
+
export const evalEvidenceArtifactSchema = z.strictObject({
|
|
157
|
+
path: z.string().startsWith("/").min(2),
|
|
158
|
+
size: z.number().int().nonnegative(),
|
|
159
|
+
mediaType: id,
|
|
160
|
+
});
|
|
161
|
+
export const evalEvidenceArtifactsSchema = z.strictObject({
|
|
162
|
+
artifacts: z.array(evalEvidenceArtifactSchema),
|
|
163
|
+
});
|
|
164
|
+
export function readEvalTrajectoryPageQuery(params) {
|
|
165
|
+
const afterSeq = Number(params.get("afterSeq") ?? "0");
|
|
166
|
+
const limit = Number(params.get("limit") ?? String(EVAL_EVIDENCE_TRAJECTORY_PAGE_MAX));
|
|
167
|
+
if (!Number.isSafeInteger(afterSeq) || afterSeq < 0 ||
|
|
168
|
+
!Number.isSafeInteger(limit) || limit < 1 || limit > EVAL_EVIDENCE_TRAJECTORY_PAGE_MAX) {
|
|
169
|
+
throw new Error("Invalid trajectory page");
|
|
170
|
+
}
|
|
171
|
+
return { afterSeq, limit };
|
|
172
|
+
}
|
|
173
|
+
/** A Space path for one evidence file; `relative` keeps the bundle's own layout below `/evidence`. */
|
|
174
|
+
export function evalEvidencePath(relative) {
|
|
175
|
+
const parts = evalPathParts(relative);
|
|
176
|
+
if (!parts)
|
|
177
|
+
throw new Error("Invalid evidence path");
|
|
178
|
+
return `${EVAL_EVIDENCE_ROOT}/${parts.join("/")}`;
|
|
179
|
+
}
|
|
180
|
+
export function isEvalSha256(value) {
|
|
181
|
+
return value !== null && /^[0-9a-f]{64}$/u.test(value);
|
|
182
|
+
}
|
|
183
|
+
/** Compare creation and operation facts without depending on JSON property order. */
|
|
184
|
+
export function stableEvalFacts(value) {
|
|
185
|
+
return JSON.stringify(sortFacts(value));
|
|
186
|
+
}
|
|
187
|
+
function sortFacts(value) {
|
|
188
|
+
if (Array.isArray(value))
|
|
189
|
+
return value.map(sortFacts);
|
|
190
|
+
if (!value || typeof value !== "object")
|
|
191
|
+
return value;
|
|
192
|
+
return Object.fromEntries(Object.entries(value)
|
|
193
|
+
.filter(([, item]) => item !== undefined)
|
|
194
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
195
|
+
.map(([key, item]) => [key, sortFacts(item)]));
|
|
196
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,34 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@springbrand/eval-harness",
|
|
3
|
-
"version": "0.0.0
|
|
4
|
-
"
|
|
5
|
-
"
|
|
3
|
+
"version": "0.1.0-alpha.0",
|
|
4
|
+
"description": "SP service contract for Eval-managed Chats: routes, headers and wire schemas.",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": {
|
|
9
|
+
"types": "./dist/index.d.ts",
|
|
10
|
+
"import": "./dist/index.js"
|
|
11
|
+
}
|
|
12
|
+
},
|
|
13
|
+
"files": [
|
|
14
|
+
"dist",
|
|
15
|
+
"README.md"
|
|
16
|
+
],
|
|
17
|
+
"publishConfig": {
|
|
18
|
+
"access": "public",
|
|
19
|
+
"tag": "alpha"
|
|
20
|
+
},
|
|
21
|
+
"dependencies": {
|
|
22
|
+
"zod": "^4.6.5"
|
|
23
|
+
},
|
|
24
|
+
"devDependencies": {
|
|
25
|
+
"typescript": "^7.0.2",
|
|
26
|
+
"vitest": "^4.1.10"
|
|
27
|
+
},
|
|
28
|
+
"scripts": {
|
|
29
|
+
"build": "tsc --noEmit",
|
|
30
|
+
"check": "tsc --noEmit",
|
|
31
|
+
"test": "vitest run",
|
|
32
|
+
"build:dist": "rm -rf dist && tsc -p tsconfig.build.json"
|
|
33
|
+
}
|
|
6
34
|
}
|