@springbrand/eval-harness 0.1.0-alpha.0 → 0.1.0-alpha.1

package/README.md

@@ -38,11 +38,30 @@ independent Eval SP deployment; the shared SP dev API and database are not used.
 
 The initialization body includes the Data sample and, only when referenced by
 that sample, `initialSession: {bundleId, sessionId, context?: {format, content}}`
-and/or `initialMaterials: {bundleId, materialId, files}`. `content` is canonical
+and/or `initialMaterials: [{bundleId, materialId, files}]`. `content` is canonical
 Session JSON or JSONL; each material file supplies its path, media type and
-base64 bytes. Supplied Session context is imported into the Chat before the Prompt and only
-material files are written to SpaceDO. A sample may specify either source, both,
-or neither. There is no Dataset-level source binding.
+base64 bytes. The material list must match the sample's `initialMaterials`
+references position by position, without duplicates. Supplied Session context is
+imported into the Chat before the Prompt and only files are written to SpaceDO,
+each material under its own `evalMaterialRoot`. A sample may specify either
+source, both, or neither. There is no Dataset-level source binding. An Evaluator
+initialization must carry `evaluationOutputSchema: {content}`, written as UTF-8
+`application/schema+json` to `EVAL_OUTPUT_SCHEMA_PATH`
+(`/evaluation-output.schema.json`); a Subject Chat rejects it. Evaluators do not
+import an initial Session. Schema validity and result validation belong to Eval
+Platform; SP preserves the supplied text. `evalMaterialFilesSchema`
+validates one material's `EvalMaterialFile` list: at most `EVAL_MATERIAL_MAX_FILES`, with unique
+paths. One initialization carries at most `EVAL_INITIAL_FILE_MAX_BYTES` per file
+and `EVAL_INITIALIZATION_MAX_BYTES` in total; size violations are Zod `too_big`
+issues. `evalMaterialFilesByteSize` returns one material's decoded bytes exactly as
+that total counts them. `readEvalCanonicalContext` accepts at most
+`EVAL_SESSION_CONTENT_MAX_CHARS` characters and `EVAL_SESSION_MAX_ENTRIES`
+entries. `EVAL_INITIALIZATION_BODY_MAX_BYTES` bounds transport JSON, accounting
+for base64 material bytes, escaped Session or Schema text, and JSON framing.
+Eval consumes these constants instead of copying the values. Its result is
+`evalInitializationResultSchema`: `{manifestHash, target, materials:
+[{bundleId, materialId, rootPath}], evaluationOutputSchemaPath?}` in request
+order; replaying the same request returns the stored result.
 
 An Evaluator is an ordinary Eval Chat with its own Session, Submission and
 Space; its creation facts add only `evaluationId`. It never reads or writes the
@@ -61,6 +80,46 @@ hash, writes `/evidence/<path>` and returns `{path, size, sha256}`. Only an
 Evaluator imports; an identical repeat is a no-op, different content is a 409,
 and imports stop with the Chat's stop latch.
 
+The Evaluator Workspace layout is one public contract, exported by this package:
+
+```text
+/initial-materials/{bundleId}/{materialId}/...  # Evaluator's own materials
+/evidence/subject-inputs/{bundleId}/{materialId}/...  # Subject's original input materials
+/evidence/artifacts/{original-relative-path}   # Subject's produced files
+/evidence/trajectory/events-000001.jsonl       # Complete raw trajectory, ordered parts
+/evidence/manifest.json                       # Evidence bundle inventory and provenance
+/evaluation-output.schema.json                # Evaluation output format
+/output/result.json                           # Evaluator's scored result
+```
+
+`EVAL_EVIDENCE_PATHS` names the bundle-relative directories. The import route takes
+paths **without** the `/evidence` prefix: `/subject-inputs/...`, `/artifacts/...`,
+`/trajectory/...` or `/manifest.json`. Other paths are rejected. The exported
+`evalSubjectInputEvidencePath`, `evalArtifactEvidencePath` and
+`evalTrajectoryEvidencePath` build those import paths; `evalEvidencePath` turns
+an import path into its absolute Workspace path. IDs are encoded as individual
+path segments and original file hierarchy is preserved, so equal basenames in
+different materials or evidence categories cannot overwrite each other.
+
+Eval Platform copies Subject inputs from the original immutable Material bundles
+referenced by this Sampling, not from the Subject's potentially edited working
+files. SP continues to exclude these inputs from artifact exports. Eval Platform
+collects **every** trajectory page until `nextAfterSeq` is null, retaining raw
+`{ingestSeq,event}` records in order; it splits oversized JSONL at event boundaries
+so every part fits `EVAL_EVIDENCE_FILE_MAX_BYTES`. Eval Platform owns the frozen
+bundle, manifest body, file hashes and preparation retries. It imports all files,
+verifies their receipts and uploads `manifest.json` last, then submits its Prompt
+with explicit evidence, Schema and result paths.
+
+SP requires completed initialization before evidence import, and a nonempty
+evidence manifest before an Evaluator's first Submission. Input writes and
+Submission admission are serialized; the first reserved Submission durably closes
+evidence imports, including after a DO restart. Replaying initialization returns
+its saved result without restoring files. Stop closes admission immediately and
+waits for accepted input writes and submissions. Evidence, initial materials and
+the fixed Schema input path are never exported as produced artifacts. SP does not
+interpret the manifest or implement Eval Platform's bundle assembly or scoring.
+
 Eval Platform writes the Evaluator instructions into the Prompt; SP submits it
 unchanged. The Evaluator's `output` adds `structuredOutput: {path, content,
 contentHash}` with the raw UTF-8 text of `/output/result.json` in its own Space

package/dist/index.d.ts

@@ -4,8 +4,8 @@ export declare const EVAL_HARNESS_PATH = "/internal/v1/eval-harness";
 export declare const evalEnsureChatSchema: z.ZodObject<{
     chatId: z.ZodUUID;
     role: z.ZodEnum<{
-        evaluator: "evaluator";
         subject: "subject";
+        evaluator: "evaluator";
     }>;
     framework: z.ZodLiteral<"sp">;
     samplingId: z.ZodString;
@@ -19,11 +19,29 @@ export declare const evalEnsureChatSchema: z.ZodObject<{
         timeoutMs: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strict>>;
 }, z.core.$strict>;
-export declare const evalMaterialFileSchema: z.ZodObject<{
+export declare const EVAL_INITIAL_FILE_MAX_BYTES = 5000000;
+export declare const EVAL_INITIALIZATION_MAX_BYTES = 20000000;
+export declare const EVAL_MATERIAL_MAX_FILES = 500;
+export declare const EVAL_SESSION_CONTENT_MAX_CHARS = 2000000;
+export declare const EVAL_SESSION_MAX_ENTRIES = 10000;
+export declare const EVAL_OUTPUT_SCHEMA_PATH = "/evaluation-output.schema.json";
+export declare const EVAL_INITIAL_MATERIALS_ROOT = "/initial-materials";
+/** Subject Session JSON and Evaluator Schema JSON can expand to six bytes per character. */
+export declare const EVAL_INITIALIZATION_BODY_MAX_BYTES: number;
+/** Decoded bytes of validated Material files, counted exactly as the initialization budget counts them. */
+export declare const evalMaterialFilesByteSize: (files: readonly {
+    readonly contentBase64: string;
+}[]) => number;
+declare const evalMaterialFileSchema: z.ZodObject<{
     path: z.ZodString;
     mediaType: z.ZodString;
     contentBase64: z.ZodBase64;
 }, z.core.$strict>;
+export declare const evalMaterialFilesSchema: z.ZodArray<z.ZodObject<{
+    path: z.ZodString;
+    mediaType: z.ZodString;
+    contentBase64: z.ZodBase64;
+}, z.core.$strict>>;
 export declare const evalInitialContextSchema: z.ZodObject<{
     format: z.ZodEnum<{
         json: "json";
@@ -47,7 +65,7 @@ export declare const evalInitializationSchema: z.ZodObject<{
             content: z.ZodString;
         }, z.core.$strict>>;
     }, z.core.$strict>>;
-    initialMaterials: z.ZodOptional<z.ZodObject<{
+    initialMaterials: z.ZodOptional<z.ZodArray<z.ZodObject<{
         bundleId: z.ZodString;
         materialId: z.ZodString;
         files: z.ZodArray<z.ZodObject<{
@@ -55,8 +73,21 @@ export declare const evalInitializationSchema: z.ZodObject<{
             mediaType: z.ZodString;
             contentBase64: z.ZodBase64;
         }, z.core.$strict>>;
+    }, z.core.$strict>>>;
+    evaluationOutputSchema: z.ZodOptional<z.ZodObject<{
+        content: z.ZodString;
     }, z.core.$strict>>;
 }, z.core.$strict>;
+export declare const evalInitializationResultSchema: z.ZodObject<{
+    manifestHash: z.ZodString;
+    target: z.ZodLiteral<"workspace">;
+    materials: z.ZodArray<z.ZodObject<{
+        bundleId: z.ZodString;
+        materialId: z.ZodString;
+        rootPath: z.ZodString;
+    }, z.core.$strict>>;
+    evaluationOutputSchemaPath: z.ZodExactOptional<z.ZodLiteral<"/evaluation-output.schema.json">>;
+}, z.core.$strict>;
 export declare const evalSubmitSchema: z.ZodObject<{
     idempotencyKey: z.ZodString;
     prompt: z.ZodString;
@@ -66,7 +97,9 @@ export declare const evalSubmitSchema: z.ZodObject<{
     }, z.core.$strict>>;
 }, z.core.$strict>;
 export type EvalEnsureChat = z.output<typeof evalEnsureChatSchema>;
+export type EvalMaterialFile = z.output<typeof evalMaterialFileSchema>;
 export type EvalInitialization = z.output<typeof evalInitializationSchema>;
+export type EvalInitializationResult = z.output<typeof evalInitializationResultSchema>;
 export type EvalSubmit = z.output<typeof evalSubmitSchema>;
 export type EvalSubmissionReceipt = {
     readonly submissionId: string;
@@ -135,6 +168,13 @@ export declare function evalMaterialRoot(bundleId: string, materialId: string):
 export declare function evalMaterialPath(root: string, path: string): string;
 /** Where imported evidence lives in the Evaluator's own Space. */
 export declare const EVAL_EVIDENCE_ROOT = "/evidence";
+/** Bundle-relative paths, used by the import route; evalEvidencePath returns their Workspace paths. */
+export declare const EVAL_EVIDENCE_PATHS: {
+    readonly subjectInputs: "/subject-inputs";
+    readonly artifacts: "/artifacts";
+    readonly trajectory: "/trajectory";
+    readonly manifest: "/manifest.json";
+};
 /** One exported or imported file crosses a single Durable Object RPC as bytes. */
 export declare const EVAL_EVIDENCE_FILE_MAX_BYTES = 25000000;
 export declare const EVAL_EVIDENCE_TRAJECTORY_PAGE_MAX = 200;
@@ -169,6 +209,19 @@ export declare function readEvalTrajectoryPageQuery(params: URLSearchParams): {
 };
 /** A Space path for one evidence file; `relative` keeps the bundle's own layout below `/evidence`. */
 export declare function evalEvidencePath(relative: string): string;
+/** Original Subject inputs stay separate from produced artifacts and the Evaluator's own materials. */
+export declare function evalSubjectInputEvidencePath(bundleId: string, materialId: string, path: string): string;
+export declare function evalArtifactEvidencePath(path: string): string;
+/** Positive part numbers preserve the complete raw JSONL trajectory's order. */
+export declare function evalTrajectoryEvidencePath(part: number): string;
+/** One raw evidence file imported into the Evaluator's own Space. */
+export type EvalEvidenceFileInput = {
+    /** Bundle-relative path under one of EVAL_EVIDENCE_PATHS, without the /evidence prefix. */
+    readonly path: string;
+    readonly mediaType: string;
+    readonly sha256: string;
+    readonly bytes: Uint8Array;
+};
 /** The receipt for one imported file; identical repeats return the same receipt. */
 export type EvalEvidenceFileReceipt = {
     readonly path: string;
@@ -178,3 +231,4 @@ export type EvalEvidenceFileReceipt = {
 export declare function isEvalSha256(value: string | null): value is string;
 /** Compare creation and operation facts without depending on JSON property order. */
 export declare function stableEvalFacts(value: unknown): string;
+export {};

package/dist/index.js

@@ -1,196 +1,294 @@
+// packages/eval-harness/src/index.ts
 import { z } from "zod";
-export const EVAL_SERVICE_HEADER = "X-Eval-Service-Token";
-export const EVAL_HARNESS_PATH = "/internal/v1/eval-harness";
-const id = z.string().trim().min(1);
-const uuid = z.uuid();
-export const evalEnsureChatSchema = z.strictObject({
-    chatId: uuid,
-    role: z.enum(["subject", "evaluator"]),
-    framework: z.literal("sp"),
-    samplingId: id,
-    projectId: id,
-    agentConfig: z.strictObject({ ownerUserId: uuid, userAgentId: uuid }),
-    /** Only an Evaluator carries it; the Evaluator still owns an ordinary Chat and its own Space. */
-    evaluationId: id.optional(),
-    runtime: z.strictObject({ timeoutMs: z.number().finite().positive().optional() }).optional(),
+
+// packages/utils/src/json.ts
+function canonicalJsonStringify(value) {
+  return JSON.stringify(
+    value,
+    (_key, item) => item !== null && typeof item === "object" && !Array.isArray(item) ? Object.fromEntries(Object.entries(item).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : item
+  );
+}
+
+// packages/utils/src/content-hash.ts
+async function sha256Hex(value) {
+  const bytes = typeof value === "string" ? new TextEncoder().encode(value) : value;
+  const buffer = new ArrayBuffer(bytes.byteLength);
+  new Uint8Array(buffer).set(bytes);
+  const digest = await crypto.subtle.digest("SHA-256", buffer);
+  return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+
+// packages/eval-harness/src/index.ts
+var EVAL_SERVICE_HEADER = "X-Eval-Service-Token";
+var EVAL_HARNESS_PATH = "/internal/v1/eval-harness";
+var id = z.string().trim().min(1);
+var uuid = z.uuid();
+var evalEnsureChatSchema = z.strictObject({
+  chatId: uuid,
+  role: z.enum(["subject", "evaluator"]),
+  framework: z.literal("sp"),
+  samplingId: id,
+  projectId: id,
+  agentConfig: z.strictObject({ ownerUserId: uuid, userAgentId: uuid }),
+  /** Only an Evaluator carries it; the Evaluator still owns an ordinary Chat and its own Space. */
+  evaluationId: id.optional(),
+  runtime: z.strictObject({ timeoutMs: z.number().finite().positive().optional() }).optional()
 }).superRefine((value, context) => {
-    if ((value.role === "evaluator") !== (value.evaluationId !== undefined)) {
-        context.addIssue({ code: "custom", path: ["evaluationId"], message: "Evaluation ID must match role" });
-    }
+  if (value.role === "evaluator" !== (value.evaluationId !== void 0)) {
+    context.addIssue({ code: "custom", path: ["evaluationId"], message: "Evaluation ID must match role" });
+  }
+});
+var EVAL_INITIAL_FILE_MAX_BYTES = 5e6;
+var EVAL_INITIALIZATION_MAX_BYTES = 2e7;
+var EVAL_MATERIAL_MAX_FILES = 500;
+var EVAL_SESSION_CONTENT_MAX_CHARS = 2e6;
+var EVAL_SESSION_MAX_ENTRIES = 1e4;
+var EVAL_OUTPUT_SCHEMA_PATH = "/evaluation-output.schema.json";
+var EVAL_INITIAL_MATERIALS_ROOT = "/initial-materials";
+var EVAL_INITIALIZATION_BODY_MAX_BYTES = Math.max(
+  Math.ceil(EVAL_INITIALIZATION_MAX_BYTES * 4 / 3) + EVAL_SESSION_CONTENT_MAX_CHARS * 6,
+  Math.ceil((EVAL_INITIALIZATION_MAX_BYTES - EVAL_INITIAL_FILE_MAX_BYTES) * 4 / 3) + EVAL_INITIAL_FILE_MAX_BYTES * 6
+) + 1e6;
+var base64Bytes = (value) => value.length / 4 * 3 - (value.endsWith("==") ? 2 : value.endsWith("=") ? 1 : 0);
+var utf8Bytes = (value) => new TextEncoder().encode(value).byteLength;
+var evalMaterialFilesByteSize = (files) => files.reduce((sum, file) => sum + base64Bytes(file.contentBase64), 0);
+var tooBig = (context, maximum, message, path = []) => context.addIssue({ code: "too_big", origin: "file", maximum, inclusive: true, path, message });
+var maxFileBytes = (bytes) => (value, context) => {
+  if (bytes(value) > EVAL_INITIAL_FILE_MAX_BYTES) tooBig(context, EVAL_INITIAL_FILE_MAX_BYTES, "File is too large");
+};
+var evalMaterialFileSchema = z.strictObject({
+  path: z.string().refine((path) => evalPathParts(path) !== null, "Invalid initial material path"),
+  mediaType: id,
+  contentBase64: z.base64().superRefine(maxFileBytes(base64Bytes))
 });
-export const evalMaterialFileSchema = z.strictObject({
-    path: z.string().startsWith("/").min(2),
-    mediaType: id,
-    contentBase64: z.base64(),
+var evalMaterialFilesSchema = z.array(evalMaterialFileSchema).max(EVAL_MATERIAL_MAX_FILES).superRefine((files, context) => {
+  const seen = /* @__PURE__ */ new Set();
+  files.forEach(({ path }, index) => {
+    if (seen.has(path)) context.addIssue({ code: "custom", path: [index, "path"], message: "Duplicate material path" });
+    seen.add(path);
+  });
 });
-export const evalInitialContextSchema = z.strictObject({
-    format: z.enum(["json", "jsonl"]),
-    content: z.string().min(1),
+var evalInitialMaterialSchema = z.strictObject({
+  bundleId: id,
+  materialId: id,
+  files: evalMaterialFilesSchema
 });
-export const evalInitializationSchema = z.strictObject({
-    idempotencyKey: id,
-    data: z.record(z.string(), z.unknown()).and(z.object({ userPrompt: z.string().trim().min(1) })),
-    initialSession: z.strictObject({
-        bundleId: id,
-        sessionId: id,
-        context: evalInitialContextSchema.optional(),
-    }).optional(),
-    initialMaterials: z.strictObject({
-        bundleId: id,
-        materialId: id,
-        files: z.array(evalMaterialFileSchema).max(500),
-    }).optional(),
+var evalInitialContextSchema = z.strictObject({
+  format: z.enum(["json", "jsonl"]),
+  content: z.string().min(1)
+});
+var evalInitializationSchema = z.strictObject({
+  idempotencyKey: id,
+  data: z.record(z.string(), z.unknown()).and(z.object({ userPrompt: z.string().trim().min(1) })),
+  initialSession: z.strictObject({
+    bundleId: id,
+    sessionId: id,
+    context: evalInitialContextSchema.optional()
+  }).optional(),
+  initialMaterials: z.array(evalInitialMaterialSchema).min(1).optional(),
+  evaluationOutputSchema: z.strictObject({
+    content: z.string().min(1).superRefine(maxFileBytes(utf8Bytes))
+  }).optional()
 }).superRefine((value, context) => {
-    const session = value.data.initialSession;
-    const materials = value.data.initialMaterials;
-    if (session ? !value.initialSession || session.bundleId !== value.initialSession.bundleId ||
-        session.sessionId !== value.initialSession.sessionId : value.initialSession !== undefined) {
-        context.addIssue({ code: "custom", path: ["initialSession"], message: "Session content must match the Data reference" });
-    }
-    if (materials ? !value.initialMaterials || materials.bundleId !== value.initialMaterials.bundleId ||
-        materials.materialId !== value.initialMaterials.materialId : value.initialMaterials !== undefined) {
-        context.addIssue({ code: "custom", path: ["initialMaterials"], message: "Material content must match the Data reference" });
-    }
+  const session = value.data.initialSession;
+  const refs = value.data.initialMaterials;
+  const materials = value.initialMaterials;
+  if (session ? !value.initialSession || session.bundleId !== value.initialSession.bundleId || session.sessionId !== value.initialSession.sessionId : value.initialSession !== void 0) {
+    context.addIssue({ code: "custom", path: ["initialSession"], message: "Session content must match the Data reference" });
+  }
+  if (refs === void 0 ? materials !== void 0 : !Array.isArray(refs) || refs.length !== materials?.length || materials.some((material, index) => {
+    const ref = refs[index];
+    return ref?.bundleId !== material.bundleId || ref?.materialId !== material.materialId;
+  })) {
+    context.addIssue({ code: "custom", path: ["initialMaterials"], message: "Material content must match the Data reference" });
+  }
+  const seen = /* @__PURE__ */ new Set();
+  materials?.forEach(({ bundleId, materialId }, index) => {
+    const key = JSON.stringify([bundleId, materialId]);
+    if (seen.has(key)) context.addIssue({ code: "custom", path: ["initialMaterials", index], message: "Duplicate material" });
+    seen.add(key);
+  });
+  const total = (materials ?? []).reduce(
+    (sum, material) => sum + evalMaterialFilesByteSize(material.files),
+    value.evaluationOutputSchema ? utf8Bytes(value.evaluationOutputSchema.content) : 0
+  );
+  if (total > EVAL_INITIALIZATION_MAX_BYTES) {
+    tooBig(context, EVAL_INITIALIZATION_MAX_BYTES, "Initialization files are too large");
+  }
 });
-export const evalSubmitSchema = z.strictObject({
-    idempotencyKey: id,
-    prompt: z.string().trim().min(1),
-    unattended: z.literal(true),
-    runtime: z.strictObject({ timeoutMs: z.number().finite().positive().optional() }).optional(),
+var evalInitializationResultSchema = z.strictObject({
+  manifestHash: id,
+  target: z.literal("workspace"),
+  materials: z.array(z.strictObject({ bundleId: id, materialId: id, rootPath: z.string().startsWith("/") })),
+  evaluationOutputSchemaPath: z.literal(EVAL_OUTPUT_SCHEMA_PATH).exactOptional()
 });
-export const evalBillingResourcesSchema = z.strictObject({
-    usageEvents: z.array(z.record(z.string(), z.unknown())),
-    billingSettlements: z.array(z.record(z.string(), z.unknown())),
+var evalSubmitSchema = z.strictObject({
+  idempotencyKey: id,
+  prompt: z.string().trim().min(1),
+  unattended: z.literal(true),
+  runtime: z.strictObject({ timeoutMs: z.number().finite().positive().optional() }).optional()
 });
-export function projectEvalSubmission(input) {
-    const normalizedStatus = {
-        pending: "queued",
-        running: "running",
-        completed: "completed",
-        error: "failed",
-        skipped: "failed",
-        aborted: "interrupted",
-    }[input.status];
-    if (!normalizedStatus)
-        throw new Error("Unknown SP Submission status");
-    return { ...input, normalizedStatus: normalizedStatus };
-}
-/** JSON stores one canonical snapshot; JSONL stores one canonical entry per line. */
-export function readEvalCanonicalContext(input) {
-    if (input.content.length > 2_000_000)
-        throw new Error("Initial context is too large");
-    const parsed = input.format === "json"
-        ? JSON.parse(input.content)
-        : { entries: input.content.split(/\r?\n/u).filter((line) => line.trim()).map((line) => JSON.parse(line)) };
-    if (!parsed || typeof parsed !== "object" || !Array.isArray(parsed.entries)) {
-        throw new Error("Initial context must contain canonical entries");
-    }
-    const entries = parsed.entries;
-    if (entries.length > 10_000)
-        throw new Error("Initial context has too many entries");
-    const seen = new Set();
-    for (const entry of entries) {
-        if (!entry || typeof entry !== "object")
-            throw new Error("Invalid canonical context entry");
-        const row = entry;
-        if (typeof row.id !== "string" || !row.id || seen.has(row.id) ||
-            !Number.isSafeInteger(row.createdAt) || Number(row.createdAt) < 0 ||
-            !row.message || typeof row.message !== "object" || Array.isArray(row.message) ||
-            !["user", "assistant", "toolResult"].includes(row.message.role ?? "")) {
-            throw new Error("Invalid or duplicate canonical context entry");
-        }
-        seen.add(row.id);
-    }
-    if (entries.length > 0 && entries[0].message.role !== "user") {
-        throw new Error("Initial context must begin with a user message");
-    }
-    return { entries: entries };
-}
-/** The one file an Evaluator writes its scored result to, in its own Space. */
-export const EVAL_RESULT_FILE = "result.json";
-export const EVAL_RESULT_PATH = `/output/${EVAL_RESULT_FILE}`;
-export const EVAL_RESULT_MAX_BYTES = 1_000_000;
-/** Reads the result file as raw UTF-8 text; any byte order mark and formatting are kept. */
-export async function readEvalStructuredOutput(path, bytes) {
-    if (bytes.byteLength > EVAL_RESULT_MAX_BYTES)
-        throw new Error(`${EVAL_RESULT_FILE} exceeds ${EVAL_RESULT_MAX_BYTES} bytes`);
-    let content;
-    try {
-        content = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
-    }
-    catch {
-        throw new Error(`${EVAL_RESULT_FILE} is not valid UTF-8`);
+var evalBillingResourcesSchema = z.strictObject({
+  usageEvents: z.array(z.record(z.string(), z.unknown())),
+  billingSettlements: z.array(z.record(z.string(), z.unknown()))
+});
+function projectEvalSubmission(input) {
+  const normalizedStatus = {
+    pending: "queued",
+    running: "running",
+    completed: "completed",
+    error: "failed",
+    skipped: "failed",
+    aborted: "interrupted"
+  }[input.status];
+  if (!normalizedStatus) throw new Error("Unknown SP Submission status");
+  return { ...input, normalizedStatus };
+}
+function readEvalCanonicalContext(input) {
+  if (input.content.length > EVAL_SESSION_CONTENT_MAX_CHARS) throw new Error("Initial context is too large");
+  const parsed = input.format === "json" ? JSON.parse(input.content) : { entries: input.content.split(/\r?\n/u).filter((line) => line.trim()).map((line) => JSON.parse(line)) };
+  if (!parsed || typeof parsed !== "object" || !Array.isArray(parsed.entries)) {
+    throw new Error("Initial context must contain canonical entries");
+  }
+  const entries = parsed.entries;
+  if (entries.length > EVAL_SESSION_MAX_ENTRIES) throw new Error("Initial context has too many entries");
+  const seen = /* @__PURE__ */ new Set();
+  for (const entry of entries) {
+    if (!entry || typeof entry !== "object") throw new Error("Invalid canonical context entry");
+    const row = entry;
+    if (typeof row.id !== "string" || !row.id || seen.has(row.id) || !Number.isSafeInteger(row.createdAt) || Number(row.createdAt) < 0 || !row.message || typeof row.message !== "object" || Array.isArray(row.message) || !["user", "assistant", "toolResult"].includes(row.message.role ?? "")) {
+      throw new Error("Invalid or duplicate canonical context entry");
     }
-    const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", new Uint8Array(bytes)));
-    const contentHash = `sha256:${[...digest].map((byte) => byte.toString(16).padStart(2, "0")).join("")}`;
-    return { path, content, contentHash };
+    seen.add(row.id);
+  }
+  if (entries.length > 0 && entries[0].message.role !== "user") {
+    throw new Error("Initial context must begin with a user message");
+  }
+  return { entries };
 }
-export function evalMaterialRoot(bundleId, materialId) {
-    return `/initial-materials/${encodeURIComponent(bundleId)}/${encodeURIComponent(materialId)}`;
+var EVAL_RESULT_FILE = "result.json";
+var EVAL_RESULT_PATH = `/output/${EVAL_RESULT_FILE}`;
+var EVAL_RESULT_MAX_BYTES = 1e6;
+async function readEvalStructuredOutput(path, bytes) {
+  if (bytes.byteLength > EVAL_RESULT_MAX_BYTES) throw new Error(`${EVAL_RESULT_FILE} exceeds ${EVAL_RESULT_MAX_BYTES} bytes`);
+  let content;
+  try {
+    content = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
+  } catch {
+    throw new Error(`${EVAL_RESULT_FILE} is not valid UTF-8`);
+  }
+  const contentHash = `sha256:${await sha256Hex(bytes)}`;
+  return { path, content, contentHash };
+}
+function evalMaterialRoot(bundleId, materialId) {
+  return `${EVAL_INITIAL_MATERIALS_ROOT}/${evalPathSegment(bundleId)}/${evalPathSegment(materialId)}`;
+}
+function evalPathSegment(value) {
+  return encodeURIComponent(value).replaceAll(".", "%2E");
 }
-/** Absolute paths below a root; `.`/`..`, `.git`, backslashes and NUL never reach the Space. */
 function evalPathParts(path) {
-    const parts = path.split("/").slice(1);
-    if (!path.startsWith("/") || parts.length === 0 || parts.some((part) => !part || part === "." || part === ".." || part === ".git" || part.includes("\\") || part.includes("\0"))) {
-        return null;
-    }
-    return parts;
-}
-export function evalMaterialPath(root, path) {
-    const parts = evalPathParts(path);
-    if (!parts)
-        throw new Error("Invalid initial material path");
-    return `${root}/${parts.join("/")}`;
-}
-/*
- * Evidence: a Subject's artifacts and complete trajectory are exported once, then imported into
- * the Evaluator's own Space under `/evidence`. Neither Chat ever reads the other's Space.
- */
-/** Where imported evidence lives in the Evaluator's own Space. */
-export const EVAL_EVIDENCE_ROOT = "/evidence";
-/** One exported or imported file crosses a single Durable Object RPC as bytes. */
-export const EVAL_EVIDENCE_FILE_MAX_BYTES = 25_000_000;
-export const EVAL_EVIDENCE_TRAJECTORY_PAGE_MAX = 200;
-export const EVAL_EVIDENCE_SHA256_HEADER = "X-Eval-Content-Sha256";
-/** One Subject artifact; its bytes are read separately so the inventory stays small. */
-export const evalEvidenceArtifactSchema = z.strictObject({
-    path: z.string().startsWith("/").min(2),
-    size: z.number().int().nonnegative(),
-    mediaType: id,
+  const parts = path.split("/").slice(1);
+  if (!path.startsWith("/") || parts.length === 0 || parts.some((part) => !part || part === "." || part === ".." || part === ".git" || part.includes("\\") || part.includes("\0"))) {
+    return null;
+  }
+  return parts;
+}
+function evalMaterialPath(root, path) {
+  const parts = evalPathParts(path);
+  if (!parts) throw new Error("Invalid initial material path");
+  return `${root}/${parts.join("/")}`;
+}
+var EVAL_EVIDENCE_ROOT = "/evidence";
+var EVAL_EVIDENCE_PATHS = {
+  subjectInputs: "/subject-inputs",
+  artifacts: "/artifacts",
+  trajectory: "/trajectory",
+  manifest: "/manifest.json"
+};
+var EVAL_EVIDENCE_FILE_MAX_BYTES = 25e6;
+var EVAL_EVIDENCE_TRAJECTORY_PAGE_MAX = 200;
+var EVAL_EVIDENCE_SHA256_HEADER = "X-Eval-Content-Sha256";
+var evalEvidenceArtifactSchema = z.strictObject({
+  path: z.string().startsWith("/").min(2),
+  size: z.number().int().nonnegative(),
+  mediaType: id
 });
-export const evalEvidenceArtifactsSchema = z.strictObject({
-    artifacts: z.array(evalEvidenceArtifactSchema),
+var evalEvidenceArtifactsSchema = z.strictObject({
+  artifacts: z.array(evalEvidenceArtifactSchema)
 });
-export function readEvalTrajectoryPageQuery(params) {
-    const afterSeq = Number(params.get("afterSeq") ?? "0");
-    const limit = Number(params.get("limit") ?? String(EVAL_EVIDENCE_TRAJECTORY_PAGE_MAX));
-    if (!Number.isSafeInteger(afterSeq) || afterSeq < 0 ||
-        !Number.isSafeInteger(limit) || limit < 1 || limit > EVAL_EVIDENCE_TRAJECTORY_PAGE_MAX) {
-        throw new Error("Invalid trajectory page");
-    }
-    return { afterSeq, limit };
-}
-/** A Space path for one evidence file; `relative` keeps the bundle's own layout below `/evidence`. */
-export function evalEvidencePath(relative) {
-    const parts = evalPathParts(relative);
-    if (!parts)
-        throw new Error("Invalid evidence path");
-    return `${EVAL_EVIDENCE_ROOT}/${parts.join("/")}`;
-}
-export function isEvalSha256(value) {
-    return value !== null && /^[0-9a-f]{64}$/u.test(value);
-}
-/** Compare creation and operation facts without depending on JSON property order. */
-export function stableEvalFacts(value) {
-    return JSON.stringify(sortFacts(value));
-}
-function sortFacts(value) {
-    if (Array.isArray(value))
-        return value.map(sortFacts);
-    if (!value || typeof value !== "object")
-        return value;
-    return Object.fromEntries(Object.entries(value)
-        .filter(([, item]) => item !== undefined)
-        .sort(([left], [right]) => left.localeCompare(right))
-        .map(([key, item]) => [key, sortFacts(item)]));
+function readEvalTrajectoryPageQuery(params) {
+  const afterSeq = Number(params.get("afterSeq") ?? "0");
+  const limit = Number(params.get("limit") ?? String(EVAL_EVIDENCE_TRAJECTORY_PAGE_MAX));
+  if (!Number.isSafeInteger(afterSeq) || afterSeq < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > EVAL_EVIDENCE_TRAJECTORY_PAGE_MAX) {
+    throw new Error("Invalid trajectory page");
+  }
+  return { afterSeq, limit };
+}
+function evalEvidencePath(relative) {
+  const parts = evalPathParts(relative);
+  if (!parts || relative !== EVAL_EVIDENCE_PATHS.manifest && ![EVAL_EVIDENCE_PATHS.subjectInputs, EVAL_EVIDENCE_PATHS.artifacts, EVAL_EVIDENCE_PATHS.trajectory].some((root) => relative.startsWith(`${root}/`))) throw new Error("Invalid evidence path");
+  return `${EVAL_EVIDENCE_ROOT}/${parts.join("/")}`;
+}
+function evalSubjectInputEvidencePath(bundleId, materialId, path) {
+  return evalMaterialPath(
+    `${EVAL_EVIDENCE_PATHS.subjectInputs}/${evalPathSegment(bundleId)}/${evalPathSegment(materialId)}`,
+    path
+  );
+}
+function evalArtifactEvidencePath(path) {
+  return evalMaterialPath(EVAL_EVIDENCE_PATHS.artifacts, path);
+}
+function evalTrajectoryEvidencePath(part) {
+  if (!Number.isSafeInteger(part) || part < 1) throw new Error("Invalid trajectory part");
+  return `${EVAL_EVIDENCE_PATHS.trajectory}/events-${String(part).padStart(6, "0")}.jsonl`;
+}
+function isEvalSha256(value) {
+  return value !== null && /^[0-9a-f]{64}$/u.test(value);
+}
+function stableEvalFacts(value) {
+  const facts = canonicalJsonStringify(value);
+  if (facts === void 0) throw new TypeError("Eval facts must be JSON");
+  return facts;
 }
+export {
+  EVAL_EVIDENCE_FILE_MAX_BYTES,
+  EVAL_EVIDENCE_PATHS,
+  EVAL_EVIDENCE_ROOT,
+  EVAL_EVIDENCE_SHA256_HEADER,
+  EVAL_EVIDENCE_TRAJECTORY_PAGE_MAX,
+  EVAL_HARNESS_PATH,
+  EVAL_INITIALIZATION_BODY_MAX_BYTES,
+  EVAL_INITIALIZATION_MAX_BYTES,
+  EVAL_INITIAL_FILE_MAX_BYTES,
+  EVAL_INITIAL_MATERIALS_ROOT,
+  EVAL_MATERIAL_MAX_FILES,
+  EVAL_OUTPUT_SCHEMA_PATH,
+  EVAL_RESULT_FILE,
+  EVAL_RESULT_MAX_BYTES,
+  EVAL_RESULT_PATH,
+  EVAL_SERVICE_HEADER,
+  EVAL_SESSION_CONTENT_MAX_CHARS,
+  EVAL_SESSION_MAX_ENTRIES,
+  evalArtifactEvidencePath,
+  evalBillingResourcesSchema,
+  evalEnsureChatSchema,
+  evalEvidenceArtifactSchema,
+  evalEvidenceArtifactsSchema,
+  evalEvidencePath,
+  evalInitialContextSchema,
+  evalInitializationResultSchema,
+  evalInitializationSchema,
+  evalMaterialFilesByteSize,
+  evalMaterialFilesSchema,
+  evalMaterialPath,
+  evalMaterialRoot,
+  evalSubjectInputEvidencePath,
+  evalSubmitSchema,
+  evalTrajectoryEvidencePath,
+  isEvalSha256,
+  projectEvalSubmission,
+  readEvalCanonicalContext,
+  readEvalStructuredOutput,
+  readEvalTrajectoryPageQuery,
+  stableEvalFacts
+};

package/package.json

@@ -1,9 +1,8 @@
 {
   "name": "@springbrand/eval-harness",
-  "version": "0.1.0-alpha.0",
-  "description": "SP service contract for Eval-managed Chats: routes, headers and wire schemas.",
-  "license": "MIT",
+  "version": "0.1.0-alpha.1",
   "type": "module",
+  "license": "UNLICENSED",
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
@@ -14,21 +13,17 @@
     "dist",
     "README.md"
   ],
-  "publishConfig": {
-    "access": "public",
-    "tag": "alpha"
-  },
+  "sideEffects": false,
   "dependencies": {
     "zod": "^4.6.5"
   },
-  "devDependencies": {
-    "typescript": "^7.0.2",
-    "vitest": "^4.1.10"
+  "publishConfig": {
+    "access": "public",
+    "tag": "alpha"
   },
-  "scripts": {
-    "build": "tsc --noEmit",
-    "check": "tsc --noEmit",
-    "test": "vitest run",
-    "build:dist": "rm -rf dist && tsc -p tsconfig.build.json"
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/springbrand-lab/sp-platform.git",
+    "directory": "packages/eval-harness"
   }
-}
+}