@springbrand/eval-harness 0.1.0-alpha.1 → 0.1.0-alpha.2

package/dist/index.js

@@ -1,294 +0,0 @@
-// packages/eval-harness/src/index.ts
-import { z } from "zod";
-
-// packages/utils/src/json.ts
-function canonicalJsonStringify(value) {
-  return JSON.stringify(
-    value,
-    (_key, item) => item !== null && typeof item === "object" && !Array.isArray(item) ? Object.fromEntries(Object.entries(item).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : item
-  );
-}
-
-// packages/utils/src/content-hash.ts
-async function sha256Hex(value) {
-  const bytes = typeof value === "string" ? new TextEncoder().encode(value) : value;
-  const buffer = new ArrayBuffer(bytes.byteLength);
-  new Uint8Array(buffer).set(bytes);
-  const digest = await crypto.subtle.digest("SHA-256", buffer);
-  return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
-}
-
-// packages/eval-harness/src/index.ts
-var EVAL_SERVICE_HEADER = "X-Eval-Service-Token";
-var EVAL_HARNESS_PATH = "/internal/v1/eval-harness";
-var id = z.string().trim().min(1);
-var uuid = z.uuid();
-var evalEnsureChatSchema = z.strictObject({
-  chatId: uuid,
-  role: z.enum(["subject", "evaluator"]),
-  framework: z.literal("sp"),
-  samplingId: id,
-  projectId: id,
-  agentConfig: z.strictObject({ ownerUserId: uuid, userAgentId: uuid }),
-  /** Only an Evaluator carries it; the Evaluator still owns an ordinary Chat and its own Space. */
-  evaluationId: id.optional(),
-  runtime: z.strictObject({ timeoutMs: z.number().finite().positive().optional() }).optional()
-}).superRefine((value, context) => {
-  if (value.role === "evaluator" !== (value.evaluationId !== void 0)) {
-    context.addIssue({ code: "custom", path: ["evaluationId"], message: "Evaluation ID must match role" });
-  }
-});
-var EVAL_INITIAL_FILE_MAX_BYTES = 5e6;
-var EVAL_INITIALIZATION_MAX_BYTES = 2e7;
-var EVAL_MATERIAL_MAX_FILES = 500;
-var EVAL_SESSION_CONTENT_MAX_CHARS = 2e6;
-var EVAL_SESSION_MAX_ENTRIES = 1e4;
-var EVAL_OUTPUT_SCHEMA_PATH = "/evaluation-output.schema.json";
-var EVAL_INITIAL_MATERIALS_ROOT = "/initial-materials";
-var EVAL_INITIALIZATION_BODY_MAX_BYTES = Math.max(
-  Math.ceil(EVAL_INITIALIZATION_MAX_BYTES * 4 / 3) + EVAL_SESSION_CONTENT_MAX_CHARS * 6,
-  Math.ceil((EVAL_INITIALIZATION_MAX_BYTES - EVAL_INITIAL_FILE_MAX_BYTES) * 4 / 3) + EVAL_INITIAL_FILE_MAX_BYTES * 6
-) + 1e6;
-var base64Bytes = (value) => value.length / 4 * 3 - (value.endsWith("==") ? 2 : value.endsWith("=") ? 1 : 0);
-var utf8Bytes = (value) => new TextEncoder().encode(value).byteLength;
-var evalMaterialFilesByteSize = (files) => files.reduce((sum, file) => sum + base64Bytes(file.contentBase64), 0);
-var tooBig = (context, maximum, message, path = []) => context.addIssue({ code: "too_big", origin: "file", maximum, inclusive: true, path, message });
-var maxFileBytes = (bytes) => (value, context) => {
-  if (bytes(value) > EVAL_INITIAL_FILE_MAX_BYTES) tooBig(context, EVAL_INITIAL_FILE_MAX_BYTES, "File is too large");
-};
-var evalMaterialFileSchema = z.strictObject({
-  path: z.string().refine((path) => evalPathParts(path) !== null, "Invalid initial material path"),
-  mediaType: id,
-  contentBase64: z.base64().superRefine(maxFileBytes(base64Bytes))
-});
-var evalMaterialFilesSchema = z.array(evalMaterialFileSchema).max(EVAL_MATERIAL_MAX_FILES).superRefine((files, context) => {
-  const seen = /* @__PURE__ */ new Set();
-  files.forEach(({ path }, index) => {
-    if (seen.has(path)) context.addIssue({ code: "custom", path: [index, "path"], message: "Duplicate material path" });
-    seen.add(path);
-  });
-});
-var evalInitialMaterialSchema = z.strictObject({
-  bundleId: id,
-  materialId: id,
-  files: evalMaterialFilesSchema
-});
-var evalInitialContextSchema = z.strictObject({
-  format: z.enum(["json", "jsonl"]),
-  content: z.string().min(1)
-});
-var evalInitializationSchema = z.strictObject({
-  idempotencyKey: id,
-  data: z.record(z.string(), z.unknown()).and(z.object({ userPrompt: z.string().trim().min(1) })),
-  initialSession: z.strictObject({
-    bundleId: id,
-    sessionId: id,
-    context: evalInitialContextSchema.optional()
-  }).optional(),
-  initialMaterials: z.array(evalInitialMaterialSchema).min(1).optional(),
-  evaluationOutputSchema: z.strictObject({
-    content: z.string().min(1).superRefine(maxFileBytes(utf8Bytes))
-  }).optional()
-}).superRefine((value, context) => {
-  const session = value.data.initialSession;
-  const refs = value.data.initialMaterials;
-  const materials = value.initialMaterials;
-  if (session ? !value.initialSession || session.bundleId !== value.initialSession.bundleId || session.sessionId !== value.initialSession.sessionId : value.initialSession !== void 0) {
-    context.addIssue({ code: "custom", path: ["initialSession"], message: "Session content must match the Data reference" });
-  }
-  if (refs === void 0 ? materials !== void 0 : !Array.isArray(refs) || refs.length !== materials?.length || materials.some((material, index) => {
-    const ref = refs[index];
-    return ref?.bundleId !== material.bundleId || ref?.materialId !== material.materialId;
-  })) {
-    context.addIssue({ code: "custom", path: ["initialMaterials"], message: "Material content must match the Data reference" });
-  }
-  const seen = /* @__PURE__ */ new Set();
-  materials?.forEach(({ bundleId, materialId }, index) => {
-    const key = JSON.stringify([bundleId, materialId]);
-    if (seen.has(key)) context.addIssue({ code: "custom", path: ["initialMaterials", index], message: "Duplicate material" });
-    seen.add(key);
-  });
-  const total = (materials ?? []).reduce(
-    (sum, material) => sum + evalMaterialFilesByteSize(material.files),
-    value.evaluationOutputSchema ? utf8Bytes(value.evaluationOutputSchema.content) : 0
-  );
-  if (total > EVAL_INITIALIZATION_MAX_BYTES) {
-    tooBig(context, EVAL_INITIALIZATION_MAX_BYTES, "Initialization files are too large");
-  }
-});
-var evalInitializationResultSchema = z.strictObject({
-  manifestHash: id,
-  target: z.literal("workspace"),
-  materials: z.array(z.strictObject({ bundleId: id, materialId: id, rootPath: z.string().startsWith("/") })),
-  evaluationOutputSchemaPath: z.literal(EVAL_OUTPUT_SCHEMA_PATH).exactOptional()
-});
-var evalSubmitSchema = z.strictObject({
-  idempotencyKey: id,
-  prompt: z.string().trim().min(1),
-  unattended: z.literal(true),
-  runtime: z.strictObject({ timeoutMs: z.number().finite().positive().optional() }).optional()
-});
-var evalBillingResourcesSchema = z.strictObject({
-  usageEvents: z.array(z.record(z.string(), z.unknown())),
-  billingSettlements: z.array(z.record(z.string(), z.unknown()))
-});
-function projectEvalSubmission(input) {
-  const normalizedStatus = {
-    pending: "queued",
-    running: "running",
-    completed: "completed",
-    error: "failed",
-    skipped: "failed",
-    aborted: "interrupted"
-  }[input.status];
-  if (!normalizedStatus) throw new Error("Unknown SP Submission status");
-  return { ...input, normalizedStatus };
-}
-function readEvalCanonicalContext(input) {
-  if (input.content.length > EVAL_SESSION_CONTENT_MAX_CHARS) throw new Error("Initial context is too large");
-  const parsed = input.format === "json" ? JSON.parse(input.content) : { entries: input.content.split(/\r?\n/u).filter((line) => line.trim()).map((line) => JSON.parse(line)) };
-  if (!parsed || typeof parsed !== "object" || !Array.isArray(parsed.entries)) {
-    throw new Error("Initial context must contain canonical entries");
-  }
-  const entries = parsed.entries;
-  if (entries.length > EVAL_SESSION_MAX_ENTRIES) throw new Error("Initial context has too many entries");
-  const seen = /* @__PURE__ */ new Set();
-  for (const entry of entries) {
-    if (!entry || typeof entry !== "object") throw new Error("Invalid canonical context entry");
-    const row = entry;
-    if (typeof row.id !== "string" || !row.id || seen.has(row.id) || !Number.isSafeInteger(row.createdAt) || Number(row.createdAt) < 0 || !row.message || typeof row.message !== "object" || Array.isArray(row.message) || !["user", "assistant", "toolResult"].includes(row.message.role ?? "")) {
-      throw new Error("Invalid or duplicate canonical context entry");
-    }
-    seen.add(row.id);
-  }
-  if (entries.length > 0 && entries[0].message.role !== "user") {
-    throw new Error("Initial context must begin with a user message");
-  }
-  return { entries };
-}
-var EVAL_RESULT_FILE = "result.json";
-var EVAL_RESULT_PATH = `/output/${EVAL_RESULT_FILE}`;
-var EVAL_RESULT_MAX_BYTES = 1e6;
-async function readEvalStructuredOutput(path, bytes) {
-  if (bytes.byteLength > EVAL_RESULT_MAX_BYTES) throw new Error(`${EVAL_RESULT_FILE} exceeds ${EVAL_RESULT_MAX_BYTES} bytes`);
-  let content;
-  try {
-    content = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
-  } catch {
-    throw new Error(`${EVAL_RESULT_FILE} is not valid UTF-8`);
-  }
-  const contentHash = `sha256:${await sha256Hex(bytes)}`;
-  return { path, content, contentHash };
-}
-function evalMaterialRoot(bundleId, materialId) {
-  return `${EVAL_INITIAL_MATERIALS_ROOT}/${evalPathSegment(bundleId)}/${evalPathSegment(materialId)}`;
-}
-function evalPathSegment(value) {
-  return encodeURIComponent(value).replaceAll(".", "%2E");
-}
-function evalPathParts(path) {
-  const parts = path.split("/").slice(1);
-  if (!path.startsWith("/") || parts.length === 0 || parts.some((part) => !part || part === "." || part === ".." || part === ".git" || part.includes("\\") || part.includes("\0"))) {
-    return null;
-  }
-  return parts;
-}
-function evalMaterialPath(root, path) {
-  const parts = evalPathParts(path);
-  if (!parts) throw new Error("Invalid initial material path");
-  return `${root}/${parts.join("/")}`;
-}
-var EVAL_EVIDENCE_ROOT = "/evidence";
-var EVAL_EVIDENCE_PATHS = {
-  subjectInputs: "/subject-inputs",
-  artifacts: "/artifacts",
-  trajectory: "/trajectory",
-  manifest: "/manifest.json"
-};
-var EVAL_EVIDENCE_FILE_MAX_BYTES = 25e6;
-var EVAL_EVIDENCE_TRAJECTORY_PAGE_MAX = 200;
-var EVAL_EVIDENCE_SHA256_HEADER = "X-Eval-Content-Sha256";
-var evalEvidenceArtifactSchema = z.strictObject({
-  path: z.string().startsWith("/").min(2),
-  size: z.number().int().nonnegative(),
-  mediaType: id
-});
-var evalEvidenceArtifactsSchema = z.strictObject({
-  artifacts: z.array(evalEvidenceArtifactSchema)
-});
-function readEvalTrajectoryPageQuery(params) {
-  const afterSeq = Number(params.get("afterSeq") ?? "0");
-  const limit = Number(params.get("limit") ?? String(EVAL_EVIDENCE_TRAJECTORY_PAGE_MAX));
-  if (!Number.isSafeInteger(afterSeq) || afterSeq < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > EVAL_EVIDENCE_TRAJECTORY_PAGE_MAX) {
-    throw new Error("Invalid trajectory page");
-  }
-  return { afterSeq, limit };
-}
-function evalEvidencePath(relative) {
-  const parts = evalPathParts(relative);
-  if (!parts || relative !== EVAL_EVIDENCE_PATHS.manifest && ![EVAL_EVIDENCE_PATHS.subjectInputs, EVAL_EVIDENCE_PATHS.artifacts, EVAL_EVIDENCE_PATHS.trajectory].some((root) => relative.startsWith(`${root}/`))) throw new Error("Invalid evidence path");
-  return `${EVAL_EVIDENCE_ROOT}/${parts.join("/")}`;
-}
-function evalSubjectInputEvidencePath(bundleId, materialId, path) {
-  return evalMaterialPath(
-    `${EVAL_EVIDENCE_PATHS.subjectInputs}/${evalPathSegment(bundleId)}/${evalPathSegment(materialId)}`,
-    path
-  );
-}
-function evalArtifactEvidencePath(path) {
-  return evalMaterialPath(EVAL_EVIDENCE_PATHS.artifacts, path);
-}
-function evalTrajectoryEvidencePath(part) {
-  if (!Number.isSafeInteger(part) || part < 1) throw new Error("Invalid trajectory part");
-  return `${EVAL_EVIDENCE_PATHS.trajectory}/events-${String(part).padStart(6, "0")}.jsonl`;
-}
-function isEvalSha256(value) {
-  return value !== null && /^[0-9a-f]{64}$/u.test(value);
-}
-function stableEvalFacts(value) {
-  const facts = canonicalJsonStringify(value);
-  if (facts === void 0) throw new TypeError("Eval facts must be JSON");
-  return facts;
-}
-export {
-  EVAL_EVIDENCE_FILE_MAX_BYTES,
-  EVAL_EVIDENCE_PATHS,
-  EVAL_EVIDENCE_ROOT,
-  EVAL_EVIDENCE_SHA256_HEADER,
-  EVAL_EVIDENCE_TRAJECTORY_PAGE_MAX,
-  EVAL_HARNESS_PATH,
-  EVAL_INITIALIZATION_BODY_MAX_BYTES,
-  EVAL_INITIALIZATION_MAX_BYTES,
-  EVAL_INITIAL_FILE_MAX_BYTES,
-  EVAL_INITIAL_MATERIALS_ROOT,
-  EVAL_MATERIAL_MAX_FILES,
-  EVAL_OUTPUT_SCHEMA_PATH,
-  EVAL_RESULT_FILE,
-  EVAL_RESULT_MAX_BYTES,
-  EVAL_RESULT_PATH,
-  EVAL_SERVICE_HEADER,
-  EVAL_SESSION_CONTENT_MAX_CHARS,
-  EVAL_SESSION_MAX_ENTRIES,
-  evalArtifactEvidencePath,
-  evalBillingResourcesSchema,
-  evalEnsureChatSchema,
-  evalEvidenceArtifactSchema,
-  evalEvidenceArtifactsSchema,
-  evalEvidencePath,
-  evalInitialContextSchema,
-  evalInitializationResultSchema,
-  evalInitializationSchema,
-  evalMaterialFilesByteSize,
-  evalMaterialFilesSchema,
-  evalMaterialPath,
-  evalMaterialRoot,
-  evalSubjectInputEvidencePath,
-  evalSubmitSchema,
-  evalTrajectoryEvidencePath,
-  isEvalSha256,
-  projectEvalSubmission,
-  readEvalCanonicalContext,
-  readEvalStructuredOutput,
-  readEvalTrajectoryPageQuery,
-  stableEvalFacts
-};