@coder/aegis 0.2.0 → 0.3.0

package/lib/index.js

@@ -238,14 +238,7 @@ var ExperimentBuilder = class {
 			strict(this.baseRepos.has(task.seed.base_repo), `task ${task.id}: base repo ${task.seed.base_repo} was not declared`);
 			validateTaskGraders(task, graders);
 		}
-		const subject = {
-			harness: this.config.subject.harness,
-			harness_version: this.config.subject.harnessVersion,
-			control_transport: this.config.subject.controlTransport,
-			model: this.config.subject.model,
-			allowed_tools: [...this.config.subject.allowedTools],
-			env: sortedStringRecord(this.config.subject.env ?? {})
-		};
+		const subjectFields = this.config.subject !== void 0 ? { subject: experimentSubjectDoc(this.config.subject) } : { subjects: codeUnitSortedSubjects(this.config.subjects ?? {}) };
 		const budget = {
 			max_usd: this.config.budget.maxUsd,
 			confirm_required: this.config.budget.confirmRequired,
@@ -262,7 +255,7 @@ var ExperimentBuilder = class {
 				backend: this.config.backend,
 				...backendOptions === void 0 ? {} : { backend_options: backendOptions },
 				hypothesis: this.config.hypothesis,
-				subject,
+				...subjectFields,
 				suite: suiteID,
 				grader_registry: ["graders/common.yaml"],
 				suites: suiteDocs,
@@ -595,12 +588,23 @@ function assertExperimentConfig(id, config) {
 		}
 	}
 	assertNonEmpty(`experiment ${id}.hypothesis`, config.hypothesis);
-	strict(config.subject && typeof config.subject === "object", `experiment ${id}: subject is required`);
-	assertNonEmpty(`experiment ${id}.subject.harness`, config.subject.harness);
-	assertNonEmpty(`experiment ${id}.subject.harnessVersion`, config.subject.harnessVersion);
-	assertNonEmpty(`experiment ${id}.subject.controlTransport`, config.subject.controlTransport);
-	assertNonEmpty(`experiment ${id}.subject.model`, config.subject.model);
-	assertStringArray(`experiment ${id}.subject.allowedTools`, config.subject.allowedTools);
+	strict(config.subject === void 0 !== (config.subjects === void 0), `experiment ${id}: exactly one of subject or subjects must be set`);
+	if (config.subject !== void 0) {
+		strict(config.subject && typeof config.subject === "object", `experiment ${id}: subject is required`);
+		assertExperimentSubjectInput(`experiment ${id}.subject`, config.subject);
+	} else {
+		strict(config.subjects !== null && typeof config.subjects === "object" && !Array.isArray(config.subjects), `experiment ${id}.subjects must be an object`);
+		const subjectIDs = Object.keys(config.subjects ?? {});
+		strict(subjectIDs.length > 0, `experiment ${id}.subjects must declare at least one subject`);
+		for (const subjectID of subjectIDs) {
+			assertSafeID("subject", subjectID);
+			strict(!/^[0-9]+$/.test(subjectID), `experiment ${id}.subjects id ${JSON.stringify(subjectID)} must not be purely numeric: JSON object key order for integer-like keys diverges between emitters`);
+			strict(!subjectID.includes("__"), `experiment ${id}.subjects id ${JSON.stringify(subjectID)} must not contain "__": trial ids join task, subject, variant, and repeat segments with it`);
+			const subject = (config.subjects ?? {})[subjectID];
+			strict(subject && typeof subject === "object", `experiment ${id}.subjects[${JSON.stringify(subjectID)}] must be an object`);
+			assertExperimentSubjectInput(`experiment ${id}.subjects[${JSON.stringify(subjectID)}]`, subject);
+		}
+	}
 	strict(Number.isInteger(config.repeats) && config.repeats > 0, `experiment ${id}: repeats must be a positive integer`);
 	strict(config.budget && typeof config.budget === "object", `experiment ${id}: budget is required`);
 	assertFiniteNumber(`experiment ${id}.budget.maxUsd`, config.budget.maxUsd);
@@ -613,6 +617,13 @@ function assertExperimentConfig(id, config) {
 	assertNonEmpty(`experiment ${id}.scheduling.order`, config.scheduling.order);
 	assertNonEmpty(`experiment ${id}.scheduling.retry`, config.scheduling.retry);
 }
+function assertExperimentSubjectInput(context, subject) {
+	assertNonEmpty(`${context}.harness`, subject.harness);
+	assertNonEmpty(`${context}.harnessVersion`, subject.harnessVersion);
+	assertNonEmpty(`${context}.controlTransport`, subject.controlTransport);
+	assertNonEmpty(`${context}.model`, subject.model);
+	assertStringArray(`${context}.allowedTools`, subject.allowedTools);
+}
 function assertGraderRef(context, grader) {
 	strict(grader && typeof grader === "object", `${context}: grader ref must be an object`);
 	strict(grader.kind === "aegis.grader-ref", `${context}: expected an Aegis grader helper result`);
@@ -825,6 +836,20 @@ function normalizePrompt(input) {
 	strict(content.length > 0, "prompt must not be blank");
 	return `${content}\n`;
 }
+function experimentSubjectDoc(subject) {
+	return {
+		harness: subject.harness,
+		harness_version: subject.harnessVersion,
+		control_transport: subject.controlTransport,
+		model: subject.model,
+		allowed_tools: [...subject.allowedTools],
+		env: sortedStringRecord(subject.env ?? {})
+	};
+}
+function codeUnitSortedSubjects(input) {
+	const entries = Object.entries(input).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
+	return Object.fromEntries(entries.map(([id, subject]) => [id, experimentSubjectDoc(subject)]));
+}
 function sortedStringRecord(input) {
 	const entries = Object.entries(input).sort(([a], [b]) => a.localeCompare(b));
 	for (const [key, value] of entries) {

package/lib/types/builder.d.ts

@@ -10,7 +10,10 @@ export interface ExperimentConfig {
     backend: string;
     backendOptions?: BackendOptionsInput;
     hypothesis: string;
-    subject: ExperimentSubjectInput;
+    /** Exactly one of subject or subjects must be set. */
+    subject?: ExperimentSubjectInput;
+    /** Multi-subject axis keyed by repo-safe subject id. */
+    subjects?: Record<string, ExperimentSubjectInput>;
     repeats: number;
     budget: BudgetInput;
     primaryMetric: string;

package/lib/types/schema.d.ts

@@ -4,7 +4,8 @@ export interface ExperimentDoc {
     backend: string;
     backend_options?: BackendOptionsDoc;
     hypothesis: string;
-    subject: ExperimentSubjectDoc;
+    subject?: ExperimentSubjectDoc;
+    subjects?: Record<string, ExperimentSubjectDoc>;
     suite: string;
     grader_registry: string[];
     suites: Record<string, SuiteDoc>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@coder/aegis",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "aegis CLI and TypeScript eval-authoring facade (Go binary with embedded TS eval compiler and web UI, plus the @coder/aegis import surface)",
   "type": "module",
   "bin": {
@@ -23,10 +23,10 @@
     "url": "git+https://github.com/coder/aegis.git"
   },
   "optionalDependencies": {
-    "@coder/aegis-linux-x64": "0.2.0",
-    "@coder/aegis-linux-arm64": "0.2.0",
-    "@coder/aegis-darwin-x64": "0.2.0",
-    "@coder/aegis-darwin-arm64": "0.2.0",
-    "@coder/aegis-win32-x64": "0.2.0"
+    "@coder/aegis-linux-x64": "0.3.0",
+    "@coder/aegis-linux-arm64": "0.3.0",
+    "@coder/aegis-darwin-x64": "0.3.0",
+    "@coder/aegis-darwin-arm64": "0.3.0",
+    "@coder/aegis-win32-x64": "0.3.0"
   }
 }