@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.40

package/src/drivers/subprocess/index.ts

@@ -2,13 +2,15 @@ import type { ChildProcessWithoutNullStreams } from "node:child_process";
 import { createInterface } from "node:readline";
 import { fileURLToPath } from "node:url";
 import { z } from "zod";
+import { agentEnvironment, InheritEnvironmentSchema } from "../../core/agent-environment.js";
+import { DurationSchema, durationMs } from "../../core/duration.js";
 import {
   type AgentDriver,
   type Answer,
   type Artifact,
   DoneSchema,
   type DriverEventMap,
-  type DriverPlugin,
+  defineDriver,
   ErrorEventSchema,
   type FormatAnswerFn,
   formatWireAnswer,
@@ -25,6 +27,7 @@ import {
   TypedEmitter,
 } from "../../core/index.js";
 import { walkArtifacts } from "../shared/artifacts.js";
+import { prepareSubprocessExperimentConfig } from "../shared/experiment-policy.js";
 
 export const SubprocessDriverConfigSchema = z
   .object({
@@ -33,9 +36,9 @@ export const SubprocessDriverConfigSchema = z
     cwd: z.string().optional(),
     env: z.record(z.string(), z.string()).optional(),
     /** Inherit the host environment. Experiment mode always forces this off. */
-    inheritEnv: z.boolean().optional(),
+    inheritEnv: z.union([InheritEnvironmentSchema, z.literal(false)]).optional(),
     protocol: z.enum(["lux-jsonl", "custom"]).default("lux-jsonl"),
-    idleTimeoutMs: z.number().positive().default(120_000),
+    idleTimeout: DurationSchema.default("2 minutes"),
     maxInterrupts: z.number().positive().default(20),
     /**
      * Maximum files inventoried from LUX_ARTIFACTS_DIR. Keep the conservative
@@ -81,6 +84,8 @@ export type SubprocessDriverConfig = z.infer<typeof SubprocessDriverConfigSchema
 const KILL_GRACE_MS = 5_000;
 const DEFAULT_MAX_LINE_BYTES = 10_000_000;
 const MAX_STDERR_TAIL_CHARS = 8_192;
+const MAX_STDERR_CHARS = 65_536;
+const MAX_STDERR_EVENTS = 128;
 const DEFAULT_MAX_ARTIFACT_FILES = 500;
 const MAX_FIXTURE_VALUES_BYTES = 64 * 1024;
 const ARTIFACT_SNAPSHOT_MODULE = fileURLToPath(
@@ -145,9 +150,11 @@ class SubprocessDriverImpl implements AgentDriver {
       throw new Error(`subprocess fixture values exceed ${MAX_FIXTURE_VALUES_BYTES} bytes`);
     }
     const managed = spawnManagedProcess(this.config.command, this.config.args ?? [], {
-      cwd: this.config.cwd ?? ctx.runDir,
+      cwd: this.config.cwd ?? ctx.artifactsDir,
       env: {
-        ...(this.config.inheritEnv === false ? {} : process.env),
+        ...(this.config.inheritEnv === false
+          ? {}
+          : agentEnvironment(process.env, this.config.inheritEnv)),
         ...this.config.env,
         ...(ctx.caseId ? { LUX_CASE_ID: ctx.caseId } : {}),
         LUX_RUN_DIR: ctx.runDir,
@@ -174,15 +181,38 @@ class SubprocessDriverImpl implements AgentDriver {
     // boundary from being mangled.
     child.stderr.setEncoding("utf8");
     let stderrTail = "";
+    let stderrChars = 0;
+    let stderrEvents = 0;
+    let stderrTruncated = false;
+    const recordStderr = (line: string): void => {
+      if (stderrTruncated || this.terminalSent) return;
+      const text = line.slice(0, Math.min(MAX_STDERR_TAIL_CHARS, MAX_STDERR_CHARS - stderrChars));
+      stderrChars += text.length;
+      stderrEvents++;
+      stderrTruncated =
+        text.length < line.length ||
+        stderrChars >= MAX_STDERR_CHARS ||
+        stderrEvents >= MAX_STDERR_EVENTS;
+      this.emitInOrder("progress", {
+        id: `subprocess-stderr-${stderrEvents}`,
+        kind: "driver-stderr",
+        payload: { text, truncated: stderrTruncated },
+        ts: Date.now(),
+      });
+    };
     child.stderr.on("data", (chunk: string) => {
       const lines = (stderrTail + chunk).split("\n");
       // Cap the retained partial line so unterminated stderr can't grow
       // without bound.
-      stderrTail = (lines.pop() ?? "").slice(0, MAX_STDERR_TAIL_CHARS);
-      for (const line of lines) process.stderr.write(`[subprocess] ${line}\n`);
+      stderrTail = lines.pop() ?? "";
+      for (const line of lines) recordStderr(line);
+      if (stderrTail.length > MAX_STDERR_TAIL_CHARS) {
+        recordStderr(stderrTail);
+        stderrTail = "";
+      }
     });
     child.stderr.on("close", () => {
-      if (stderrTail) process.stderr.write(`[subprocess] ${stderrTail}\n`);
+      if (stderrTail) recordStderr(stderrTail);
       stderrTail = "";
     });
 
@@ -459,7 +489,7 @@ class SubprocessDriverImpl implements AgentDriver {
   private bumpIdleTimer(): void {
     this.clearIdleTimer();
     this.idleTimer = setTimeout(() => {
-      const reason = `no event for ${this.config.idleTimeoutMs}ms`;
+      const reason = `no event for ${durationMs(this.config.idleTimeout)}ms`;
       // Idle before ready means start() is still pending; reject it (and kill
       // the child) or the orchestrator waits forever.
       const preReady = this.readyRejected !== null;
@@ -471,7 +501,7 @@ class SubprocessDriverImpl implements AgentDriver {
       // Kill immediately: the run is lost and the child would keep doing
       // billable work until close().
       this.terminate();
-    }, this.config.idleTimeoutMs);
+    }, durationMs(this.config.idleTimeout));
   }
 
   private clearIdleTimer(): void {
@@ -489,8 +519,21 @@ class SubprocessDriverImpl implements AgentDriver {
   }
 }
 
-export const subprocessDriver: DriverPlugin<SubprocessDriverConfig> = {
+export const subprocessDriver = defineDriver({
   name: "subprocess",
   configSchema: SubprocessDriverConfigSchema,
+  runtime: (config) => ({
+    commands: [config.command],
+    capabilities: {
+      interactive: true,
+      artifacts: true,
+      isolation: false,
+    },
+  }),
+  experiment: {
+    version: "1",
+    prepare: prepareSubprocessExperimentConfig,
+    identity: () => null,
+  },
   create: (config) => new SubprocessDriverImpl(config),
-};
+});

package/src/index.ts

@@ -1,9 +1,9 @@
+export { DurationSchema, durationMs } from "./core/duration.js";
 // Public surface of @prismatic-io/lux. The package root exposes authoring
 // contracts, extension points, built-ins, and supported programmatic
 // lifecycles. Explicit exports keep implementation and test helpers private.
 
 export { z } from "zod";
-
 // Bundled answerers and drivers. Test injection and protocol-normalization
 // helpers stay in their implementation modules.
 export {
@@ -87,6 +87,8 @@ export {
   registerCoreAssertions,
   runSucceededAssertion,
   subagentToolCalls,
+  type TimeToFirstActionUnderConfig,
+  TimeToFirstActionUnderConfigSchema,
   type TokensUnderConfig,
   TokensUnderConfigSchema,
   type ToolCall,
@@ -104,6 +106,7 @@ export {
   ToolResultIsErrorConfigSchema,
   type ToolResultJsonPointerEqualsConfig,
   ToolResultJsonPointerEqualsConfigSchema,
+  timeToFirstActionUnderAssertion,
   tokensUnderAssertion,
   toolCallCountAssertion,
   toolCalledAssertion,
@@ -136,11 +139,34 @@ export {
   type RubricConfig,
   RubricConfigSchema,
   type RubricEvidence,
+  type RubricEvidenceInput,
   RubricEvidenceSchema,
   RubricModelAccountingDetailSchema,
   rubricAssertion,
 } from "./assertions/rubric/index.js";
 // Authoring and extension contracts.
+export {
+  type ActionCategory,
+  ActionCategorySchema,
+  type ActionProgressPayload,
+  ActionProgressPayloadSchema,
+  type ActionTiming,
+  ActionTimingSchema,
+  actionTiming,
+} from "./core/action-events.js";
+export {
+  type AgentActivity,
+  AgentActivitySchema,
+  summarizeAgentActivity,
+} from "./core/agent-activity.js";
+export {
+  AGENT_COMPLETED_KIND,
+  AGENT_SPAWN_KIND,
+  type AgentCompletedPayload,
+  AgentCompletedPayloadSchema,
+  type AgentSpawnPayload,
+  AgentSpawnPayloadSchema,
+} from "./core/agent-events.js";
 export {
   defineGraderAnnotations,
   type GraderAnnotation,
@@ -158,11 +184,14 @@ export {
   AnswererModelCallAccountingSchema,
   type AnswererPlugin,
   type AnswerHistoryEntry,
+  answererRef,
+  defineAnswerer,
 } from "./core/answerer.js";
 export {
   inspectRunArtifact,
   type ReadRunArtifactTextOptions,
   type RunArtifactInspection,
+  type RunArtifactIssueReason,
   readRunArtifactText,
 } from "./core/artifact-evidence.js";
 export {
@@ -201,12 +230,14 @@ export {
   type EvalCase,
   type EvalCaseAuthoringOptions,
   type EvalCaseDefinition,
+  type EvalCaseInput,
   EvalCaseSchema,
   type FixturesRef,
   FixturesRefSchema,
   type PersistedCase,
   PersistedCaseSchema,
 } from "./core/case.js";
+export type { AssertionDefaults, EvalCaseDefaults } from "./core/case-defaults.js";
 export {
   CONVERSATION_TURN_REQUEST,
   type ConversationTurnRequest,
@@ -221,14 +252,44 @@ export {
   type AgentDriver,
   type Artifact,
   ArtifactSchema,
+  type DriverCapability,
+  DriverCapabilitySchema,
   type DriverEventMap,
+  type DriverExperiment,
   type DriverPlugin,
+  type DriverRuntime,
+  defineDriver,
+  driverRef,
+  type HarnessSelection,
   type ResolvedFixtures,
   ResolvedFixturesSchema,
   type StartContext,
+  selectModelAndReasoningEffort,
 } from "./core/driver.js";
+export {
+  type ContainerExecution,
+  ContainerExecutionSchema,
+  type DriverExecution,
+  DriverExecutionSchema,
+} from "./core/driver-execution.js";
+export {
+  type DriverMetric,
+  type DriverMetricDescriptor,
+  DriverMetricDescriptorSchema,
+  DriverMetricSchema,
+  type DriverMetrics,
+  DriverMetricsSchema,
+  type DriverMetricUnit,
+  DriverMetricUnitSchema,
+  normalizeDriverMetrics,
+  type ReportedDriverMetrics,
+  ReportedDriverMetricsSchema,
+} from "./core/driver-metrics.js";
 export { type EventHandler, TypedEmitter } from "./core/emitter.js";
+export { type EnvironmentDeclaration, EnvironmentDeclarationSchema } from "./core/environment.js";
 export {
+  type AgentAttribution,
+  AgentAttributionSchema,
   type Answer,
   AnswerSchema,
   ApproveAnswerSchema,
@@ -292,6 +353,40 @@ export {
   type Optimizer,
   type OptimizerInput,
 } from "./core/experiment.js";
+export { type GlobalIdParts, tryParseGlobalId } from "./core/global-id.js";
+export {
+  type CatalogModels,
+  catalogEfforts,
+  defaultCatalogEffort,
+  defineHarnessCatalog,
+  HARNESS_EFFORTS,
+  type HarnessCatalog,
+  type ReasoningEffort,
+  ReasoningEffortSchema,
+} from "./core/harness-catalog.js";
+export {
+  antigravityCatalog,
+  type BuiltInHarnessDriver,
+  builtInHarnessCatalogs,
+  claudeCodeCatalog,
+  codexCatalog,
+  copilotCatalog,
+  cursorCatalog,
+  grokBuildCatalog,
+  harnessCatalogFor,
+} from "./core/harness-catalogs/index.js";
+export {
+  type BuiltInHarnessMatrixEntry,
+  BuiltInHarnessMatrixEntrySchema,
+  type CustomHarnessMatrixEntry,
+  expandHarnessMatrix,
+  type HarnessMatrix,
+  type HarnessMatrixEntry,
+  HarnessMatrixEntrySchema,
+  HarnessMatrixSchema,
+  type HarnessPermutation,
+  permutationId,
+} from "./core/harness-matrix.js";
 export {
   defineFixtures,
   type LifecycleFixture,
@@ -327,14 +422,11 @@ export {
 export {
   DEFAULT_CLAUDE_MODEL,
   DEFAULT_CODEX_MODEL,
-  DEFAULT_CODEX_REASONING_EFFORT,
   defaultHarnessModel,
   type HarnessModelConfig,
   HarnessModelConfigSchema,
   type ModelCliProvider,
   ModelCliProviderSchema,
-  type ReasoningEffort,
-  ReasoningEffortSchema,
 } from "./core/model-cli.js";
 export {
   type AssertionProgress,
@@ -420,6 +512,11 @@ export {
   type WireEvent,
   WireEventSchema,
 } from "./core/wire.js";
+export {
+  type AntigravityDriverConfig,
+  AntigravityDriverConfigSchema,
+  antigravityDriver,
+} from "./drivers/antigravity/index.js";
 export {
   type ClaudeCodeDriverConfig,
   ClaudeCodeDriverConfigSchema,
@@ -430,6 +527,22 @@ export {
   CodexDriverConfigSchema,
   codexDriver,
 } from "./drivers/codex/index.js";
+export {
+  type CopilotDriverConfig,
+  CopilotDriverConfigSchema,
+  copilotDriver,
+} from "./drivers/copilot/index.js";
+export {
+  type CursorDriverConfig,
+  CursorDriverConfigSchema,
+  cursorDriver,
+} from "./drivers/cursor/index.js";
+export {
+  buildGrokBuildArgs,
+  type GrokBuildDriverConfig,
+  GrokBuildDriverConfigSchema,
+  grokBuildDriver,
+} from "./drivers/grok-build/index.js";
 export {
   type McpProbeDriverConfig,
   McpProbeDriverConfigSchema,
@@ -441,13 +554,11 @@ export {
   SubprocessDriverConfigSchema,
   subprocessDriver,
 } from "./drivers/subprocess/index.js";
-
 export {
   createReflectiveOptimizer,
   type ReflectiveOptimizerConfig,
   ReflectiveOptimizerConfigSchema,
 } from "./optimization/reflective.js";
-
 // Supported programmatic orchestration.
 export {
   applyCandidate,

package/src/optimization/reflective.ts

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { DurationSchema, durationMs } from "../core/duration.js";
 import {
   type CandidateEvaluation,
   type CandidateProposal,
@@ -16,7 +17,7 @@ export const ReflectiveOptimizerConfigSchema = z
     model: z.string().optional(),
     command: z.string().optional(),
     reasoningEffort: ReasoningEffortSchema.optional(),
-    timeoutMs: z.number().int().positive().default(180_000),
+    timeout: DurationSchema.default("3 minutes"),
     maxComponentChars: z.number().int().positive().default(80_000),
     maxAlternativeChars: z.number().int().positive().default(40_000),
     maxFeedbackChars: z.number().int().positive().default(50_000),
@@ -273,7 +274,7 @@ const reflectiveOptimizer = (config: ReflectiveOptimizerConfig): Optimizer => ({
         model,
         command: config.command,
         reasoningEffort: config.reasoningEffort,
-        timeoutMs: config.timeoutMs,
+        timeoutMs: durationMs(config.timeout),
         system: systemPrompt(input.campaign.evaluation.kind),
         signal: input.abortSignal,
         prompt: builtPrompt.prompt,

package/src/orchestrator/annotation-loader.ts

@@ -2,6 +2,7 @@ import { readFile } from "node:fs/promises";
 import { dirname, extname, resolve, sep } from "node:path";
 import { pathToFileURL } from "node:url";
 import { z } from "zod";
+import { assertDisjointClusters } from "../core/evaluation-clusters.js";
 import {
   type ExperimentSelector,
   formatZodIssues,
@@ -73,6 +74,15 @@ export const selectGraderAnnotations = (
 export const assertDisjointAnnotationSplits = (
   splits: Array<{ name: string; annotations: readonly GraderAnnotation[] }>,
 ): void => {
+  assertDisjointClusters(
+    splits.map((split) => ({
+      name: split.name,
+      items: split.annotations.map((annotation) => ({
+        id: annotation.id,
+        cluster: annotation.meta?.cluster,
+      })),
+    })),
+  );
   const idOwners = new Map<string, string>();
   const runOwners = new Map<string, string>();
   for (const split of splits) {

package/src/orchestrator/annotation-store.ts

@@ -82,11 +82,10 @@ const acquireAnnotationLock = async (path: string): Promise<() => Promise<void>>
   }
 };
 
-export const upsertGraderAnnotation = async (
-  filePath: string,
+const updateAnnotation = async (
+  path: string,
   annotation: GraderAnnotation,
 ): Promise<GraderAnnotationDataset> => {
-  const path = resolve(filePath);
   await mkdir(dirname(path), { recursive: true });
   const release = await acquireAnnotationLock(path);
   const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
@@ -114,3 +113,25 @@ export const upsertGraderAnnotation = async (
     await release();
   }
 };
+
+// Serialize local writers before acquiring the cross-process lock. Otherwise
+// a burst of updates spends its lock deadline contending with its own process.
+const pendingUpdates = new Map<string, Promise<void>>();
+
+export const upsertGraderAnnotation = (
+  filePath: string,
+  annotation: GraderAnnotation,
+): Promise<GraderAnnotationDataset> => {
+  const path = resolve(filePath);
+  const previous = pendingUpdates.get(path) ?? Promise.resolve();
+  const update = previous.then(() => updateAnnotation(path, annotation));
+  const settled = update.then(
+    () => undefined,
+    () => undefined,
+  );
+  pendingUpdates.set(path, settled);
+  void settled.then(() => {
+    if (pendingUpdates.get(path) === settled) pendingUpdates.delete(path);
+  });
+  return update;
+};

package/src/orchestrator/artifact-origin.ts

@@ -0,0 +1,50 @@
+import { isAbsolute, relative, resolve, sep } from "node:path";
+import {
+  type Artifact,
+  contentHash,
+  inspectRunArtifact,
+  readRunArtifactBytes,
+} from "../core/index.js";
+
+const MAX_ORIGIN_BYTES = 10 * 1024 * 1024;
+
+export const identifyArtifactOrigins = async (
+  runDir: string,
+  artifacts: readonly Artifact[],
+  fixtureHashes: ReadonlyMap<string, string>,
+): Promise<Artifact[]> => {
+  const run = { runDir, artifacts };
+  const result: Artifact[] = [];
+  for (const artifact of artifacts) {
+    const { origin: _claimedOrigin, ...captured } = artifact;
+    const inspected = await inspectRunArtifact(run, artifact.path);
+    if (!inspected.stats?.isFile()) {
+      result.push(captured);
+      continue;
+    }
+    const path = relative(resolve(runDir, "artifacts"), inspected.target);
+    const expected =
+      !isAbsolute(path) && path !== ".." && !path.startsWith(`..${sep}`)
+        ? fixtureHashes.get(path)
+        : undefined;
+    if (expected === undefined) {
+      result.push({ ...captured, origin: "generated" });
+      continue;
+    }
+    if (inspected.stats.size > MAX_ORIGIN_BYTES) {
+      result.push(captured);
+      continue;
+    }
+    try {
+      const bytes = await readRunArtifactBytes(run, artifact.path, { maxBytes: MAX_ORIGIN_BYTES });
+      result.push({
+        ...captured,
+        origin: contentHash(bytes) === expected ? "fixture" : "modified",
+      });
+    } catch {
+      // Advisory provenance must not discard the rest of a safely archived index.
+      result.push(captured);
+    }
+  }
+  return result;
+};

package/src/orchestrator/campaign-lifecycle.ts

@@ -16,6 +16,7 @@ type CampaignLifecycleContext = {
   work: unknown;
   result: unknown;
   error: unknown;
+  failed: boolean;
   terminalStatus: "passed" | "failed" | "aborted" | undefined;
 };
 type CampaignLifecycleOutcome =
@@ -57,6 +58,7 @@ const campaignLifecycleMachine = setup({
         "output" in event ? context.effects.resultStatus(event.output) : "failed",
     }),
     captureError: assign({
+      failed: true,
       error: ({ event }) =>
         "error" in event ? event.error : new Error("campaign lifecycle failed"),
       terminalStatus: ({ context, event }) =>
@@ -65,6 +67,7 @@ const campaignLifecycleMachine = setup({
         ),
     }),
     captureReleaseError: assign({
+      failed: true,
       error: ({ event }) => ("error" in event ? event.error : new Error("campaign release failed")),
       terminalStatus: "failed",
     }),
@@ -76,6 +79,7 @@ const campaignLifecycleMachine = setup({
     work: undefined,
     result: undefined,
     error: undefined,
+    failed: false,
     terminalStatus: undefined,
   }),
   initial: "preparing",
@@ -145,7 +149,7 @@ const campaignLifecycleMachine = setup({
     failed: { type: "final" },
   },
   output: ({ context }) => {
-    if (context.error !== undefined) {
+    if (context.failed) {
       return {
         status: context.terminalStatus === "aborted" ? "aborted" : "failed",
         error: context.error,

package/src/orchestrator/candidates.ts

@@ -5,6 +5,7 @@ import { basename, dirname, isAbsolute, join, normalize, relative, resolve, sep
 import { temporaryDirectory } from "tempy";
 import { glob } from "tinyglobby";
 import { z } from "zod";
+import { durationMs } from "../core/duration.js";
 import { exec } from "../core/exec.js";
 import {
   type Candidate,
@@ -226,7 +227,7 @@ export const validateCandidate = async (
       try {
         return await exec(validation.command, validation.args, {
           cwd: materializedRoot,
-          timeoutMs: validation.timeoutMs,
+          timeoutMs: durationMs(validation.timeout),
           env,
           ...(signal ? { signal } : {}),
         });
@@ -236,7 +237,7 @@ export const validateCandidate = async (
     })().finally(() => rm(validationHome, { recursive: true, force: true }));
     if (result.timedOut || result.exitCode !== 0) {
       const reason = result.timedOut
-        ? `timed out after ${validation.timeoutMs}ms`
+        ? `timed out after ${durationMs(validation.timeout)}ms`
         : `exited ${result.exitCode}`;
       throw new Error(
         `candidate validation '${validation.command}' ${reason}: ${result.stderr.slice(0, 500)}`,