@effect-agent/testing 0.1.0-beta.8 → 0.1.0-beta.81

package/src/{code-executor-conformance.ts → CodeExecutorConformance.ts}

@@ -1,19 +1,21 @@
 import {
-  CodeExecutionError,
   CodeExecutionHost,
   CodeExecutionLimits,
   CodeExecutionNamespace,
   CodeExecutionRequest,
-  CodeExecutionResult,
   CodeExecutor,
-  CodeHostCall,
   CodeHostCallFailure,
-  CodeHostCallResult,
   CodeHostCallSuccess,
+  type CodeExecutionError,
+  type CodeExecutionResult,
+  type CodeHostCall,
+  type CodeHostCallResult,
+} from "@effect-agent/sandbox/CodeExecutor";
+import {
   NetworkAllowlist,
   NetworkDisabled,
-  SandboxImplementation,
-} from "@effect-agent/sandbox";
+  type SandboxImplementation,
+} from "@effect-agent/sandbox/Sandbox";
 import { Deferred, Duration, Effect, Fiber, Schema } from "effect";
 
 /**
@@ -88,12 +90,14 @@ const respondingHost = (
   respond: (call: CodeHostCall) => CodeHostCallResult,
 ): { readonly host: CodeExecutionHost["Service"]; readonly calls: Array<CodeHostCall> } => {
   const calls: Array<CodeHostCall> = [];
+
   return {
     calls,
     host: {
       call: (call) =>
         Effect.sync(() => {
           calls.push(call);
+
           return respond(call);
         }),
     },
@@ -106,6 +110,7 @@ const runPass = (
 ): Effect.Effect<CodeExecutionResult, CodeExecutionError, CodeExecutor> =>
   Effect.gen(function* () {
     const executor = yield* CodeExecutor;
+
     return yield* executor
       .execute(request)
       .pipe(Effect.provideService(CodeExecutionHost, CodeExecutionHost.of(host)));
@@ -134,6 +139,7 @@ const expectSuccess = (
     ),
     Effect.flatMap((result) => {
       const complaint = check(result);
+
       return complaint === undefined ? Effect.void : Effect.fail(violation(caseName, complaint));
     }),
   );
@@ -157,6 +163,7 @@ const expectFailure = (
         );
       }
       const complaint = check?.(error);
+
       return complaint === undefined ? Effect.void : Effect.fail(violation(caseName, complaint));
     }),
   );
@@ -165,6 +172,7 @@ export const codeExecutorConformanceCases = (
   options: CodeExecutorConformanceOptions,
 ): ReadonlyArray<CodeExecutorConformanceCase> => {
   const posture = options.implementation;
+
   return [
     {
       name: "TEST-015 executes bounded JSON computation and returns the program value",
@@ -184,9 +192,11 @@ export const codeExecutorConformanceCases = (
       name: "CAP-015 reports its isolation posture honestly in results and errors",
       run: Effect.gen(function* () {
         const caseName = "CAP-015 reports its isolation posture honestly in results and errors";
+
         const result = yield* runPass(makeRequest("async () => 1"), unusedHost).pipe(
           Effect.mapError((error) => violation(caseName, `expected success, got ${error._tag}`)),
         );
+
         if (
           result.implementation.isolation !== posture.isolation ||
           result.implementation.identity !== posture.identity
@@ -196,10 +206,12 @@ export const codeExecutorConformanceCases = (
             `result posture ${preview(result.implementation)} does not match the declared ${preview(posture)}`,
           );
         }
+
         const error = yield* runPass(makeRequest("async () => {"), unusedHost).pipe(
           Effect.flip,
           Effect.mapError(() => violation(caseName, "expected the invalid-source pass to fail")),
         );
+
         // Every expected execution failure carries the posture; an adapter
         // omitting the field must fail this case, not slip past a probe.
         if (
@@ -219,11 +231,13 @@ export const codeExecutorConformanceCases = (
       run: Effect.gen(function* () {
         const caseName =
           "TEST-015 routes host calls through the CodeExecutionHost in program order";
+
         const { host, calls } = respondingHost((call) =>
           call.method === "query"
             ? CodeHostCallSuccess.make({ value: { rows: [1, 2, 3] } })
             : CodeHostCallSuccess.make({ value: 3 }),
         );
+
         const result = yield* runPass(
           makeRequest(
             "async () => { const q = await warehouse.query({ sql: 'select' }); const c = await warehouse.count({ table: 't' }); return { rows: q.rows, count: c }; }",
@@ -235,10 +249,12 @@ export const codeExecutorConformanceCases = (
             violation(caseName, `expected success, got ${error._tag}: ${preview(error)}`),
           ),
         );
+
         if (JSON.stringify(result.value) !== JSON.stringify({ rows: [1, 2, 3], count: 3 })) {
           return yield* violation(caseName, `unexpected value ${preview(result.value)}`);
         }
         const observed = calls.map((call) => `${call.namespace}.${call.method}`);
+
         if (JSON.stringify(observed) !== JSON.stringify(["warehouse.query", "warehouse.count"])) {
           return yield* violation(caseName, `unexpected host call order ${preview(observed)}`);
         }
@@ -376,6 +392,7 @@ export const codeExecutorConformanceCases = (
       run: Effect.gen(function* () {
         const caseName = "TEST-015 fails typed when host calls exceed the executor cap";
         const { host, calls } = respondingHost(() => CodeHostCallSuccess.make({ value: null }));
+
         yield* expectFailure(
           caseName,
           makeRequest(
@@ -404,6 +421,8 @@ export const codeExecutorConformanceCases = (
         "TEST-015 fails typed on a host outcome outside the protocol schema",
         makeRequest("async () => warehouse.query({})", { namespaces: [warehouseNamespace] }),
         {
+          // Deliberately violate the compile-time host contract to verify that an adapter
+          // independently decodes the runtime protocol boundary.
           call: () => Effect.succeed({ bogus: true } as unknown as CodeHostCallResult),
         },
         "CodeExecutionProtocolError",
@@ -460,6 +479,7 @@ export const codeExecutorConformanceCases = (
         const caseName = "TEST-015 interruption reaches in-flight host calls and pass teardown";
         const started = yield* Deferred.make<void>();
         const witness = { hostCallInterrupted: false };
+
         const host: CodeExecutionHost["Service"] = {
           call: () =>
             Deferred.succeed(started, undefined).pipe(
@@ -471,16 +491,19 @@ export const codeExecutorConformanceCases = (
               ),
             ),
         };
+
         const fiber = yield* runPass(
           makeRequest("async () => warehouse.query({})", { namespaces: [warehouseNamespace] }),
           host,
         ).pipe(Effect.forkChild);
+
         // Guard against a broken adapter that settles the pass without ever
         // reaching the host: the case must report a violation, not hang.
         const winner = yield* Effect.raceFirst(
           Deferred.await(started).pipe(Effect.as("started" as const)),
           Fiber.join(fiber).pipe(Effect.exit, Effect.as("exited" as const)),
         );
+
         if (winner === "exited") {
           return yield* violation(
             caseName,

package/src/{code-executor-substitute.ts → CodeExecutorSubstitute.ts}

@@ -1,9 +1,6 @@
 import {
   CodeExecutionHost,
-  CodeExecutionLimits,
-  CodeExecutionNamespace,
   CodeExecutionProtocolError,
-  CodeExecutionRequest,
   CodeExecutionResourceUse,
   CodeExecutionResult,
   CodeExecutionTimeoutError,
@@ -16,8 +13,11 @@ import {
   CodeOutputLimitError,
   CodeProgramFailedError,
   CodeSourceError,
-  SandboxImplementation,
-} from "@effect-agent/sandbox";
+  type CodeExecutionLimits,
+  type CodeExecutionNamespace,
+  type CodeExecutionRequest,
+} from "@effect-agent/sandbox/CodeExecutor";
+import { SandboxImplementation } from "@effect-agent/sandbox/Sandbox";
 import { Clock, Duration, Effect, Fiber, Layer, Option, Queue, Schema } from "effect";
 
 /**
@@ -91,6 +91,7 @@ const formatLogValue = (value: unknown): string => {
     if (typeof value === "string") {
       return value;
     }
+
     return JSON.stringify(value) ?? String(value);
   } catch {
     try {
@@ -104,17 +105,21 @@ const formatLogValue = (value: unknown): string => {
 const makeConsole = (capture: LogCapture, limits: CodeExecutionLimits) => {
   const write = (...values: ReadonlyArray<unknown>): void => {
     const joined = values.map(formatLogValue).join(" ");
+
     const line =
       joined.length > MAX_LOG_LINE_CHARACTERS
         ? `${joined.slice(0, MAX_LOG_LINE_CHARACTERS - 1)}…`
         : joined;
+
     const bytes = utf8ByteLength(line);
+
     if (capture.lines.length >= MAX_LOG_LINES || capture.bytes + bytes > limits.maxLogBytes) {
       throw new LogLimitSignal(capture.bytes + bytes);
     }
     capture.lines.push(line);
     capture.bytes += bytes;
   };
+
   return { debug: write, error: write, info: write, log: write, warn: write };
 };
 
@@ -131,18 +136,21 @@ const buildNamespaceObject = (
   offer: (pending: PendingHostCall) => void,
 ): Record<string, unknown> => {
   const methods: Record<string, unknown> = {};
+
   for (const method of namespace.methods) {
     methods[method] = (argument: unknown) =>
       new Promise((resolve, reject) => {
         offer({ namespace: namespace.name, method, argument, resolve, reject });
       });
   }
+
   return methods;
 };
 
 const boundedText = (value: unknown): string => {
   try {
     const text = value instanceof Error ? `${value.name}: ${value.message}` : formatLogValue(value);
+
     return text.slice(0, MAX_THROWN_CHARACTERS);
   } catch {
     return "[unserializable thrown value]";
@@ -158,24 +166,44 @@ const safeDecodeJson = (value: unknown): Option.Option<Schema.Json> => {
   }
 };
 
+const decodeJsonText = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Json));
+
+/** Own serialized result text before bounding and decoding its detached JSON snapshot. */
+const serializeJson = (value: unknown): Option.Option<string> => {
+  const decoded = safeDecodeJson(value);
+
+  if (Option.isNone(decoded)) return Option.none();
+  try {
+    return Option.fromUndefinedOr(JSON.stringify(decoded.value));
+  } catch {
+    return Option.none();
+  }
+};
+
 const boundedThrown = (value: unknown): Schema.Json => {
   const decoded = safeDecodeJson(value);
+
   if (Option.isSome(decoded)) {
     try {
       const encoded = JSON.stringify(decoded.value);
+
       if (encoded !== undefined && encoded.length <= MAX_THROWN_CHARACTERS) {
-        return decoded.value;
+        const snapshot = decodeJsonText(encoded);
+
+        if (Option.isSome(snapshot)) return snapshot.value;
       }
     } catch {
       // fall through to the bounded string form
     }
   }
+
   return boundedText(value);
 };
 
 const encodedJsonByteLength = (value: Schema.Json): number | undefined => {
   try {
     const encoded = JSON.stringify(value);
+
     return encoded === undefined ? undefined : utf8ByteLength(encoded);
   } catch {
     return undefined;
@@ -213,6 +241,7 @@ const validateRequest = (
     }
     const reservedNames = new Set<string>([...shadowedGlobals, "console"]);
     const seen = new Set<string>();
+
     for (const namespace of request.namespaces) {
       if (reservedNames.has(namespace.name) || seen.has(namespace.name)) {
         return yield* CodeExecutorUnsupportedError.make({
@@ -224,6 +253,7 @@ const validateRequest = (
       seen.add(namespace.name);
     }
     const sourceBytes = utf8ByteLength(request.source);
+
     if (sourceBytes > request.limits.maxSourceBytes) {
       return yield* CodeSourceError.make({
         implementation: inProcessCodeExecutorImplementation,
@@ -246,6 +276,7 @@ const serveHostCalls = (
   Effect.gen(function* () {
     while (true) {
       const pending = yield* Queue.take(queue);
+
       counter.calls += 1;
       if (counter.calls > limits.maxHostCalls) {
         return yield* CodeHostCallLimitError.make({
@@ -255,11 +286,13 @@ const serveHostCalls = (
         });
       }
       const argument = safeDecodeJson(pending.argument);
+
       if (Option.isNone(argument)) {
         pending.reject(new TypeError("host call arguments must be JSON values"));
         continue;
       }
       const argumentBytes = encodedJsonByteLength(argument.value);
+
       if (argumentBytes === undefined || argumentBytes > limits.maxHostCallArgumentBytes) {
         return yield* CodeOutputLimitError.make({
           implementation: inProcessCodeExecutorImplementation,
@@ -269,6 +302,7 @@ const serveHostCalls = (
           logs: [...capture.lines],
         });
       }
+
       const rawOutcome = yield* host.call(
         CodeHostCall.make({
           namespace: pending.namespace,
@@ -276,7 +310,9 @@ const serveHostCalls = (
           argument: argument.value,
         }),
       );
+
       const outcome = decodeHostOutcome(rawOutcome);
+
       if (Option.isNone(outcome)) {
         return yield* CodeExecutionProtocolError.make({
           implementation: inProcessCodeExecutorImplementation,
@@ -288,6 +324,7 @@ const serveHostCalls = (
         continue;
       }
       const resultBytes = encodedJsonByteLength(outcome.value.value);
+
       if (resultBytes === undefined || resultBytes > limits.maxHostCallResultBytes) {
         return yield* CodeOutputLimitError.make({
           implementation: inProcessCodeExecutorImplementation,
@@ -306,27 +343,37 @@ const classifyProgramFailure = (
   limits: CodeExecutionLimits,
   capture: LogCapture,
 ): CodeOutputLimitError | CodeSourceError | CodeProgramFailedError => {
-  const inner = thrown instanceof EvaluationThrew ? thrown.inner : thrown;
-  if (inner instanceof LogLimitSignal) {
-    return CodeOutputLimitError.make({
-      implementation: inProcessCodeExecutorImplementation,
-      surface: "logs",
-      limit: limits.maxLogBytes,
-      observed: inner.observed,
-      logs: [...capture.lines],
-    });
-  }
-  if (inner instanceof NotAFunction) {
-    return CodeSourceError.make({
-      implementation: inProcessCodeExecutorImplementation,
-      reason: "not-a-function",
-      message: `The source expression evaluated to ${inner.actual}; it must evaluate to one async function`,
-    });
+  let inner = thrown;
+  let reason: "threw" | "rejected" = "rejected";
+
+  try {
+    const evaluationThrew = thrown instanceof EvaluationThrew;
+
+    inner = evaluationThrew ? thrown.inner : thrown;
+    if (inner instanceof LogLimitSignal) {
+      return CodeOutputLimitError.make({
+        implementation: inProcessCodeExecutorImplementation,
+        surface: "logs",
+        limit: limits.maxLogBytes,
+        observed: inner.observed,
+        logs: [...capture.lines],
+      });
+    }
+    if (inner instanceof NotAFunction) {
+      return CodeSourceError.make({
+        implementation: inProcessCodeExecutorImplementation,
+        reason: "not-a-function",
+        message: `The source expression evaluated to ${inner.actual}; it must evaluate to one async function`,
+      });
+    }
+    // Async body throws become rejections: exception-like values read as `threw`,
+    // while plain values such as uncaught host failure envelopes read as `rejected`.
+    reason = evaluationThrew || inner instanceof Error ? "threw" : "rejected";
+  } catch {
+    // A program-owned Proxy can throw from instanceof's prototype lookup. Keep
+    // that failure inside the same guarded diagnostic boundary as its value.
   }
-  // An async function converts a body-level `throw` into a rejection, so the
-  // split is by value shape: exception-like values read as `threw`, plain
-  // rejection values (an uncaught host failure envelope) read as `rejected`.
-  const reason = thrown instanceof EvaluationThrew || inner instanceof Error ? "threw" : "rejected";
+
   return CodeProgramFailedError.make({
     implementation: inProcessCodeExecutorImplementation,
     reason,
@@ -346,6 +393,9 @@ const executeInProcess: CodeExecutorExecute = Effect.fn("InProcessCodeExecutor.e
 
     const factory = yield* Effect.try({
       try: () =>
+        // This substitute intentionally evaluates authored test programs in-process and reports
+        // an `unisolated` posture. Real adapters own the security boundary and never use this path.
+        // oxlint-disable-next-line typescript/no-implied-eval
         new Function(
           ...shadowedGlobals,
           "console",
@@ -367,24 +417,32 @@ const executeInProcess: CodeExecutorExecute = Effect.fn("InProcessCodeExecutor.e
     // rejected synchronously, so the queue stays bounded against hostile
     // programs.
     let issuedHostCalls = 0;
+
     const namespaceObjects = request.namespaces.map((namespace) =>
       buildNamespaceObject(namespace, (pending) => {
         issuedHostCalls += 1;
         if (issuedHostCalls > request.limits.maxHostCalls + 1) {
           pending.reject(new Error(`host-call limit of ${request.limits.maxHostCalls} exceeded`));
+
           return;
         }
         Queue.offerUnsafe(queue, pending);
       }),
     );
 
-    const server = yield* serveHostCalls(host, queue, request.limits, capture, counter).pipe(
-      Effect.forkScoped,
-    );
+    const concurrency = request.limits.maxHostCallConcurrency ?? 4;
+
+    const server = yield* Effect.all(
+      Array.from({ length: concurrency }, () =>
+        serveHostCalls(host, queue, request.limits, capture, counter),
+      ),
+      { concurrency, discard: true },
+    ).pipe(Effect.andThen(Effect.never), Effect.forkScoped);
 
     const program = Effect.tryPromise({
       try: async () => {
         let candidate: unknown;
+
         try {
           candidate = factory(
             ...shadowedGlobals.map(() => undefined),
@@ -398,17 +456,20 @@ const executeInProcess: CodeExecutorExecute = Effect.fn("InProcessCodeExecutor.e
           throw new EvaluationThrew(new NotAFunction(typeof candidate));
         }
         let outcome: unknown;
+
         try {
-          outcome = (candidate as () => unknown)();
+          outcome = candidate();
         } catch (cause) {
           throw new EvaluationThrew(cause);
         }
+
         return await Promise.resolve(outcome);
       },
       catch: (thrown) => classifyProgramFailure(thrown, request.limits, capture),
     });
 
     const startedAt = yield* Clock.currentTimeMillis;
+
     // The wall-clock deadline interrupts only at asynchronous suspension
     // points: a synchronous runaway shares the host thread and cannot be
     // stopped in-process — exactly why the platform CPU enforcement cases
@@ -428,6 +489,7 @@ const executeInProcess: CodeExecutorExecute = Effect.fn("InProcessCodeExecutor.e
       }),
       Effect.ensuring(Fiber.interrupt(server)),
     );
+
     const finishedAt = yield* Clock.currentTimeMillis;
 
     // An unawaited burst can outrun the server: the program may return before
@@ -442,31 +504,36 @@ const executeInProcess: CodeExecutorExecute = Effect.fn("InProcessCodeExecutor.e
       });
     }
 
-    const value = yield* Schema.decodeUnknownEffect(Schema.Json)(returned).pipe(
-      Effect.mapError(() =>
-        CodeProgramFailedError.make({
-          implementation: inProcessCodeExecutorImplementation,
-          reason: "non-json-result",
-          thrown: null,
-          message: "The program must return a JSON value",
-          logs: [...capture.lines],
-        }),
-      ),
-    );
-    const resultBytes = encodedJsonByteLength(value);
-    if (resultBytes === undefined || resultBytes > request.limits.maxResultBytes) {
+    const nonJsonResult = () =>
+      CodeProgramFailedError.make({
+        implementation: inProcessCodeExecutorImplementation,
+        reason: "non-json-result",
+        thrown: null,
+        message: "The program must return a JSON value",
+        logs: [...capture.lines],
+      });
+
+    const encoded = serializeJson(returned);
+
+    if (Option.isNone(encoded)) return yield* nonJsonResult();
+    const resultBytes = utf8ByteLength(encoded.value);
+
+    if (resultBytes > request.limits.maxResultBytes) {
       return yield* CodeOutputLimitError.make({
         implementation: inProcessCodeExecutorImplementation,
         surface: "result",
         limit: request.limits.maxResultBytes,
-        observed: resultBytes ?? 0,
+        observed: resultBytes,
         logs: [...capture.lines],
       });
     }
+    const decoded = decodeJsonText(encoded.value);
+
+    if (Option.isNone(decoded)) return yield* nonJsonResult();
 
     return CodeExecutionResult.make({
       implementation: inProcessCodeExecutorImplementation,
-      value,
+      value: decoded.value,
       logs: [...capture.lines],
       resourceUse: CodeExecutionResourceUse.make({
         wallTime: Duration.millis(Math.max(0, finishedAt - startedAt)),

package/src/{fixtures/docs-researcher/index.ts → DocsResearcher.ts}

@@ -1,13 +1,15 @@
+/** Shared Docs Researcher definitions and MCP delegation fixtures. */
+
 // ---------------------------------------------------------------------------
 // Docs Researcher fixture (P7 internal agent #3): S2 durable delegation on DN
-// with MCP-discovered content tools. See definition.ts / mcp.ts / harness.ts.
+// with MCP-discovered content tools. See fixtures/docs-researcher/{definition,mcp,harness}.ts.
 //
 // Authoring friction note (WP7 input; real observations from writing this
 // agent):
 //
 // 1. MCP discovery and Agent authoring do not meet in the type system. The
 //    connector's validated Toolkit arrives as `Toolkit.Any`, while
-//    `Agent.define` needs the statically typed toolkit — so the binding
+//    `Agent.make` needs the statically typed toolkit — so the binding
 //    between "what discovery served" and "what the child was authored
 //    against" had to be re-proved by hand (`assertDiscoveryMatchesAuthoredToolkit`
 //    re-deriving `Tool.getJsonSchema` on both sides). A framework helper that
@@ -29,6 +31,67 @@
 //    exactly where a reviewer looks for them.
 // ---------------------------------------------------------------------------
 
-export * from "./definition.ts";
-export * from "./harness.ts";
-export * from "./mcp.ts";
+export {
+  ResearchDocumentId,
+  BoundedSummary,
+  DocumentQuery,
+  ResearchDocument,
+  DocumentUnavailable,
+  DocumentLibrary,
+  FetchDocument,
+  DocContentToolkit,
+  docContentToolkitLayer,
+  docsDocumentBodySecret,
+  researchCorpusDocumentIds,
+  researchDocumentLookup,
+  researchDocumentFor,
+  documentBodyPhrase,
+  SummaryBrief,
+  DocumentSummary,
+  documentSummaryFor,
+  encodedDocumentSummary,
+  DocSummarizer,
+  SummaryRequest,
+  SummaryFinding,
+  DocumentSummaryFailed,
+  documentSummaryPolicy,
+  delegateDocumentSummary,
+  mapSummaryChildFailure,
+  docsSummarizerDigestStrings,
+  docsSummaryHandlersLayer,
+  ResearchRequest,
+  ResearchDigest,
+  docsCoordinatorConfidentialMarker,
+  docsMissionConfidentialMarker,
+  DocsResearcherToolkit,
+  DocsResearcher,
+  researchMissionRequest,
+  expectedResearchDigest,
+} from "./fixtures/docs-researcher/definition.ts";
+
+export {
+  docsResearcherDeploymentId,
+  docsResearcherProducerId,
+  docsResearcherPrincipal,
+  docsCoordinatorDigests,
+  docsSummarizerDigests,
+  docsResearcherSubmitOptions,
+  docsResearcherSubmitAgent,
+  summarizeCallId,
+  fetchCallId,
+  type DocsResearcherHarnessOptions,
+  type DocsResearcherHarness,
+  makeDocsResearcherHarness,
+  redactedDocumentPreview,
+  type DocsResearcherHarnessRequirements,
+} from "./fixtures/docs-researcher/harness.ts";
+
+export {
+  docsMcpRequest,
+  docsMcpIdentity,
+  docsMcpConnectorLayer,
+  docsMcpOversizedConnectorLayer,
+  docsMcpMismatchedConnectorLayer,
+  assertDiscoveryMatchesAuthoredToolkit,
+  DocsMcpDiscoveryEvidence,
+} from "./fixtures/docs-researcher/mcp.ts";