@hassangameryt/freeflow 0.4.0 → 0.6.0

package/skills/evaluate-skill/scripts/lib/grade-interaction.mjs

@@ -1,4 +1,5 @@
-import { realpath } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import { readFile, realpath } from "node:fs/promises";
 import path from "node:path";
 
 import {
@@ -16,6 +17,10 @@ import {
 } from "./grade-support.mjs";
 
 const ACTIVATION_EXPECTATIONS = new Set(["never", "on-turn", "by-turn", "not-before-turn"]);
+const TOOL_CALL_EXPECTATIONS = new Set(["called", "succeeded", "failed", "not-called"]);
+const CONTEXT_TEXT_EXPECTATIONS = new Set(["contains", "not-contains", "equals"]);
+const CONTEXT_SURFACES = new Set(["system-prompt", "provider-context"]);
+const CONTEXT_REQUEST_SELECTORS = new Set(["first", "last"]);
 
 export async function gradeSkillRead(group, runs, expectation) {
   const turn = expectation.turn ?? null;
@@ -100,6 +105,95 @@ export async function gradeResourceRead(group, runs, expectation) {
   );
 }
 
+export async function gradeContextText(group, runs, expectation) {
+  const turn = expectation.turn ?? null;
+  const surface = expectation.surface ?? "system-prompt";
+  const request = expectation.request ?? "last";
+  if (
+    !validBodyExpectation(group, expectation, turn) ||
+    !CONTEXT_SURFACES.has(surface) ||
+    (!CONTEXT_REQUEST_SELECTORS.has(request) && !Number.isInteger(request)) ||
+    (Number.isInteger(request) && request < 1) ||
+    !CONTEXT_TEXT_EXPECTATIONS.has(expectation.expect) ||
+    typeof expectation.value !== "string"
+  ) {
+    return { error: "invalid context-text expectation" };
+  }
+
+  const expected = { expect: expectation.expect, value: expectation.value, surface, request, turn };
+  const run = runs[expectation.variant];
+  if (!run || run.state !== "complete") return unavailableCheck(expectation, expected);
+  const observed = await contextObservation(run, turn, surface, request);
+  if (observed === null) return unavailableCheck(expectation, expected);
+  const passed = textMatches(expectation.expect, expectation.value, observed.text);
+  return completedCheck(expectation, expected, compactContextObservation(observed, passed), passed);
+}
+
+export async function gradeToolCall(group, runs, expectation) {
+  const turn = expectation.turn ?? null;
+  const contains = expectation.argumentContains ?? [];
+  const notContains = expectation.argumentNotContains ?? [];
+  if (
+    !validBodyExpectation(group, expectation, turn) ||
+    typeof expectation.tool !== "string" ||
+    expectation.tool.trim() === "" ||
+    !TOOL_CALL_EXPECTATIONS.has(expectation.expect) ||
+    !Array.isArray(contains) ||
+    !contains.every((value) => typeof value === "string") ||
+    !Array.isArray(notContains) ||
+    !notContains.every((value) => typeof value === "string")
+  ) {
+    return { error: "invalid tool-call expectation" };
+  }
+
+  const expected = {
+    tool: expectation.tool,
+    expect: expectation.expect,
+    turn,
+    argumentContains: contains,
+    argumentNotContains: notContains,
+  };
+  const run = runs[expectation.variant];
+  if (!run || run.state !== "complete") return unavailableCheck(expectation, expected);
+
+  const calls = toolCalls(run, turn).filter((call) => call.toolName === expectation.tool);
+  const matchingCalls = calls.filter((call) => {
+    const argumentsText = JSON.stringify(call.args ?? null);
+    return (
+      contains.every((value) => argumentsText.includes(value)) &&
+      notContains.every((value) => !argumentsText.includes(value))
+    );
+  });
+  const succeededCalls = matchingCalls.filter((call) => call.completed === true && call.isError !== true);
+  const failedCalls = matchingCalls.filter((call) => call.completed === true && call.isError === true);
+  let passed;
+  if (expectation.expect === "called") {
+    passed = matchingCalls.length > 0;
+  } else if (expectation.expect === "succeeded") {
+    passed = succeededCalls.length > 0;
+  } else if (expectation.expect === "failed") {
+    passed = failedCalls.length > 0;
+  } else if (contains.length > 0 || notContains.length > 0) {
+    passed = matchingCalls.length === 0;
+  } else {
+    passed = calls.length === 0;
+  }
+  const observed = {
+    tool: expectation.tool,
+    turn,
+    callCount: calls.length,
+    matchingCallCount: matchingCalls.length,
+    succeededCount: succeededCalls.length,
+    failedCount: failedCalls.length,
+    calls: calls.map((call) => ({
+      args: call.args ?? null,
+      completed: call.completed ?? null,
+      isError: call.isError ?? null,
+    })),
+  };
+  return completedCheck(expectation, expected, observed, passed);
+}
+
 export async function gradeResponseText(group, runs, expectation) {
   const turn = expectation.turn ?? null;
   if (
@@ -123,6 +217,67 @@ export async function gradeResponseText(group, runs, expectation) {
   );
 }
 
+function compactContextObservation(observation, matched) {
+  return {
+    surface: observation.surface,
+    turn: observation.turn,
+    request: observation.request,
+    characters: observation.characters,
+    sha256: createHash("sha256").update(observation.text, "utf8").digest("hex"),
+    matched,
+    preview: observation.text.slice(0, 240),
+  };
+}
+
+async function contextObservation(run, turn, surface, request) {
+  const observations = await loadContextObservations(run);
+  if (observations === null) return null;
+  const matching = observations.filter((entry) => entry.surface === surface);
+  const inTurn = turn === null ? matching : matching.filter((entry) => entry.turn === turn);
+  if (inTurn.length === 0) return null;
+  let observation;
+  if (request === "first") observation = inTurn[0];
+  else if (request === "last") observation = inTurn.at(-1);
+  else observation = inTurn.find((entry) => (entry.requestInTurn ?? 1) === request);
+  const text = surface === "system-prompt" ? (observation?.text ?? observation?.systemPrompt) : observation?.text;
+  if (!observation || typeof text !== "string") return null;
+  return {
+    surface,
+    turn: observation.turn ?? turn,
+    request: observation.requestInTurn ?? 1,
+    characters: observation.characters ?? text.length,
+    text,
+  };
+}
+
+async function loadContextObservations(run) {
+  const artifact = run.contextObservationArtifact;
+  if (!artifact || typeof artifact.path !== "string" || typeof artifact.sha256 !== "string") return null;
+  const contents = await readFile(artifact.path, "utf8");
+  const actualHash = createHash("sha256").update(contents, "utf8").digest("hex");
+  if (actualHash !== artifact.sha256) throw new Error("context observation artifact changed after run persistence");
+  const observations = [];
+  for (const line of contents.split("\n").filter(Boolean)) {
+    try {
+      observations.push(JSON.parse(line));
+    } catch (error) {
+      throw new Error(
+        `context observation artifact is malformed: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+  return observations;
+}
+
+function toolCalls(run, turn) {
+  if (turn === null) {
+    if (Array.isArray(run.turns)) return run.turns.flatMap((entry) => entry.toolActivity ?? []);
+    return Array.isArray(run.toolActivity) ? run.toolActivity : [];
+  }
+  if (!Array.isArray(run.turns)) return turn === 1 && Array.isArray(run.toolActivity) ? run.toolActivity : [];
+  return run.turns.find((entry) => entry.turn === turn)?.toolActivity ?? [];
+}
+
 function activationMatches(expectation, observed) {
   if (expectation.expect === "never") return !observed.targetRead;
   if (!Number.isInteger(expectation.turn) || expectation.turn < 1) return false;

package/skills/evaluate-skill/scripts/lib/grade.mjs

@@ -1,5 +1,11 @@
 import { gradeComparisons } from "./grade-comparison.mjs";
-import { gradeResourceRead, gradeResponseText, gradeSkillRead } from "./grade-interaction.mjs";
+import {
+  gradeContextText,
+  gradeResourceRead,
+  gradeResponseText,
+  gradeSkillRead,
+  gradeToolCall,
+} from "./grade-interaction.mjs";
 import { gradeChangedPaths, gradeFileText, gradeJson, gradePath } from "./grade-workspace.mjs";
 
 const GRADERS = new Map([
@@ -10,6 +16,8 @@ const GRADERS = new Map([
   ["file-text", gradeFileText],
   ["json", gradeJson],
   ["response-text", gradeResponseText],
+  ["context-text", gradeContextText],
+  ["tool-call", gradeToolCall],
 ]);
 
 /**

package/skills/evaluate-skill/scripts/lib/pi.mjs

@@ -1,7 +1,8 @@
+import { createHash } from "node:crypto";
 import { spawn } from "node:child_process";
 import { once } from "node:events";
 import { createWriteStream } from "node:fs";
-import { realpath } from "node:fs/promises";
+import { mkdir, readFile, realpath } from "node:fs/promises";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 
@@ -21,12 +22,32 @@ import {
   fingerprintDirectory,
   materializeEnvironment,
   materializeFixture,
+  materializeRuntime,
   snapshotWorkspace,
   verifyEnvironment,
+  verifyRuntime,
   workspaceChanges,
 } from "./sandbox.mjs";
 
 const guardExtension = fileURLToPath(new URL("../pi-guard.mjs", import.meta.url));
+const observerExtension = fileURLToPath(new URL("../pi-observer.mjs", import.meta.url));
+const HOST_COMMANDS = { pi: "pi", piflow: "piflow" };
+const BASE_PROCESS_ENVIRONMENT_KEYS = [
+  "PATH",
+  "HOME",
+  "TMPDIR",
+  "TMP",
+  "TEMP",
+  "LANG",
+  "LC_ALL",
+  "XDG_CONFIG_HOME",
+  "XDG_DATA_HOME",
+  "XDG_CACHE_HOME",
+  "PIFLOW_CODING_AGENT_DIR",
+  "PIFLOW_CODING_AGENT_SESSION_DIR",
+  "PI_PACKAGE_DIR",
+  "PI_OFFLINE",
+];
 const TURN_WORKSPACE_KINDS = new Set(["path", "changed-paths", "file-text", "json"]);
 
 export class VariantSetupError extends Error {
@@ -36,19 +57,20 @@ export class VariantSetupError extends Error {
   }
 }
 
-export async function runDescription({ group, variant, root, variantDirectory, fixture, signal }) {
+export async function runDescription({ group, variant, root, variantDirectory, fixture, runtime, signal }) {
   if (group.input.turns !== undefined) {
-    return runPersistentDescription({ group, variant, root, variantDirectory, fixture, signal });
+    return runPersistentDescription({ group, variant, root, variantDirectory, fixture, runtime, signal });
   }
-  return runOneShotDescription({ group, variant, root, variantDirectory, fixture, signal });
+  return runOneShotDescription({ group, variant, root, variantDirectory, fixture, runtime, signal });
 }
 
-export async function runBody({ group, variant, root, variantDirectory, fixture, signal }) {
-  const options = { group, variant, root, variantDirectory, fixture, signal };
+export async function runBody({ group, variant, root, variantDirectory, fixture, runtime, signal }) {
+  const options = { group, variant, root, variantDirectory, fixture, runtime, signal };
   const subject = await prepareSubject(options, "rpc");
   const before = await fingerprintDirectory(subject.workspace);
   const declaredPrompts = group.input.prompt === undefined ? [...group.input.turns] : [group.input.prompt];
   const observation = await runRpcBodySession({
+    command: subject.host.command,
     args: subject.args,
     cwd: subject.workspace,
     eventsFile: subject.eventsFile,
@@ -61,6 +83,11 @@ export async function runBody({ group, variant, root, variantDirectory, fixture,
       ? (turn) => snapshotWorkspace({ workspace: subject.workspace, variantDirectory, turn, before })
       : null,
   });
+  const contextObservation = await readContextObservations(
+    subject.contextObservationPath,
+    subject.requiresContextObservation,
+  );
+  observation.contextObservationArtifact = contextObservation.artifact;
   await verifySubjectResources(subject, observation);
   const after = await fingerprintDirectory(subject.workspace);
 
@@ -132,14 +159,17 @@ function subjectRun({ group, variant }, subject, observation) {
       source: subject.environment.source,
       skills: subject.environment.skills,
       context: subject.environment.context,
+      runtime: subject.runtime,
       contextDelivery: subject.environment.contextDelivery,
       targetPath: subject.environment.targetPath,
     },
+    contextObservationArtifact: observation.contextObservationArtifact ?? null,
     response: observation.response,
     transcript: observation.transcript,
     usage: observation.usage,
     process: {
-      command: "pi",
+      command: subject.host.command,
+      host: subject.host.name,
       args: subject.args,
       exitCode: observation.exitCode,
       signal: observation.signal,
@@ -156,6 +186,7 @@ function subjectRun({ group, variant }, subject, observation) {
 async function runOneShotDescription(options) {
   const subject = await prepareSubject(options, "json");
   const observation = await runPiProcess({
+    command: subject.host.command,
     args: subject.args,
     cwd: subject.workspace,
     eventsFile: subject.eventsFile,
@@ -163,6 +194,11 @@ async function runOneShotDescription(options) {
     signal: options.signal,
     environment: subject.processEnvironment,
   });
+  const contextObservation = await readContextObservations(
+    subject.contextObservationPath,
+    subject.requiresContextObservation,
+  );
+  observation.contextObservationArtifact = contextObservation.artifact;
   await verifySubjectResources(subject, observation);
   const successfulReadPaths = await canonicalReadPaths(observation.successfulReads, subject.workspace);
   const targetRead =
@@ -181,6 +217,7 @@ async function runOneShotDescription(options) {
 async function runPersistentDescription(options) {
   const subject = await prepareSubject(options, "rpc");
   const observation = await runRpcDescriptionSession({
+    command: subject.host.command,
     args: subject.args,
     cwd: subject.workspace,
     eventsFile: subject.eventsFile,
@@ -189,6 +226,11 @@ async function runPersistentDescription(options) {
     prompts: options.group.input.turns,
     environment: subject.processEnvironment,
   });
+  const contextObservation = await readContextObservations(
+    subject.contextObservationPath,
+    subject.requiresContextObservation,
+  );
+  observation.contextObservationArtifact = contextObservation.artifact;
   await verifySubjectResources(subject, observation);
 
   const turns = [];
@@ -222,52 +264,130 @@ async function runPersistentDescription(options) {
   });
 }
 
-async function prepareSubject({ group, variant, root, variantDirectory, fixture: fixtureSnapshot }, mode) {
+async function prepareSubject({ group, variant, root, variantDirectory, fixture: fixtureSnapshot, runtime }, mode) {
   const workspace = path.join(variantDirectory, "workspace");
 
   let environment;
   let fixture;
+  let materializedRuntime;
   try {
     environment = await materializeEnvironment({
       environment: group.variants[variant],
       root,
       variantDirectory,
     });
+    materializedRuntime = await materializeRuntime({ runtime, variantDirectory });
     fixture = await materializeFixture({ fixture: fixtureSnapshot, workspace });
   } catch (error) {
     if (error instanceof VariantSetupError) throw error;
     throw new VariantSetupError(error instanceof Error ? error.message : String(error));
   }
 
-  const args = piArguments(group, environment, mode);
+  const host = { name: materializedRuntime.host, command: HOST_COMMANDS[materializedRuntime.host] };
+  const contextObservationPath = path.join(variantDirectory, "context-observations.jsonl");
+  const requiresContextObservation = group.expectations.some((expectation) => expectation.kind === "context-text");
+  const sessionDirectory = path.join(variantDirectory, "session");
+  if (materializedRuntime.session) await mkdir(sessionDirectory, { recursive: true });
+  const args = piArguments(group, environment, materializedRuntime, mode, requiresContextObservation, sessionDirectory);
   const allowedRoots = [
     await realpath(workspace),
     ...environment.skills.map((skill) => skill.path),
     ...environment.context.map((entry) => entry.path),
+    ...materializedRuntime.extensions.flatMap((bundle) => bundle.resources.map((resource) => resource.path)),
   ];
   return {
     args,
     environment,
+    runtime: materializedRuntime,
+    host,
     fixture,
     workspace,
+    contextObservationPath,
+    requiresContextObservation,
+    sessionDirectory,
     eventsFile: path.join(variantDirectory, "events.jsonl"),
     stderrFile: path.join(variantDirectory, "stderr.log"),
     startedAt: new Date().toISOString(),
-    processEnvironment: {
-      ...process.env,
-      PI_SKIP_VERSION_CHECK: "1",
-      PI_TELEMETRY: "0",
-      SKILL_EVAL_ALLOWED_ROOTS: JSON.stringify(allowedRoots),
-      SKILL_EVAL_WRITABLE_ROOT: await realpath(workspace),
-      SKILL_EVAL_ALLOWED_TOOLS: JSON.stringify(group.tools),
-      SKILL_EVAL_CONTEXT_MANIFEST: environment.contextDelivery.manifestPath ?? "",
-    },
+    processEnvironment: createProcessEnvironment({
+      runtime: materializedRuntime,
+      group,
+      workspace,
+      allowedRoots,
+      contextManifestPath: environment.contextDelivery.manifestPath,
+      contextObservationPath,
+      requiresContextObservation,
+    }),
+  };
+}
+
+function createProcessEnvironment({
+  runtime,
+  group,
+  workspace,
+  allowedRoots,
+  contextManifestPath,
+  contextObservationPath,
+  requiresContextObservation,
+}) {
+  const processEnvironment = Object.fromEntries(
+    BASE_PROCESS_ENVIRONMENT_KEYS.flatMap((key) => (process.env[key] === undefined ? [] : [[key, process.env[key]]])),
+  );
+  Object.assign(processEnvironment, {
+    PWD: workspace,
+    ...runtime.environment.literal,
+    PI_SKIP_VERSION_CHECK: "1",
+    PI_TELEMETRY: "0",
+    SKILL_EVAL_ALLOWED_ROOTS: JSON.stringify(allowedRoots),
+    SKILL_EVAL_WRITABLE_ROOT: workspace,
+    SKILL_EVAL_ALLOWED_TOOLS: JSON.stringify(group.tools),
+    SKILL_EVAL_CONTEXT_MANIFEST: contextManifestPath ?? "",
+    SKILL_EVAL_HOST: runtime.host,
+  });
+  for (const key of runtime.environment.inherit) {
+    if (process.env[key] === undefined) {
+      throw new VariantSetupError(`inherited runtime environment variable is unavailable: ${key}`);
+    }
+    processEnvironment[key] = process.env[key];
+  }
+  if (requiresContextObservation) processEnvironment.SKILL_EVAL_CONTEXT_OBSERVATION_PATH = contextObservationPath;
+  return processEnvironment;
+}
+
+async function readContextObservations(file, required) {
+  const contents = await readFile(file, "utf8").catch((error) => {
+    if (error?.code === "ENOENT") {
+      if (required) throw new Error("context observer produced no evidence");
+      return "";
+    }
+    throw error;
+  });
+  const observations = [];
+  for (const line of contents.split("\n").filter(Boolean)) {
+    try {
+      observations.push(JSON.parse(line));
+    } catch (error) {
+      throw new Error(`context observation is malformed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+  return {
+    observations,
+    artifact:
+      contents === ""
+        ? null
+        : {
+            path: file,
+            sha256: createHash("sha256").update(contents, "utf8").digest("hex"),
+            bytes: Buffer.byteLength(contents, "utf8"),
+            count: observations.length,
+            surfaces: [...new Set(observations.map((entry) => entry.surface).filter(Boolean))],
+          },
   };
 }
 
 async function verifySubjectResources(subject, observation) {
   try {
     await verifyEnvironment(subject.environment);
+    await verifyRuntime(subject.runtime);
   } catch (error) {
     observation.protocolErrors ??= [];
     observation.protocolErrors.push({
@@ -300,20 +420,15 @@ function subjectRunState(observation) {
   return "infrastructure-failed";
 }
 
-function piArguments(group, environment, mode) {
+function piArguments(group, environment, runtime, mode, observeContext, sessionDirectory) {
   const args = ["--mode", mode];
   if (mode === "json") args.push("-p");
-  args.push(
-    "--no-session",
-    "--no-extensions",
-    "--extension",
-    guardExtension,
-    "--no-skills",
-    "--no-prompt-templates",
-    "--no-themes",
-    "--no-context-files",
-    "--no-approve",
-  );
+  if (runtime.session) args.push("--session-dir", sessionDirectory);
+  else args.push("--no-session");
+  args.push("--no-extensions", "--extension", guardExtension);
+  for (const bundle of runtime.extensions) args.push("--extension", bundle.entry);
+  if (observeContext) args.push("--extension", observerExtension);
+  args.push("--no-skills", "--no-prompt-templates", "--no-themes", "--no-context-files", "--no-approve");
   if (group.tools.length === 0) args.push("--no-tools");
   else args.push("--tools", group.tools.join(","));
   for (const skill of environment.skills) args.push("--skill", skill.path);
@@ -323,8 +438,8 @@ function piArguments(group, environment, mode) {
   return args;
 }
 
-async function runPiProcess({ args, cwd, eventsFile, stderrFile, signal, environment }) {
-  const child = spawn("pi", args, {
+async function runPiProcess({ command, args, cwd, eventsFile, stderrFile, signal, environment }) {
+  const child = spawn(command, args, {
     cwd,
     detached: process.platform !== "win32",
     env: environment,

package/skills/evaluate-skill/scripts/lib/rpc.mjs

@@ -28,8 +28,8 @@ export async function runRpcBodySession(options) {
 }
 
 class RpcSession {
-  constructor({ args, cwd, eventsFile, stderrFile, signal, environment, afterTurn = null }) {
-    this.child = spawn("pi", args, {
+  constructor({ command = "pi", args, cwd, eventsFile, stderrFile, signal, environment, afterTurn = null }) {
+    this.child = spawn(command, args, {
       cwd,
       detached: process.platform !== "win32",
       env: environment,
@@ -79,17 +79,17 @@ class RpcSession {
   async run(prompts, targetPath = null) {
     let deliveredPrompts = prompts;
     /** @type {any} */
-    let delivery = { kind: "natural-prompt", turn: 1, targetPath: null };
+    let delivery = { kind: "unavailable", turn: 1, targetPath };
     try {
       await this.configure();
       if (targetPath !== null) {
-        delivery = { kind: "unavailable", turn: 1, targetPath };
         const command = await this.resolveSkillCommand(targetPath);
         deliveredPrompts = [`/${command} ${prompts[0]}`, ...prompts.slice(1)];
         delivery = { kind: "explicit-skill-command", turn: 1, targetPath, command };
       }
       for (const [index, prompt] of deliveredPrompts.entries()) {
         if (this.signal?.aborted || this.terminationReason !== null) break;
+        if (delivery.kind === "unavailable") delivery = { kind: "natural-prompt", turn: 1, targetPath: null };
         const turn = await this.runTurn(index + 1, prompt);
         this.turns.push(turn);
         if (this.afterTurn !== null) turn.workspace = await this.afterTurn(turn.turn);

package/skills/evaluate-skill/scripts/lib/runner.mjs

@@ -5,11 +5,14 @@ import { loadDefinition, selectDefinition } from "./definitions.mjs";
 import { createInvocationId, fileIdentity, writeJson, writeText } from "./evidence.mjs";
 import { gradeDeterministic } from "./grade.mjs";
 import { runBody, runDescription, VariantSetupError } from "./pi.mjs";
-import { prepareFixture } from "./sandbox.mjs";
+import { prepareFixture, prepareRuntime } from "./sandbox.mjs";
 
 const VARIANTS = ["baseline", "candidate"];
 const DESCRIPTION_TOOLS = new Set(["read"]);
 const BODY_TOOLS = new Set(["read", "write", "edit"]);
+// Native tools must be separated from extension tools before CLI tool selection is built.
+const NATIVE_TOOLS = new Set(["read", "bash", "powershell", "edit", "write", "grep", "find", "ls"]);
+const DEFAULT_RUNTIME = { host: "pi", session: false, extensions: [], environment: { literal: {}, inherit: [] } };
 
 /**
  * @param {string} definitionFile
@@ -69,8 +72,18 @@ function assertSupportedSelection(selection) {
       throw new Error("command expectations are not supported");
     }
     const supportedTools = group.type === "body" ? BODY_TOOLS : DESCRIPTION_TOOLS;
-    if (group.tools.some((tool) => !supportedTools.has(tool))) {
-      throw new Error(`${group.type} execution currently supports only ${[...supportedTools].join(", ")} tools`);
+    const unsupportedNativeTools = group.tools.filter((tool) => NATIVE_TOOLS.has(tool) && !supportedTools.has(tool));
+    if (unsupportedNativeTools.length > 0) {
+      throw new Error(`${group.type} execution does not support native tools: ${unsupportedNativeTools.join(", ")}`);
+    }
+    const customTools = group.tools.filter((tool) => !NATIVE_TOOLS.has(tool) && !supportedTools.has(tool));
+    if (group.type === "description" && customTools.length > 0) {
+      throw new Error(`${group.type} execution does not support custom tools: ${customTools.join(", ")}`);
+    }
+    if (customTools.length > 0 && (group.runtime?.extensions?.length ?? 0) === 0) {
+      throw new Error(
+        `${group.type} execution requires declared extensions for custom tools: ${customTools.join(", ")}`,
+      );
     }
   }
 }
@@ -82,12 +95,18 @@ async function runGroup({ selected, root, resultDirectory, signal }) {
   await writeJson(path.join(groupDirectory, "definition.json"), group);
 
   let fixture = null;
-  let fixtureError = null;
+  let runtime = null;
+  let resourceError = null;
   if (!signal?.aborted) {
     try {
       fixture = await prepareFixture({ declaredPath: group.fixture, root, groupDirectory });
+      runtime = await prepareRuntime({
+        runtime: group.runtime ?? DEFAULT_RUNTIME,
+        root,
+        groupDirectory,
+      });
     } catch (error) {
-      fixtureError = error instanceof Error ? error.message : String(error);
+      resourceError = error instanceof Error ? error.message : String(error);
     }
   }
 
@@ -100,10 +119,10 @@ async function runGroup({ selected, root, resultDirectory, signal }) {
       runs[variant] = notSelectedRun(group, variant);
     } else if (signal?.aborted) {
       runs[variant] = cancelledRun(group, variant);
-    } else if (fixtureError !== null) {
-      runs[variant] = invalidRun(group, variant, fixtureError);
+    } else if (resourceError === null) {
+      runs[variant] = await executeVariant({ group, variant, root, variantDirectory, fixture, runtime, signal });
     } else {
-      runs[variant] = await executeVariant({ group, variant, root, variantDirectory, fixture, signal });
+      runs[variant] = invalidRun(group, variant, resourceError);
     }
     const persisted = await persistRunOrFailure(variantDirectory, runs[variant]);
     runs[variant] = persisted.run;
@@ -120,7 +139,8 @@ async function runGroup({ selected, root, resultDirectory, signal }) {
     (variant) => runs[variant],
   );
   const selectedState = selectedGroupState(selectedRuns, persistedGrade.grade);
-  const groupState = selectedState === "cancelled" ? selectedState : fixtureError === null ? selectedState : "invalid";
+  let groupState = selectedState;
+  if (selectedState !== "cancelled" && resourceError !== null) groupState = "invalid";
   const groupResult = {
     id: group.id,
     position: selected.position,
@@ -205,10 +225,10 @@ function errorMessage(error) {
   return error instanceof Error ? error.message : String(error);
 }
 
-async function executeVariant({ group, variant, root, variantDirectory, fixture, signal }) {
+async function executeVariant({ group, variant, root, variantDirectory, fixture, runtime, signal }) {
   try {
     const run = group.type === "body" ? runBody : runDescription;
-    return await run({ group, variant, root, variantDirectory, fixture, signal });
+    return await run({ group, variant, root, variantDirectory, fixture, runtime, signal });
   } catch (error) {
     if (error instanceof VariantSetupError) return invalidRun(group, variant, error.message);
     return infrastructureFailedRun(group, variant, error);
@@ -241,16 +261,18 @@ async function persistRun(variantDirectory, run) {
   await writeText(path.join(variantDirectory, "final.md"), response === "" ? "" : `${response}\n`);
   const runFile = path.join(variantDirectory, "run.json");
   const hasProcessEvidence = run.process !== undefined;
-  await writeJson(runFile, {
+  const persisted = {
     ...run,
-    ...(Array.isArray(run.turns) ? { turns: externalizeTurnTranscripts(run.turns) } : {}),
     transcript: "transcript.json",
     artifacts: {
       events: hasProcessEvidence ? "events.jsonl" : null,
       final: "final.md",
       stderr: hasProcessEvidence ? "stderr.log" : null,
+      context: run.contextObservationArtifact ? "context-observations.jsonl" : null,
     },
-  });
+  };
+  if (Array.isArray(run.turns)) persisted.turns = externalizeTurnTranscripts(run.turns);
+  await writeJson(runFile, persisted);
   return runFile;
 }