@hassangameryt/freeflow 0.5.0 → 0.7.0

package/skills/evaluate-skill/scripts/lib/pi.mjs

@@ -1,7 +1,8 @@
+import { createHash } from "node:crypto";
 import { spawn } from "node:child_process";
 import { once } from "node:events";
 import { createWriteStream } from "node:fs";
-import { realpath } from "node:fs/promises";
+import { mkdir, readFile, realpath } from "node:fs/promises";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 
@@ -21,12 +22,32 @@ import {
   fingerprintDirectory,
   materializeEnvironment,
   materializeFixture,
+  materializeRuntime,
   snapshotWorkspace,
   verifyEnvironment,
+  verifyRuntime,
   workspaceChanges,
 } from "./sandbox.mjs";
 
 const guardExtension = fileURLToPath(new URL("../pi-guard.mjs", import.meta.url));
+const observerExtension = fileURLToPath(new URL("../pi-observer.mjs", import.meta.url));
+const HOST_COMMANDS = { pi: "pi", piflow: "piflow" };
+const BASE_PROCESS_ENVIRONMENT_KEYS = [
+  "PATH",
+  "HOME",
+  "TMPDIR",
+  "TMP",
+  "TEMP",
+  "LANG",
+  "LC_ALL",
+  "XDG_CONFIG_HOME",
+  "XDG_DATA_HOME",
+  "XDG_CACHE_HOME",
+  "PIFLOW_CODING_AGENT_DIR",
+  "PIFLOW_CODING_AGENT_SESSION_DIR",
+  "PI_PACKAGE_DIR",
+  "PI_OFFLINE",
+];
 const TURN_WORKSPACE_KINDS = new Set(["path", "changed-paths", "file-text", "json"]);
 
 export class VariantSetupError extends Error {
@@ -36,19 +57,20 @@ export class VariantSetupError extends Error {
   }
 }
 
-export async function runDescription({ group, variant, root, variantDirectory, fixture, signal }) {
+export async function runDescription({ group, variant, root, variantDirectory, fixture, runtime, signal }) {
   if (group.input.turns !== undefined) {
-    return runPersistentDescription({ group, variant, root, variantDirectory, fixture, signal });
+    return runPersistentDescription({ group, variant, root, variantDirectory, fixture, runtime, signal });
   }
-  return runOneShotDescription({ group, variant, root, variantDirectory, fixture, signal });
+  return runOneShotDescription({ group, variant, root, variantDirectory, fixture, runtime, signal });
 }
 
-export async function runBody({ group, variant, root, variantDirectory, fixture, signal }) {
-  const options = { group, variant, root, variantDirectory, fixture, signal };
+export async function runBody({ group, variant, root, variantDirectory, fixture, runtime, signal }) {
+  const options = { group, variant, root, variantDirectory, fixture, runtime, signal };
   const subject = await prepareSubject(options, "rpc");
   const before = await fingerprintDirectory(subject.workspace);
   const declaredPrompts = group.input.prompt === undefined ? [...group.input.turns] : [group.input.prompt];
   const observation = await runRpcBodySession({
+    command: subject.host.command,
     args: subject.args,
     cwd: subject.workspace,
     eventsFile: subject.eventsFile,
@@ -61,6 +83,11 @@ export async function runBody({ group, variant, root, variantDirectory, fixture,
       ? (turn) => snapshotWorkspace({ workspace: subject.workspace, variantDirectory, turn, before })
       : null,
   });
+  const contextObservation = await readContextObservations(
+    subject.contextObservationPath,
+    subject.requiresContextObservation,
+  );
+  observation.contextObservationArtifact = contextObservation.artifact;
   await verifySubjectResources(subject, observation);
   const after = await fingerprintDirectory(subject.workspace);
 
@@ -132,14 +159,17 @@ function subjectRun({ group, variant }, subject, observation) {
       source: subject.environment.source,
       skills: subject.environment.skills,
       context: subject.environment.context,
+      runtime: subject.runtime,
       contextDelivery: subject.environment.contextDelivery,
       targetPath: subject.environment.targetPath,
     },
+    contextObservationArtifact: observation.contextObservationArtifact ?? null,
     response: observation.response,
     transcript: observation.transcript,
     usage: observation.usage,
     process: {
-      command: "pi",
+      command: subject.host.command,
+      host: subject.host.name,
       args: subject.args,
       exitCode: observation.exitCode,
       signal: observation.signal,
@@ -156,6 +186,7 @@ function subjectRun({ group, variant }, subject, observation) {
 async function runOneShotDescription(options) {
   const subject = await prepareSubject(options, "json");
   const observation = await runPiProcess({
+    command: subject.host.command,
     args: subject.args,
     cwd: subject.workspace,
     eventsFile: subject.eventsFile,
@@ -163,6 +194,11 @@ async function runOneShotDescription(options) {
     signal: options.signal,
     environment: subject.processEnvironment,
   });
+  const contextObservation = await readContextObservations(
+    subject.contextObservationPath,
+    subject.requiresContextObservation,
+  );
+  observation.contextObservationArtifact = contextObservation.artifact;
   await verifySubjectResources(subject, observation);
   const successfulReadPaths = await canonicalReadPaths(observation.successfulReads, subject.workspace);
   const targetRead =
@@ -181,6 +217,7 @@ async function runOneShotDescription(options) {
 async function runPersistentDescription(options) {
   const subject = await prepareSubject(options, "rpc");
   const observation = await runRpcDescriptionSession({
+    command: subject.host.command,
     args: subject.args,
     cwd: subject.workspace,
     eventsFile: subject.eventsFile,
@@ -189,6 +226,11 @@ async function runPersistentDescription(options) {
     prompts: options.group.input.turns,
     environment: subject.processEnvironment,
   });
+  const contextObservation = await readContextObservations(
+    subject.contextObservationPath,
+    subject.requiresContextObservation,
+  );
+  observation.contextObservationArtifact = contextObservation.artifact;
   await verifySubjectResources(subject, observation);
 
   const turns = [];
@@ -222,52 +264,130 @@ async function runPersistentDescription(options) {
   });
 }
 
-async function prepareSubject({ group, variant, root, variantDirectory, fixture: fixtureSnapshot }, mode) {
+async function prepareSubject({ group, variant, root, variantDirectory, fixture: fixtureSnapshot, runtime }, mode) {
   const workspace = path.join(variantDirectory, "workspace");
 
   let environment;
   let fixture;
+  let materializedRuntime;
   try {
     environment = await materializeEnvironment({
       environment: group.variants[variant],
       root,
       variantDirectory,
     });
+    materializedRuntime = await materializeRuntime({ runtime, variantDirectory });
     fixture = await materializeFixture({ fixture: fixtureSnapshot, workspace });
   } catch (error) {
     if (error instanceof VariantSetupError) throw error;
     throw new VariantSetupError(error instanceof Error ? error.message : String(error));
   }
 
-  const args = piArguments(group, environment, mode);
+  const host = { name: materializedRuntime.host, command: HOST_COMMANDS[materializedRuntime.host] };
+  const contextObservationPath = path.join(variantDirectory, "context-observations.jsonl");
+  const requiresContextObservation = group.expectations.some((expectation) => expectation.kind === "context-text");
+  const sessionDirectory = path.join(variantDirectory, "session");
+  if (materializedRuntime.session) await mkdir(sessionDirectory, { recursive: true });
+  const args = piArguments(group, environment, materializedRuntime, mode, requiresContextObservation, sessionDirectory);
   const allowedRoots = [
     await realpath(workspace),
     ...environment.skills.map((skill) => skill.path),
     ...environment.context.map((entry) => entry.path),
+    ...materializedRuntime.extensions.flatMap((bundle) => bundle.resources.map((resource) => resource.path)),
   ];
   return {
     args,
     environment,
+    runtime: materializedRuntime,
+    host,
     fixture,
     workspace,
+    contextObservationPath,
+    requiresContextObservation,
+    sessionDirectory,
     eventsFile: path.join(variantDirectory, "events.jsonl"),
     stderrFile: path.join(variantDirectory, "stderr.log"),
     startedAt: new Date().toISOString(),
-    processEnvironment: {
-      ...process.env,
-      PI_SKIP_VERSION_CHECK: "1",
-      PI_TELEMETRY: "0",
-      SKILL_EVAL_ALLOWED_ROOTS: JSON.stringify(allowedRoots),
-      SKILL_EVAL_WRITABLE_ROOT: await realpath(workspace),
-      SKILL_EVAL_ALLOWED_TOOLS: JSON.stringify(group.tools),
-      SKILL_EVAL_CONTEXT_MANIFEST: environment.contextDelivery.manifestPath ?? "",
-    },
+    processEnvironment: createProcessEnvironment({
+      runtime: materializedRuntime,
+      group,
+      workspace,
+      allowedRoots,
+      contextManifestPath: environment.contextDelivery.manifestPath,
+      contextObservationPath,
+      requiresContextObservation,
+    }),
+  };
+}
+
+function createProcessEnvironment({
+  runtime,
+  group,
+  workspace,
+  allowedRoots,
+  contextManifestPath,
+  contextObservationPath,
+  requiresContextObservation,
+}) {
+  const processEnvironment = Object.fromEntries(
+    BASE_PROCESS_ENVIRONMENT_KEYS.flatMap((key) => (process.env[key] === undefined ? [] : [[key, process.env[key]]])),
+  );
+  Object.assign(processEnvironment, {
+    PWD: workspace,
+    ...runtime.environment.literal,
+    PI_SKIP_VERSION_CHECK: "1",
+    PI_TELEMETRY: "0",
+    SKILL_EVAL_ALLOWED_ROOTS: JSON.stringify(allowedRoots),
+    SKILL_EVAL_WRITABLE_ROOT: workspace,
+    SKILL_EVAL_ALLOWED_TOOLS: JSON.stringify(group.tools),
+    SKILL_EVAL_CONTEXT_MANIFEST: contextManifestPath ?? "",
+    SKILL_EVAL_HOST: runtime.host,
+  });
+  for (const key of runtime.environment.inherit) {
+    if (process.env[key] === undefined) {
+      throw new VariantSetupError(`inherited runtime environment variable is unavailable: ${key}`);
+    }
+    processEnvironment[key] = process.env[key];
+  }
+  if (requiresContextObservation) processEnvironment.SKILL_EVAL_CONTEXT_OBSERVATION_PATH = contextObservationPath;
+  return processEnvironment;
+}
+
+async function readContextObservations(file, required) {
+  const contents = await readFile(file, "utf8").catch((error) => {
+    if (error?.code === "ENOENT") {
+      if (required) throw new Error("context observer produced no evidence");
+      return "";
+    }
+    throw error;
+  });
+  const observations = [];
+  for (const line of contents.split("\n").filter(Boolean)) {
+    try {
+      observations.push(JSON.parse(line));
+    } catch (error) {
+      throw new Error(`context observation is malformed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+  return {
+    observations,
+    artifact:
+      contents === ""
+        ? null
+        : {
+            path: file,
+            sha256: createHash("sha256").update(contents, "utf8").digest("hex"),
+            bytes: Buffer.byteLength(contents, "utf8"),
+            count: observations.length,
+            surfaces: [...new Set(observations.map((entry) => entry.surface).filter(Boolean))],
+          },
   };
 }
 
 async function verifySubjectResources(subject, observation) {
   try {
     await verifyEnvironment(subject.environment);
+    await verifyRuntime(subject.runtime);
   } catch (error) {
     observation.protocolErrors ??= [];
     observation.protocolErrors.push({
@@ -300,20 +420,15 @@ function subjectRunState(observation) {
   return "infrastructure-failed";
 }
 
-function piArguments(group, environment, mode) {
+function piArguments(group, environment, runtime, mode, observeContext, sessionDirectory) {
   const args = ["--mode", mode];
   if (mode === "json") args.push("-p");
-  args.push(
-    "--no-session",
-    "--no-extensions",
-    "--extension",
-    guardExtension,
-    "--no-skills",
-    "--no-prompt-templates",
-    "--no-themes",
-    "--no-context-files",
-    "--no-approve",
-  );
+  if (runtime.session) args.push("--session-dir", sessionDirectory);
+  else args.push("--no-session");
+  args.push("--no-extensions", "--extension", guardExtension);
+  for (const bundle of runtime.extensions) args.push("--extension", bundle.entry);
+  if (observeContext) args.push("--extension", observerExtension);
+  args.push("--no-skills", "--no-prompt-templates", "--no-themes", "--no-context-files", "--no-approve");
   if (group.tools.length === 0) args.push("--no-tools");
   else args.push("--tools", group.tools.join(","));
   for (const skill of environment.skills) args.push("--skill", skill.path);
@@ -323,8 +438,8 @@ function piArguments(group, environment, mode) {
   return args;
 }
 
-async function runPiProcess({ args, cwd, eventsFile, stderrFile, signal, environment }) {
-  const child = spawn("pi", args, {
+async function runPiProcess({ command, args, cwd, eventsFile, stderrFile, signal, environment }) {
+  const child = spawn(command, args, {
     cwd,
     detached: process.platform !== "win32",
     env: environment,

package/skills/evaluate-skill/scripts/lib/rpc.mjs

@@ -28,8 +28,8 @@ export async function runRpcBodySession(options) {
 }
 
 class RpcSession {
-  constructor({ args, cwd, eventsFile, stderrFile, signal, environment, afterTurn = null }) {
-    this.child = spawn("pi", args, {
+  constructor({ command = "pi", args, cwd, eventsFile, stderrFile, signal, environment, afterTurn = null }) {
+    this.child = spawn(command, args, {
       cwd,
       detached: process.platform !== "win32",
       env: environment,
@@ -79,17 +79,17 @@ class RpcSession {
   async run(prompts, targetPath = null) {
     let deliveredPrompts = prompts;
     /** @type {any} */
-    let delivery = { kind: "natural-prompt", turn: 1, targetPath: null };
+    let delivery = { kind: "unavailable", turn: 1, targetPath };
     try {
       await this.configure();
       if (targetPath !== null) {
-        delivery = { kind: "unavailable", turn: 1, targetPath };
         const command = await this.resolveSkillCommand(targetPath);
         deliveredPrompts = [`/${command} ${prompts[0]}`, ...prompts.slice(1)];
         delivery = { kind: "explicit-skill-command", turn: 1, targetPath, command };
       }
       for (const [index, prompt] of deliveredPrompts.entries()) {
         if (this.signal?.aborted || this.terminationReason !== null) break;
+        if (delivery.kind === "unavailable") delivery = { kind: "natural-prompt", turn: 1, targetPath: null };
         const turn = await this.runTurn(index + 1, prompt);
         this.turns.push(turn);
         if (this.afterTurn !== null) turn.workspace = await this.afterTurn(turn.turn);

package/skills/evaluate-skill/scripts/lib/runner.mjs

@@ -5,11 +5,14 @@ import { loadDefinition, selectDefinition } from "./definitions.mjs";
 import { createInvocationId, fileIdentity, writeJson, writeText } from "./evidence.mjs";
 import { gradeDeterministic } from "./grade.mjs";
 import { runBody, runDescription, VariantSetupError } from "./pi.mjs";
-import { prepareFixture } from "./sandbox.mjs";
+import { prepareFixture, prepareRuntime } from "./sandbox.mjs";
 
 const VARIANTS = ["baseline", "candidate"];
 const DESCRIPTION_TOOLS = new Set(["read"]);
 const BODY_TOOLS = new Set(["read", "write", "edit"]);
+// Native tools must be separated from extension tools before CLI tool selection is built.
+const NATIVE_TOOLS = new Set(["read", "bash", "powershell", "edit", "write", "grep", "find", "ls"]);
+const DEFAULT_RUNTIME = { host: "pi", session: false, extensions: [], environment: { literal: {}, inherit: [] } };
 
 /**
  * @param {string} definitionFile
@@ -69,8 +72,18 @@ function assertSupportedSelection(selection) {
       throw new Error("command expectations are not supported");
     }
     const supportedTools = group.type === "body" ? BODY_TOOLS : DESCRIPTION_TOOLS;
-    if (group.tools.some((tool) => !supportedTools.has(tool))) {
-      throw new Error(`${group.type} execution currently supports only ${[...supportedTools].join(", ")} tools`);
+    const unsupportedNativeTools = group.tools.filter((tool) => NATIVE_TOOLS.has(tool) && !supportedTools.has(tool));
+    if (unsupportedNativeTools.length > 0) {
+      throw new Error(`${group.type} execution does not support native tools: ${unsupportedNativeTools.join(", ")}`);
+    }
+    const customTools = group.tools.filter((tool) => !NATIVE_TOOLS.has(tool) && !supportedTools.has(tool));
+    if (group.type === "description" && customTools.length > 0) {
+      throw new Error(`${group.type} execution does not support custom tools: ${customTools.join(", ")}`);
+    }
+    if (customTools.length > 0 && (group.runtime?.extensions?.length ?? 0) === 0) {
+      throw new Error(
+        `${group.type} execution requires declared extensions for custom tools: ${customTools.join(", ")}`,
+      );
     }
   }
 }
@@ -82,12 +95,18 @@ async function runGroup({ selected, root, resultDirectory, signal }) {
   await writeJson(path.join(groupDirectory, "definition.json"), group);
 
   let fixture = null;
-  let fixtureError = null;
+  let runtime = null;
+  let resourceError = null;
   if (!signal?.aborted) {
     try {
       fixture = await prepareFixture({ declaredPath: group.fixture, root, groupDirectory });
+      runtime = await prepareRuntime({
+        runtime: group.runtime ?? DEFAULT_RUNTIME,
+        root,
+        groupDirectory,
+      });
     } catch (error) {
-      fixtureError = error instanceof Error ? error.message : String(error);
+      resourceError = error instanceof Error ? error.message : String(error);
     }
   }
 
@@ -100,10 +119,10 @@ async function runGroup({ selected, root, resultDirectory, signal }) {
       runs[variant] = notSelectedRun(group, variant);
     } else if (signal?.aborted) {
       runs[variant] = cancelledRun(group, variant);
-    } else if (fixtureError !== null) {
-      runs[variant] = invalidRun(group, variant, fixtureError);
+    } else if (resourceError === null) {
+      runs[variant] = await executeVariant({ group, variant, root, variantDirectory, fixture, runtime, signal });
     } else {
-      runs[variant] = await executeVariant({ group, variant, root, variantDirectory, fixture, signal });
+      runs[variant] = invalidRun(group, variant, resourceError);
     }
     const persisted = await persistRunOrFailure(variantDirectory, runs[variant]);
     runs[variant] = persisted.run;
@@ -120,7 +139,8 @@ async function runGroup({ selected, root, resultDirectory, signal }) {
     (variant) => runs[variant],
   );
   const selectedState = selectedGroupState(selectedRuns, persistedGrade.grade);
-  const groupState = selectedState === "cancelled" ? selectedState : fixtureError === null ? selectedState : "invalid";
+  let groupState = selectedState;
+  if (selectedState !== "cancelled" && resourceError !== null) groupState = "invalid";
   const groupResult = {
     id: group.id,
     position: selected.position,
@@ -205,10 +225,10 @@ function errorMessage(error) {
   return error instanceof Error ? error.message : String(error);
 }
 
-async function executeVariant({ group, variant, root, variantDirectory, fixture, signal }) {
+async function executeVariant({ group, variant, root, variantDirectory, fixture, runtime, signal }) {
   try {
     const run = group.type === "body" ? runBody : runDescription;
-    return await run({ group, variant, root, variantDirectory, fixture, signal });
+    return await run({ group, variant, root, variantDirectory, fixture, runtime, signal });
   } catch (error) {
     if (error instanceof VariantSetupError) return invalidRun(group, variant, error.message);
     return infrastructureFailedRun(group, variant, error);
@@ -241,16 +261,18 @@ async function persistRun(variantDirectory, run) {
   await writeText(path.join(variantDirectory, "final.md"), response === "" ? "" : `${response}\n`);
   const runFile = path.join(variantDirectory, "run.json");
   const hasProcessEvidence = run.process !== undefined;
-  await writeJson(runFile, {
+  const persisted = {
     ...run,
-    ...(Array.isArray(run.turns) ? { turns: externalizeTurnTranscripts(run.turns) } : {}),
     transcript: "transcript.json",
     artifacts: {
       events: hasProcessEvidence ? "events.jsonl" : null,
       final: "final.md",
       stderr: hasProcessEvidence ? "stderr.log" : null,
+      context: run.contextObservationArtifact ? "context-observations.jsonl" : null,
     },
-  });
+  };
+  if (Array.isArray(run.turns)) persisted.turns = externalizeTurnTranscripts(run.turns);
+  await writeJson(runFile, persisted);
   return runFile;
 }
 

package/skills/evaluate-skill/scripts/lib/sandbox.mjs

@@ -69,6 +69,98 @@ export async function materializeFixture({ fixture, workspace }) {
   return { ...fixture, materializedPath };
 }
 
+export async function prepareRuntime({ runtime, root, groupDirectory }) {
+  const canonicalRoot = await realpath(root);
+  const runtimeRoot = path.join(groupDirectory, "resources", "runtime", "extensions");
+  await mkdir(runtimeRoot, { recursive: true });
+  const bundles = [];
+
+  for (const [index, bundle] of runtime.extensions.entries()) {
+    const bundleRoot = path.join(runtimeRoot, String(index));
+    await mkdir(bundleRoot, { recursive: true });
+    const resources = [];
+    for (const declaredPath of bundle.resources) {
+      resources.push(
+        await snapshotWorkingTreeResource({
+          canonicalRoot,
+          declaredPath,
+          destination: path.join(bundleRoot, declaredPath),
+          requireDirectory: true,
+          label: "runtime resource",
+        }),
+      );
+    }
+    const root = await realpath(bundleRoot);
+    const entry = await realpath(path.join(root, bundle.entry)).catch(() => null);
+    if (entry === null || !isContained(root, entry)) {
+      throw new Error(`declared runtime entry is missing or escapes its bundle: ${bundle.entry}`);
+    }
+    const entryStat = await lstat(entry);
+    if (!entryStat.isFile()) throw new Error(`declared runtime entry is not a file: ${bundle.entry}`);
+    if (!resources.some((resource) => isContained(resource.path, entry))) {
+      throw new Error(`declared runtime entry is not contained by a resource: ${bundle.entry}`);
+    }
+    bundles.push({
+      entry,
+      root,
+      resources,
+    });
+  }
+
+  return {
+    host: runtime.host,
+    session: runtime.session,
+    environment: { ...runtime.environment },
+    extensions: bundles,
+  };
+}
+
+export async function materializeRuntime({ runtime, variantDirectory }) {
+  const runtimeRoot = path.join(variantDirectory, "resources", "runtime", "extensions");
+  await mkdir(runtimeRoot, { recursive: true });
+  const bundles = [];
+
+  for (const [index, bundle] of runtime.extensions.entries()) {
+    const destination = path.join(runtimeRoot, String(index));
+    await cp(bundle.root, destination, {
+      recursive: true,
+      dereference: false,
+      errorOnExist: true,
+      force: false,
+    });
+    const root = await realpath(destination);
+    const resources = [];
+    for (const resource of bundle.resources) {
+      const resourceRoot = await realpath(path.join(root, resource.declaredPath));
+      const files = await fingerprintDirectory(resourceRoot);
+      if (JSON.stringify(files) !== JSON.stringify(resource.files)) {
+        throw new Error(`declared runtime resource changed while being copied: ${resource.declaredPath}`);
+      }
+      resources.push({ ...resource, path: resourceRoot, files });
+    }
+    const entry = await realpath(path.join(root, path.relative(bundle.root, bundle.entry)));
+    bundles.push({ ...bundle, entry, root, resources });
+  }
+
+  return {
+    host: runtime.host,
+    session: runtime.session,
+    environment: { ...runtime.environment },
+    extensions: bundles,
+  };
+}
+
+export async function verifyRuntime(runtime) {
+  for (const bundle of runtime.extensions) {
+    for (const resource of bundle.resources) {
+      const current = await fingerprintDirectory(resource.path);
+      if (JSON.stringify(current) !== JSON.stringify(resource.files)) {
+        throw new Error(`declared runtime resource changed during subject execution: ${resource.declaredPath}`);
+      }
+    }
+  }
+}
+
 export async function materializeEnvironment({ environment, root, variantDirectory }) {
   const canonicalRoot = await realpath(root);
   let source;

package/skills/evaluate-skill/scripts/pi-guard.mjs

@@ -15,7 +15,7 @@ function parseStringArray(value) {
 
 function isContained(root, target) {
   const relative = path.relative(root, target);
-  return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
+  return relative === "" || (!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative));
 }
 
 function resolveWritablePath(cwd, requested) {

package/skills/evaluate-skill/scripts/pi-observer.mjs

@@ -0,0 +1,66 @@
+import { appendFileSync } from "node:fs";
+
+const outputPath = process.env.SKILL_EVAL_CONTEXT_OBSERVATION_PATH;
+let turn = 0;
+let requestInTurn = 0;
+let providerRequest = 0;
+
+function textFromContent(content) {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+  return content
+    .map((block) => {
+      if (!block || typeof block !== "object") return "";
+      if (block.type === "text" && typeof block.text === "string") return block.text;
+      if (block.type === "image" && typeof block.mimeType === "string") return `[image:${block.mimeType}]`;
+      return "";
+    })
+    .filter(Boolean)
+    .join("\n");
+}
+
+function append(event) {
+  if (!outputPath) return;
+  appendFileSync(outputPath, `${JSON.stringify({ timestamp: new Date().toISOString(), turn, ...event })}\n`, "utf8");
+}
+
+export default function skillEvaluationObserver(pi) {
+  pi.on("before_agent_start", (event, ctx) => {
+    turn += 1;
+    requestInTurn = 0;
+    const systemPrompt =
+      typeof event?.systemPrompt === "string" ? event.systemPrompt : String(ctx?.getSystemPrompt?.() ?? "");
+    append({
+      kind: "context",
+      surface: "system-prompt",
+      prompt: typeof event?.prompt === "string" ? event.prompt : null,
+      characters: systemPrompt.length,
+      text: systemPrompt,
+      systemPrompt,
+    });
+  });
+
+  pi.on("context", (event) => {
+    providerRequest += 1;
+    requestInTurn += 1;
+    const messages = Array.isArray(event?.messages) ? event.messages : [];
+    const normalized = messages.map((message) => ({
+      role: message?.role ?? null,
+      toolName: message?.toolName ?? null,
+      text: textFromContent(message?.content),
+    }));
+    const text = normalized
+      .map((message) => message.text)
+      .filter(Boolean)
+      .join("\n");
+    append({
+      kind: "context",
+      surface: "provider-context",
+      providerRequest,
+      requestInTurn,
+      characters: text.length,
+      text,
+      messages: normalized,
+    });
+  });
+}

package/skills/evaluate-skill/scripts/skill-eval.mjs

@@ -34,8 +34,9 @@ Selection:
 Current run support:
   Description and explicit-body groups with prompt or ordered turns
   Working-tree or Git-backed ordered skills/context; optional fresh fixture copies
-  Description tools: read; body tools: read, write, edit
-  Deterministic reads, paths, changed paths, text, JSON, and factual comparisons
+  Description tools: read; body tools: read, write, edit, or declared extension tools
+  Runtime profiles: Pi or PiFlow hosts, ordered immutable extension bundles, environment overrides
+  Deterministic reads, paths, changed paths, text, JSON, tool calls, context observations, and factual comparisons
   Ordered suites run serially and continue after isolated variant, group, or post-processing failures
   Grade-first views show compact criterion details, usage, and result-relative artifact paths
 `,
@@ -71,9 +72,7 @@ function parseOperation(values) {
 
 if (!command || command === "--help" || command === "-h") {
   printUsage();
-} else if (!commands.has(command)) {
-  fail(`Unknown command: ${command}`);
-} else {
+} else if (commands.has(command)) {
   try {
     const operation = parseOperation(argv.slice(1));
     if (command === "view") {
@@ -98,4 +97,6 @@ if (!command || command === "--help" || command === "-h") {
   } catch (error) {
     fail(error instanceof Error ? error.message : String(error), 1);
   }
+} else {
+  fail(`Unknown command: ${command}`);
 }