@hassangameryt/freeflow 0.7.3 → 0.9.0

package/skills/evaluate-skill/scripts/lib/runner.mjs

@@ -1,368 +0,0 @@
-import { mkdir, realpath } from "node:fs/promises";
-import path from "node:path";
-
-import { loadDefinition, selectDefinition } from "./definitions.mjs";
-import { createInvocationId, fileIdentity, writeJson, writeText } from "./evidence.mjs";
-import { gradeDeterministic } from "./grade.mjs";
-import { runBody, runDescription, VariantSetupError } from "./pi.mjs";
-import { prepareFixture, prepareRuntime } from "./sandbox.mjs";
-
-const VARIANTS = ["baseline", "candidate"];
-const DESCRIPTION_TOOLS = new Set(["read"]);
-const BODY_TOOLS = new Set(["read", "write", "edit"]);
-// Native tools must be separated from extension tools before CLI tool selection is built.
-const NATIVE_TOOLS = new Set(["read", "bash", "powershell", "edit", "write", "grep", "find", "ls"]);
-const DEFAULT_RUNTIME = { host: "pi", session: false, extensions: [], environment: { literal: {}, inherit: [] } };
-
-/**
- * @param {string} definitionFile
- * @param {any} selectors
- * @param {{root?: string, signal?: AbortSignal}} options
- */
-export async function runEvaluation(definitionFile, selectors = {}, { root = process.cwd(), signal } = {}) {
-  const definitionPath = await realpath(path.resolve(root, definitionFile));
-  const definition = await loadDefinition(definitionPath, { root });
-  const selection = selectDefinition(definition, selectors);
-  assertSupportedSelection(selection);
-  const invocationId = createInvocationId();
-  const runsDirectory = path.join(root, ".skill-eval", "runs");
-  const resultDirectory = path.join(runsDirectory, invocationId);
-  await mkdir(runsDirectory, { recursive: true });
-  await mkdir(resultDirectory);
-  await writeJson(path.join(resultDirectory, "invocation.json"), {
-    schema_version: 1,
-    id: invocationId,
-    definition: definitionPath,
-    selection: {
-      group: selectors.group ?? null,
-      variant: selectors.variant ?? null,
-      groups: selection.groups.map((entry) => ({
-        id: entry.group.id,
-        position: entry.position,
-        variants: entry.variants,
-      })),
-    },
-    startedAt: new Date().toISOString(),
-  });
-
-  const groups = [];
-  for (const selected of selection.groups) {
-    groups.push(await runGroup({ selected, root, resultDirectory, signal }));
-  }
-  const state = batchState(groups);
-  const summary = {
-    schema_version: 1,
-    id: invocationId,
-    state,
-    definitionKind: selection.definitionKind,
-    groups,
-    completedAt: new Date().toISOString(),
-  };
-  await writeJson(path.join(resultDirectory, "summary.json"), summary);
-  return { id: invocationId, path: resultDirectory, state, summary };
-}
-
-function assertSupportedSelection(selection) {
-  for (const selected of selection.groups) {
-    const { group } = selected;
-    if (group.type !== "description" && group.type !== "body") {
-      throw new Error("skill-eval run currently supports description and body groups only");
-    }
-    if (group.expectations.some((expectation) => expectation.kind === "command")) {
-      throw new Error("command expectations are not supported");
-    }
-    const supportedTools = group.type === "body" ? BODY_TOOLS : DESCRIPTION_TOOLS;
-    const unsupportedNativeTools = group.tools.filter((tool) => NATIVE_TOOLS.has(tool) && !supportedTools.has(tool));
-    if (unsupportedNativeTools.length > 0) {
-      throw new Error(`${group.type} execution does not support native tools: ${unsupportedNativeTools.join(", ")}`);
-    }
-    const customTools = group.tools.filter((tool) => !NATIVE_TOOLS.has(tool) && !supportedTools.has(tool));
-    if (group.type === "description" && customTools.length > 0) {
-      throw new Error(`${group.type} execution does not support custom tools: ${customTools.join(", ")}`);
-    }
-    if (customTools.length > 0 && (group.runtime?.extensions?.length ?? 0) === 0) {
-      throw new Error(
-        `${group.type} execution requires declared extensions for custom tools: ${customTools.join(", ")}`,
-      );
-    }
-  }
-}
-
-async function runGroup({ selected, root, resultDirectory, signal }) {
-  const { group } = selected;
-  const groupDirectory = path.join(resultDirectory, "groups", group.id);
-  await mkdir(groupDirectory, { recursive: true });
-  await writeJson(path.join(groupDirectory, "definition.json"), group);
-
-  let fixture = null;
-  let runtime = null;
-  let resourceError = null;
-  if (!signal?.aborted) {
-    try {
-      fixture = await prepareFixture({ declaredPath: group.fixture, root, groupDirectory });
-      runtime = await prepareRuntime({
-        runtime: group.runtime ?? DEFAULT_RUNTIME,
-        root,
-        groupDirectory,
-      });
-    } catch (error) {
-      resourceError = error instanceof Error ? error.message : String(error);
-    }
-  }
-
-  const runs = {};
-  const runFiles = {};
-  for (const variant of VARIANTS) {
-    const variantDirectory = path.join(groupDirectory, variant);
-    await mkdir(variantDirectory, { recursive: true });
-    if (selected.variants[variant] === "not-selected") {
-      runs[variant] = notSelectedRun(group, variant);
-    } else if (signal?.aborted) {
-      runs[variant] = cancelledRun(group, variant);
-    } else if (resourceError === null) {
-      runs[variant] = await executeVariant({ group, variant, root, variantDirectory, fixture, runtime, signal });
-    } else {
-      runs[variant] = invalidRun(group, variant, resourceError);
-    }
-    const persisted = await persistRunOrFailure(variantDirectory, runs[variant]);
-    runs[variant] = persisted.run;
-    runFiles[variant] = persisted.file;
-  }
-
-  const evidence = {
-    baseline: await fileIdentity(runFiles.baseline),
-    candidate: await fileIdentity(runFiles.candidate),
-  };
-  const grade = await createGrade(group, runs, evidence);
-  const persistedGrade = await persistGrade(groupDirectory, grade);
-  const selectedRuns = VARIANTS.filter((variant) => selected.variants[variant] === "selected").map(
-    (variant) => runs[variant],
-  );
-  const selectedState = selectedGroupState(selectedRuns, persistedGrade.grade);
-  let groupState = selectedState;
-  if (selectedState !== "cancelled" && resourceError !== null) groupState = "invalid";
-  const groupResult = {
-    id: group.id,
-    position: selected.position,
-    state: groupState,
-    variants: Object.fromEntries(VARIANTS.map((variant) => [variant, runs[variant].state])),
-    grade: persistedGrade.grade.state,
-    artifacts: { grade: persistedGrade.artifact, group: "group.json" },
-    errors: persistedGrade.errors,
-  };
-  return persistGroupResult(groupDirectory, groupResult);
-}
-
-function batchState(groups) {
-  if (groups.some((group) => group.state === "cancelled")) return "cancelled";
-  if (groups.every((group) => group.state === "complete")) return "complete";
-  return "partially-complete";
-}
-
-function selectedGroupState(runs, grade) {
-  if (runs.some((run) => run.state === "cancelled")) return "cancelled";
-  if (runs.every((run) => run.state === "complete") && grade.state !== "grade-error") return "complete";
-  return "partially-complete";
-}
-
-async function createGrade(group, runs, evidence) {
-  try {
-    return await gradeDeterministic(group, runs, evidence);
-  } catch (error) {
-    return gradeErrorReport(evidence, `deterministic grading failed: ${errorMessage(error)}`);
-  }
-}
-
-async function persistGrade(groupDirectory, grade) {
-  const artifact = "deterministic-grade.json";
-  try {
-    await writeJson(path.join(groupDirectory, artifact), grade);
-    return { grade, artifact, errors: [] };
-  } catch (error) {
-    const reason = `failed to persist ${artifact}: ${errorMessage(error)}`;
-    const failed = gradeErrorReport(grade.evidence, reason, grade);
-    const fallback = "deterministic-grade-error.json";
-    await writeJson(path.join(groupDirectory, fallback), failed);
-    return {
-      grade: failed,
-      artifact: fallback,
-      errors: [{ kind: "persistence", artifact, message: reason }],
-    };
-  }
-}
-
-async function persistGroupResult(groupDirectory, groupResult) {
-  const artifact = "group.json";
-  try {
-    await writeJson(path.join(groupDirectory, artifact), groupResult);
-    return groupResult;
-  } catch (error) {
-    const message = `failed to persist ${artifact}: ${errorMessage(error)}`;
-    const fallback = "group-error.json";
-    const failed = {
-      ...groupResult,
-      state: groupResult.state === "complete" ? "partially-complete" : groupResult.state,
-      artifacts: { ...groupResult.artifacts, group: fallback },
-      errors: [...groupResult.errors, { kind: "persistence", artifact, message }],
-    };
-    await writeJson(path.join(groupDirectory, fallback), failed);
-    return failed;
-  }
-}
-
-function gradeErrorReport(evidence, reason, grade = null) {
-  return {
-    schema_version: 1,
-    state: "grade-error",
-    evidence,
-    checks: grade?.checks ?? [],
-    comparisons: grade?.comparisons ?? [],
-    errors: [...(grade?.errors ?? []), { id: null, reason }],
-  };
-}
-
-function errorMessage(error) {
-  return error instanceof Error ? error.message : String(error);
-}
-
-async function executeVariant({ group, variant, root, variantDirectory, fixture, runtime, signal }) {
-  try {
-    const run = group.type === "body" ? runBody : runDescription;
-    return await run({ group, variant, root, variantDirectory, fixture, runtime, signal });
-  } catch (error) {
-    if (error instanceof VariantSetupError) return invalidRun(group, variant, error.message);
-    return infrastructureFailedRun(group, variant, error);
-  }
-}
-
-async function persistRunOrFailure(variantDirectory, run) {
-  try {
-    return { run, file: await persistRun(variantDirectory, run) };
-  } catch (error) {
-    const failed = persistenceFailedRun(run, error);
-    const runFile = path.join(variantDirectory, "run.json");
-    await writeJson(runFile, {
-      ...failed,
-      transcript: failed.transcript ?? [],
-      artifacts: {
-        events: failed.process === undefined ? null : "events.jsonl",
-        final: null,
-        stderr: failed.process === undefined ? null : "stderr.log",
-      },
-    });
-    return { run: failed, file: runFile };
-  }
-}
-
-async function persistRun(variantDirectory, run) {
-  const transcript = run.transcript ?? [];
-  const response = run.response ?? "";
-  await writeJson(path.join(variantDirectory, "transcript.json"), transcript);
-  await writeText(path.join(variantDirectory, "final.md"), response === "" ? "" : `${response}\n`);
-  const runFile = path.join(variantDirectory, "run.json");
-  const hasProcessEvidence = run.process !== undefined;
-  const persisted = {
-    ...run,
-    transcript: "transcript.json",
-    artifacts: {
-      events: hasProcessEvidence ? "events.jsonl" : null,
-      final: "final.md",
-      stderr: hasProcessEvidence ? "stderr.log" : null,
-      context: run.contextObservationArtifact ? "context-observations.jsonl" : null,
-    },
-  };
-  if (Array.isArray(run.turns)) persisted.turns = externalizeTurnTranscripts(run.turns);
-  await writeJson(runFile, persisted);
-  return runFile;
-}
-
-function externalizeTurnTranscripts(turns) {
-  let offset = 0;
-  return turns.map((turn) => {
-    const length = Array.isArray(turn.transcript) ? turn.transcript.length : 0;
-    const transcript = { file: "transcript.json", start: offset, end: offset + length };
-    offset += length;
-    return { ...turn, transcript };
-  });
-}
-
-function notSelectedRun(group, variant) {
-  return {
-    schema_version: 1,
-    variant,
-    state: "not-selected",
-    evaluationType: group.type,
-    prompt: group.input.prompt ?? null,
-    prompts: group.input.turns ?? null,
-    activation: unavailableActivation(),
-    response: "",
-    transcript: [],
-    error: null,
-  };
-}
-
-function cancelledRun(group, variant) {
-  return {
-    schema_version: 1,
-    variant,
-    state: "cancelled",
-    evaluationType: group.type,
-    prompt: group.input.prompt ?? null,
-    prompts: group.input.turns ?? null,
-    activation: unavailableActivation(),
-    response: "",
-    transcript: [],
-    error: { kind: "cancellation", message: "Cancelled before subject execution" },
-  };
-}
-
-function invalidRun(group, variant, message) {
-  return {
-    schema_version: 1,
-    variant,
-    state: "invalid",
-    evaluationType: group.type,
-    prompt: group.input.prompt ?? null,
-    prompts: group.input.turns ?? null,
-    activation: unavailableActivation(),
-    response: "",
-    transcript: [],
-    error: { kind: "setup", message },
-  };
-}
-
-function persistenceFailedRun(run, error) {
-  return {
-    ...run,
-    state: "infrastructure-failed",
-    error: {
-      kind: "persistence",
-      message: error instanceof Error ? error.message : String(error),
-      cause: run.error ?? null,
-    },
-  };
-}
-
-function infrastructureFailedRun(group, variant, error) {
-  return {
-    schema_version: 1,
-    variant,
-    state: "infrastructure-failed",
-    evaluationType: group.type,
-    prompt: group.input.prompt ?? null,
-    prompts: group.input.turns ?? null,
-    activation: unavailableActivation(),
-    response: "",
-    transcript: [],
-    error: { kind: "infrastructure", message: error instanceof Error ? error.message : String(error) },
-  };
-}
-
-function unavailableActivation() {
-  return {
-    targetRead: null,
-    firstReadTurn: null,
-    readTurns: [],
-    successfulReadPaths: [],
-  };
-}

package/skills/evaluate-skill/scripts/pi-guard.mjs

@@ -1,164 +0,0 @@
-import { createHash } from "node:crypto";
-import { lstatSync, readFileSync, realpathSync } from "node:fs";
-import path from "node:path";
-
-const PATH_TOOLS = new Set(["read", "write", "edit"]);
-
-function parseStringArray(value) {
-  try {
-    const parsed = JSON.parse(value ?? "[]");
-    return Array.isArray(parsed) && parsed.every((entry) => typeof entry === "string") ? parsed : [];
-  } catch {
-    return [];
-  }
-}
-
-function isContained(root, target) {
-  const relative = path.relative(root, target);
-  return relative === "" || (!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative));
-}
-
-function resolveWritablePath(cwd, requested) {
-  const absolute = path.resolve(cwd, requested);
-  /** @type {string[]} */
-  const missingSegments = [];
-  let candidate = absolute;
-  while (true) {
-    try {
-      return path.join(realpathSync(candidate), ...missingSegments);
-    } catch {
-      try {
-        lstatSync(candidate);
-        throw new Error(`Existing writable path cannot be resolved: ${requested}`);
-      } catch (error) {
-        if (error?.code !== "ENOENT" && error?.code !== "ENOTDIR") throw error;
-      }
-      const parent = path.dirname(candidate);
-      if (parent === candidate) throw new Error(`Cannot resolve writable path: ${requested}`);
-      missingSegments.unshift(path.basename(candidate));
-      candidate = parent;
-    }
-  }
-}
-
-function escapeXml(value) {
-  const entities = { "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&apos;" };
-  return value.replace(/[&<>"']/g, (character) => entities[character]);
-}
-
-function loadDeclaredContext(manifestPath) {
-  if (!manifestPath) return [];
-  let manifest;
-  try {
-    manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
-  } catch (error) {
-    throw new Error(
-      `Declared context manifest cannot be read: ${error instanceof Error ? error.message : String(error)}`,
-    );
-  }
-  if (manifest?.schema_version !== 1 || !Array.isArray(manifest.entries)) {
-    throw new Error("Declared context manifest must use schema_version 1 and contain entries");
-  }
-  for (const entry of manifest.entries) {
-    if (typeof entry?.declaredPath !== "string" || !Array.isArray(entry.files)) {
-      throw new Error("Declared context manifest entry is malformed");
-    }
-    for (const file of entry.files) {
-      if (
-        typeof file?.path !== "string" ||
-        typeof file.sha256 !== "string" ||
-        typeof file.content !== "string" ||
-        createHash("sha256").update(file.content).digest("hex") !== file.sha256
-      ) {
-        throw new Error(`Declared context file is malformed or changed: ${file?.path ?? "unknown"}`);
-      }
-    }
-  }
-  return manifest.entries;
-}
-
-function buildSubjectSystemPrompt(options, declaredContext) {
-  const tools = options.selectedTools ?? [];
-  const toolLines = tools.map((tool) => `- ${tool}: ${options.toolSnippets?.[tool] ?? "Declared evaluation tool"}`);
-  const lines = [
-    "You are an expert coding assistant operating inside pi, a coding agent harness.",
-    "Help the user with the task using only the tools and resources listed below.",
-    "",
-    "Available tools:",
-    ...(toolLines.length > 0 ? toolLines : ["(none)"]),
-    "",
-    "Guidelines:",
-    "- Use read to load a skill's SKILL.md when the task matches its description.",
-    "- Resolve relative skill resources from the skill directory.",
-    "- Be concise in your response.",
-  ];
-  const skills = (options.skills ?? []).filter((skill) => !skill.disableModelInvocation);
-  if (skills.length > 0) {
-    lines.push("", "<available_skills>");
-    for (const skill of skills) {
-      lines.push(
-        "  <skill>",
-        `    <name>${escapeXml(skill.name)}</name>`,
-        `    <description>${escapeXml(skill.description)}</description>`,
-        `    <location>${escapeXml(skill.filePath)}</location>`,
-        "  </skill>",
-      );
-    }
-    lines.push("</available_skills>");
-  }
-  if (declaredContext.length > 0) {
-    lines.push("", "<declared_context>");
-    for (const entry of declaredContext) {
-      lines.push(`  <context declared-path="${escapeXml(entry.declaredPath)}">`);
-      for (const file of entry.files) {
-        lines.push(`    <file path="${escapeXml(file.path)}">\n${file.content}</file>`);
-      }
-      lines.push("  </context>");
-    }
-    lines.push("</declared_context>");
-  }
-  lines.push(`Current working directory: ${options.cwd}`);
-  return lines.join("\n");
-}
-
-export default function registerSkillEvalGuard(pi) {
-  const allowedRoots = parseStringArray(process.env.SKILL_EVAL_ALLOWED_ROOTS).map((root) => realpathSync(root));
-  const writableRootValue = process.env.SKILL_EVAL_WRITABLE_ROOT;
-  if (!writableRootValue) throw new Error("SKILL_EVAL_WRITABLE_ROOT is required");
-  const writableRoot = realpathSync(writableRootValue);
-  const allowedTools = new Set(parseStringArray(process.env.SKILL_EVAL_ALLOWED_TOOLS));
-  const declaredContext = loadDeclaredContext(process.env.SKILL_EVAL_CONTEXT_MANIFEST);
-
-  pi.on("before_agent_start", (event) => ({
-    systemPrompt: buildSubjectSystemPrompt(event.systemPromptOptions, declaredContext),
-  }));
-
-  pi.on("tool_call", (event, ctx) => {
-    if (!allowedTools.has(event.toolName)) {
-      return { block: true, reason: `Tool is not declared for this evaluation: ${event.toolName}` };
-    }
-    if (!PATH_TOOLS.has(event.toolName)) return undefined;
-    if (typeof event.input?.path !== "string") {
-      return { block: true, reason: `${event.toolName} requires a path inside the evaluation environment` };
-    }
-
-    const requested = event.input.path.startsWith("@") ? event.input.path.slice(1) : event.input.path;
-    let canonical;
-    try {
-      canonical =
-        event.toolName === "write"
-          ? resolveWritablePath(ctx.cwd, requested)
-          : realpathSync(path.resolve(ctx.cwd, requested));
-    } catch {
-      return {
-        block: true,
-        reason: `${event.toolName} path cannot be resolved inside the evaluation environment: ${requested}`,
-      };
-    }
-    const roots = event.toolName === "read" ? allowedRoots : [writableRoot];
-    if (!roots.some((root) => isContained(root, canonical))) {
-      return { block: true, reason: `${event.toolName} path escapes the evaluation environment: ${requested}` };
-    }
-    return undefined;
-  });
-}

package/skills/evaluate-skill/scripts/skill-eval.mjs

@@ -1,102 +0,0 @@
-#!/usr/bin/env node
-
-import { runEvaluation } from "./lib/runner.mjs";
-import { renderResult } from "./lib/view.mjs";
-
-const commands = new Set(["run", "view"]);
-const argv = process.argv.slice(2);
-const [command] = argv;
-
-function printUsage() {
-  process.stdout.write(
-    `Usage:
-  skill-eval run <suite-or-group-path> [options]
-  skill-eval view <result-id-or-directory> [options]
-
-Commands:
-  run    Execute selected evaluation groups
-  view   Render selected stored evidence
-
-Selectors:
-  --group <id-or-position>
-  --variant <baseline|candidate>
-
-Paths:
-  Definition paths resolve from the current working directory.
-  Suite group references resolve relative to the suite file.
-  Results are stored under <cwd>/.skill-eval/runs/<result-id>.
-  view accepts a stored result ID or an explicit result directory.
-
-Selection:
-  No selectors choose every suite group and both variants.
-  --group is invalid for a direct group definition or result.
-
-Current run support:
-  Description and explicit-body groups with prompt or ordered turns
-  Working-tree or Git-backed ordered skills/context; optional fresh fixture copies
-  Description tools: read; body tools: read, write, edit, or declared extension tools
-  Runtime profiles: Pi or PiFlow hosts, ordered immutable extension bundles, environment overrides
-  Deterministic reads, paths, changed paths, text, JSON, tool calls, context observations, and factual comparisons
-  Ordered suites run serially and continue after isolated variant, group, or post-processing failures
-  Grade-first views show compact criterion details, usage, and result-relative artifact paths
-`,
-  );
-}
-
-function fail(message, exitCode = 2) {
-  process.stderr.write(`${message}\n`);
-  process.exitCode = exitCode;
-}
-
-function parseOperation(values) {
-  const [target, ...options] = values;
-  if (!target) throw new Error(`skill-eval ${command} requires a target`);
-  let group = null;
-  let variant = null;
-  for (let index = 0; index < options.length; index += 1) {
-    const option = options[index];
-    const value = options[index + 1];
-    if (option !== "--group" && option !== "--variant") throw new Error(`Unknown option: ${option}`);
-    if (!value || value.startsWith("--")) throw new Error(`${option} requires a value`);
-    if (option === "--group") {
-      if (group !== null) throw new Error("--group may be supplied only once");
-      group = value;
-    } else {
-      if (variant !== null) throw new Error("--variant may be supplied only once");
-      variant = value;
-    }
-    index += 1;
-  }
-  return { target, selectors: { group, variant } };
-}
-
-if (!command || command === "--help" || command === "-h") {
-  printUsage();
-} else if (commands.has(command)) {
-  try {
-    const operation = parseOperation(argv.slice(1));
-    if (command === "view") {
-      process.stdout.write(await renderResult(operation.target, operation.selectors, { root: process.cwd() }));
-    } else {
-      const controller = new AbortController();
-      const abort = () => controller.abort();
-      process.once("SIGINT", abort);
-      process.once("SIGTERM", abort);
-      try {
-        const result = await runEvaluation(operation.target, operation.selectors, {
-          root: process.cwd(),
-          signal: controller.signal,
-        });
-        process.stdout.write(`Result: ${result.id}\nPath: ${result.path}\nState: ${result.state}\n`);
-        if (result.state !== "complete") process.exitCode = 1;
-      } finally {
-        process.removeListener("SIGINT", abort);
-        process.removeListener("SIGTERM", abort);
-      }
-    }
-  } catch (error) {
-    fail(error instanceof Error ? error.message : String(error), 1);
-  }
-} else {
-  fail(`Unknown command: ${command}`);
-}