@hassangameryt/freeflow 0.2.0 → 0.4.0

package/skills/evaluate-skill/scripts/lib/view.mjs

@@ -0,0 +1,337 @@
+import { readFile, realpath, stat } from "node:fs/promises";
+import path from "node:path";
+
+const VARIANTS = ["baseline", "candidate"];
+const DEFAULT_SELECTORS = /** @type {{group: string | null, variant: string | null}} */ ({
+  group: null,
+  variant: null,
+});
+
+class ViewError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "ViewError";
+  }
+}
+
+/**
+ * @param {string} target
+ * @param {{group: string | null, variant: string | null}} [selectors]
+ * @param {{root?: string}} [options]
+ */
+
+export async function renderResult(target, selectors = DEFAULT_SELECTORS, { root = process.cwd() } = {}) {
+  const resultDirectory = await resolveResult(target, root);
+  const summary = await readJson(path.join(resultDirectory, "summary.json"), "result summary");
+  let groups = Array.isArray(summary.groups) ? summary.groups : [];
+  if (summary.definitionKind === "group" && selectors.group !== null) {
+    throw new ViewError("--group cannot be used with a direct group result");
+  }
+  if (selectors.group !== null) {
+    const position = /^\d+$/.test(String(selectors.group)) ? Number(selectors.group) : null;
+    groups =
+      position === null
+        ? groups.filter((group) => group.id === selectors.group)
+        : groups.filter((group) => group.position === position);
+    if (groups.length === 0) throw new ViewError(`Unknown group selector: ${selectors.group}`);
+  }
+  if (selectors.variant !== null && !VARIANTS.includes(selectors.variant)) {
+    throw new ViewError("variant must be baseline or candidate");
+  }
+
+  const lines = [`Result ${summary.id} [${summary.state}]`, `Path: ${escapeCell(resultDirectory)}`];
+  for (const group of groups) {
+    lines.push("", ...(await renderGroup(resultDirectory, group, selectors.variant)));
+  }
+  return `${lines.join("\n")}\n`;
+}
+
+async function renderGroup(resultDirectory, group, selectedVariant) {
+  const groupDirectory = path.join(resultDirectory, "groups", group.id);
+  const definition = await readJson(path.join(groupDirectory, "definition.json"), `definition for ${group.id}`);
+  const gradeFile = path.join(groupDirectory, group.artifacts?.grade ?? "deterministic-grade.json");
+  const grade = await readJson(gradeFile, `grade for ${group.id}`);
+  const variants = selectedVariant === null ? VARIANTS : [selectedVariant];
+  const lines = [`Group ${group.id} [${group.state}] type=${definition.type}`];
+
+  appendSharedEvidence(lines, definition);
+  appendGradeEvidence(lines, grade, group, definition, selectedVariant);
+  for (const variant of variants) {
+    await appendVariantEvidence(lines, resultDirectory, groupDirectory, group.id, variant);
+  }
+  lines.push(
+    compactRow("artifacts", [
+      `definition=${relativeArtifact(resultDirectory, path.join(groupDirectory, "definition.json"))}`,
+      `grade=${relativeArtifact(resultDirectory, gradeFile)}`,
+      `group=${relativeArtifact(resultDirectory, path.join(groupDirectory, group.artifacts?.group ?? "group.json"))}`,
+    ]),
+  );
+  return lines;
+}
+
+function appendSharedEvidence(lines, definition) {
+  if (typeof definition.input?.prompt === "string") {
+    appendLabeledText(lines, "Prompt", definition.input.prompt);
+  } else {
+    const prompts = Array.isArray(definition.input?.turns) ? definition.input.turns : [];
+    lines.push(`Turns: ${prompts.length}`);
+    for (const [index, prompt] of prompts.entries()) {
+      appendLabeledText(lines, `  Turn ${index + 1} prompt`, prompt, "    ");
+    }
+  }
+  const reviewQuestions = Array.isArray(definition.review_questions) ? definition.review_questions : [];
+  if (reviewQuestions.length > 0) {
+    lines.push("Review questions:");
+    for (const question of reviewQuestions) lines.push(`  - ${question}`);
+  }
+}
+
+function appendGradeEvidence(lines, grade, group, definition, selectedVariant) {
+  const checks = Array.isArray(grade.checks)
+    ? grade.checks.filter((check) => selectedVariant === null || check.variant === selectedVariant)
+    : [];
+  const gradeErrors = selectedGradeErrors(grade.errors, definition, selectedVariant);
+  lines.push(`Grade [${selectedGradeState(grade.state, checks, gradeErrors, selectedVariant)}]`);
+  if (checks.length === 0) lines.push("  (no selected checks)");
+  else {
+    for (const check of checks) {
+      lines.push(
+        compactRow("check", [
+          check.id,
+          check.variant,
+          check.kind,
+          check.state,
+          `expected=${compactJson(check.expected)}`,
+          `observed=${compactObserved(check)}`,
+        ]),
+      );
+    }
+  }
+  if (gradeErrors.length > 0) {
+    lines.push("Grade errors:");
+    for (const error of gradeErrors) {
+      lines.push(compactRow("error", ["grade", error.id ?? "system", error.reason]));
+    }
+  }
+  if (Array.isArray(group.errors) && group.errors.length > 0) {
+    lines.push("Group errors:");
+    for (const error of group.errors) {
+      lines.push(compactRow("error", ["group", error.artifact ?? error.kind, error.message]));
+    }
+  }
+  if (selectedVariant === null && Array.isArray(grade.comparisons) && grade.comparisons.length > 0) {
+    lines.push("Comparisons:");
+    for (const comparison of grade.comparisons) {
+      lines.push(
+        compactRow("comparison", [
+          comparison.id,
+          comparison.kind,
+          comparison.transition,
+          `baseline=${comparison.baseline.check}:${comparison.baseline.state}`,
+          `candidate=${comparison.candidate.check}:${comparison.candidate.state}`,
+        ]),
+      );
+    }
+  }
+}
+
+function selectedGradeErrors(errors, definition, selectedVariant) {
+  if (!Array.isArray(errors)) return [];
+  if (selectedVariant === null) return errors;
+  const expectations = /** @type {Array<any>} */ (
+    Array.isArray(definition.expectations) ? definition.expectations : []
+  );
+  const expectationById = new Map(expectations.map((expectation) => [expectation.id, expectation]));
+  const comparisonIds = new Set(
+    expectations.map((expectation) => expectation.comparison).filter((comparison) => typeof comparison === "string"),
+  );
+  return errors.filter((error) => {
+    if (error.id === "system" || comparisonIds.has(error.id)) return true;
+    const expectation = /** @type {any} */ (expectationById.get(error.id));
+    return expectation === undefined || expectation.variant === selectedVariant;
+  });
+}
+
+function selectedGradeState(state, checks, errors, selectedVariant) {
+  if (selectedVariant === null) return state;
+  if (errors.length > 0) return "grade-error";
+  if (checks.length > 0 && checks.every((check) => check.state === "unavailable")) return "unavailable";
+  return "complete";
+}
+
+async function appendVariantEvidence(lines, resultDirectory, groupDirectory, groupId, variant) {
+  const runFile = path.join(groupDirectory, variant, "run.json");
+  const run = await readJson(runFile, `${variant} run for ${groupId}`);
+  lines.push("", `${title(variant)} [${run.state}]`);
+  appendRunEvidence(lines, run);
+  const variantDirectory = path.dirname(runFile);
+  const artifacts = [["run", relativeArtifact(resultDirectory, runFile)]];
+  if (typeof run.workspace === "string") {
+    artifacts.push(["workspace", relativeArtifact(resultDirectory, run.workspace)]);
+  }
+  if (typeof run.transcript === "string") {
+    artifacts.push(["transcript", relativeArtifact(resultDirectory, path.resolve(variantDirectory, run.transcript))]);
+  }
+  for (const [name, artifact] of Object.entries(run.artifacts ?? {})) {
+    if (typeof artifact === "string") {
+      artifacts.push([name, relativeArtifact(resultDirectory, path.resolve(variantDirectory, artifact))]);
+    }
+  }
+  lines.push(
+    compactRow(
+      "  artifacts",
+      artifacts.map(([name, artifact]) => `${name}=${artifact}`),
+    ),
+  );
+}
+
+async function resolveResult(target, root) {
+  const explicit = path.resolve(root, target);
+  const stored = path.join(root, ".skill-eval", "runs", target);
+  const isPath = path.isAbsolute(target) || target.includes("/") || target.includes("\\") || target.startsWith(".");
+  const candidates = isPath ? [explicit] : [stored, explicit];
+  for (const candidate of candidates) {
+    const canonical = await realpath(candidate).catch(() => null);
+    if (canonical === null) continue;
+    const candidateStat = await stat(canonical).catch(() => null);
+    if (candidateStat?.isDirectory()) return canonical;
+  }
+  throw new ViewError(`Cannot find result: ${target}`);
+}
+
+async function readJson(file, label) {
+  let contents;
+  try {
+    contents = await readFile(file, "utf8");
+  } catch {
+    throw new ViewError(`Cannot read ${label}: ${file}`);
+  }
+  try {
+    return JSON.parse(contents);
+  } catch {
+    throw new ViewError(`${label} is not valid JSON: ${file}`);
+  }
+}
+
+function title(value) {
+  return `${value[0].toUpperCase()}${value.slice(1)}`;
+}
+
+function appendRunEvidence(lines, run) {
+  const turns = Array.isArray(run.turns) ? run.turns : null;
+  const hasObservedTurns = turns !== null && turns.length > 0;
+  if (run.evaluationType === "description" && (run.state === "complete" || hasObservedTurns)) {
+    const prefix = run.state === "complete" ? "" : "observed ";
+    lines.push(
+      `  ${prefix}target-read: ${run.activation?.targetRead === true ? "yes" : "no"}`,
+      `  ${prefix}read-turns: ${(run.activation?.readTurns ?? []).join(",") || "none"}`,
+    );
+  }
+  if (run.evaluationType === "body" && run.delivery) {
+    lines.push(`  delivery: ${run.delivery.kind} on turn ${run.delivery.turn}`);
+    const activity = turns === null ? (run.toolActivity ?? []) : turns.flatMap((turn) => turn.toolActivity ?? []);
+    const toolsUsed = [...new Set(activity.map((entry) => entry.toolName).filter(Boolean))];
+    lines.push(`  tools-used: ${toolsUsed.join(",") || "none"}`);
+    const changes = run.effects?.changes;
+    if (changes) {
+      appendChangedPaths(lines, "created", changes.created);
+      appendChangedPaths(lines, "modified", changes.modified);
+      appendChangedPaths(lines, "deleted", changes.deleted);
+    }
+  }
+  if (turns !== null) {
+    for (const turn of turns) {
+      const labels = [turn.settled ? "settled" : "unsettled"];
+      if (turn.targetRead === true) labels.push("target-read");
+      lines.push(`  Turn ${turn.turn} [${labels.join(", ")}]`);
+      appendResponse(lines, turn.response ?? "", "    ");
+    }
+  } else if (run.state === "complete") {
+    lines.push("  response:", indent(run.response ?? ""));
+  }
+  if (run.usage && typeof run.usage === "object") {
+    const usage = [
+      `input=${formatMetric(run.usage.input)}`,
+      `output=${formatMetric(run.usage.output)}`,
+      `cache-read=${formatMetric(run.usage.cacheRead)}`,
+      `cache-write=${formatMetric(run.usage.cacheWrite)}`,
+    ];
+    if (typeof run.usage.cost?.total === "number") usage.push(`cost=${formatNumber(run.usage.cost.total)}`);
+    lines.push(compactRow("  usage", usage));
+  }
+  if (run.process?.assistantError) lines.push(`  assistant-error: ${run.process.assistantError}`);
+  if (run.process?.terminationReason) {
+    lines.push(`  termination: ${run.process.terminationReason}`);
+  }
+  if (Array.isArray(run.process?.protocolErrors) && run.process.protocolErrors.length > 0) {
+    lines.push(`  protocol-errors: ${run.process.protocolErrors.length}`);
+  }
+  if (run.error?.message) lines.push(`  error: ${run.error.message}`);
+}
+
+function appendChangedPaths(lines, kind, paths) {
+  lines.push(compactRow("  changes", [kind, ...(Array.isArray(paths) ? paths : [])]));
+}
+
+function appendResponse(lines, response, prefix) {
+  const value = String(response);
+  if (!value.includes("\n")) {
+    lines.push(`${prefix}response: ${value}`);
+    return;
+  }
+  lines.push(`${prefix}response:`, indentWith(value, `${prefix}  `));
+}
+
+function indentWith(value, prefix) {
+  return String(value)
+    .split("\n")
+    .map((line) => `${prefix}${line}`)
+    .join("\n");
+}
+
+function indent(value) {
+  return String(value)
+    .split("\n")
+    .map((line) => `    ${line}`)
+    .join("\n");
+}
+
+function appendLabeledText(lines, label, value, blockPrefix = "    ") {
+  const text = String(value);
+  if (!text.includes("\n")) {
+    lines.push(`${label}: ${text}`);
+    return;
+  }
+  lines.push(`${label}:`, indentWith(text, blockPrefix));
+}
+
+function compactRow(kind, values) {
+  return [kind, ...values].map(escapeCell).join("\t");
+}
+
+function compactJson(value) {
+  return value === undefined ? "-" : JSON.stringify(value);
+}
+
+function compactObserved(check) {
+  if (check.kind === "response-text" && typeof check.observed?.response === "string") {
+    return `response@turn:${check.observed.turn ?? "final"}`;
+  }
+  return compactJson(check.observed);
+}
+
+function formatNumber(value) {
+  return Number.isInteger(value) ? String(value) : String(Number(value.toFixed(8)));
+}
+
+function formatMetric(value) {
+  return typeof value === "number" && Number.isFinite(value) ? formatNumber(value) : "unavailable";
+}
+
+function escapeCell(value) {
+  return String(value).replace(/\\/g, "\\\\").replace(/\r/g, "\\r").replace(/\n/g, "\\n").replace(/\t/g, "\\t");
+}
+
+function relativeArtifact(resultDirectory, file) {
+  return path.relative(resultDirectory, file).split(path.sep).join("/");
+}

package/skills/evaluate-skill/scripts/pi-guard.mjs

@@ -0,0 +1,164 @@
+import { createHash } from "node:crypto";
+import { lstatSync, readFileSync, realpathSync } from "node:fs";
+import path from "node:path";
+
+const PATH_TOOLS = new Set(["read", "write", "edit"]);
+
+function parseStringArray(value) {
+  try {
+    const parsed = JSON.parse(value ?? "[]");
+    return Array.isArray(parsed) && parsed.every((entry) => typeof entry === "string") ? parsed : [];
+  } catch {
+    return [];
+  }
+}
+
+function isContained(root, target) {
+  const relative = path.relative(root, target);
+  return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
+}
+
+function resolveWritablePath(cwd, requested) {
+  const absolute = path.resolve(cwd, requested);
+  /** @type {string[]} */
+  const missingSegments = [];
+  let candidate = absolute;
+  while (true) {
+    try {
+      return path.join(realpathSync(candidate), ...missingSegments);
+    } catch {
+      try {
+        lstatSync(candidate);
+        throw new Error(`Existing writable path cannot be resolved: ${requested}`);
+      } catch (error) {
+        if (error?.code !== "ENOENT" && error?.code !== "ENOTDIR") throw error;
+      }
+      const parent = path.dirname(candidate);
+      if (parent === candidate) throw new Error(`Cannot resolve writable path: ${requested}`);
+      missingSegments.unshift(path.basename(candidate));
+      candidate = parent;
+    }
+  }
+}
+
+function escapeXml(value) {
+  const entities = { "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&apos;" };
+  return value.replace(/[&<>"']/g, (character) => entities[character]);
+}
+
+function loadDeclaredContext(manifestPath) {
+  if (!manifestPath) return [];
+  let manifest;
+  try {
+    manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
+  } catch (error) {
+    throw new Error(
+      `Declared context manifest cannot be read: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+  if (manifest?.schema_version !== 1 || !Array.isArray(manifest.entries)) {
+    throw new Error("Declared context manifest must use schema_version 1 and contain entries");
+  }
+  for (const entry of manifest.entries) {
+    if (typeof entry?.declaredPath !== "string" || !Array.isArray(entry.files)) {
+      throw new Error("Declared context manifest entry is malformed");
+    }
+    for (const file of entry.files) {
+      if (
+        typeof file?.path !== "string" ||
+        typeof file.sha256 !== "string" ||
+        typeof file.content !== "string" ||
+        createHash("sha256").update(file.content).digest("hex") !== file.sha256
+      ) {
+        throw new Error(`Declared context file is malformed or changed: ${file?.path ?? "unknown"}`);
+      }
+    }
+  }
+  return manifest.entries;
+}
+
+function buildSubjectSystemPrompt(options, declaredContext) {
+  const tools = options.selectedTools ?? [];
+  const toolLines = tools.map((tool) => `- ${tool}: ${options.toolSnippets?.[tool] ?? "Declared evaluation tool"}`);
+  const lines = [
+    "You are an expert coding assistant operating inside pi, a coding agent harness.",
+    "Help the user with the task using only the tools and resources listed below.",
+    "",
+    "Available tools:",
+    ...(toolLines.length > 0 ? toolLines : ["(none)"]),
+    "",
+    "Guidelines:",
+    "- Use read to load a skill's SKILL.md when the task matches its description.",
+    "- Resolve relative skill resources from the skill directory.",
+    "- Be concise in your response.",
+  ];
+  const skills = (options.skills ?? []).filter((skill) => !skill.disableModelInvocation);
+  if (skills.length > 0) {
+    lines.push("", "<available_skills>");
+    for (const skill of skills) {
+      lines.push(
+        "  <skill>",
+        `    <name>${escapeXml(skill.name)}</name>`,
+        `    <description>${escapeXml(skill.description)}</description>`,
+        `    <location>${escapeXml(skill.filePath)}</location>`,
+        "  </skill>",
+      );
+    }
+    lines.push("</available_skills>");
+  }
+  if (declaredContext.length > 0) {
+    lines.push("", "<declared_context>");
+    for (const entry of declaredContext) {
+      lines.push(`  <context declared-path="${escapeXml(entry.declaredPath)}">`);
+      for (const file of entry.files) {
+        lines.push(`    <file path="${escapeXml(file.path)}">\n${file.content}</file>`);
+      }
+      lines.push("  </context>");
+    }
+    lines.push("</declared_context>");
+  }
+  lines.push(`Current working directory: ${options.cwd}`);
+  return lines.join("\n");
+}
+
+export default function registerSkillEvalGuard(pi) {
+  const allowedRoots = parseStringArray(process.env.SKILL_EVAL_ALLOWED_ROOTS).map((root) => realpathSync(root));
+  const writableRootValue = process.env.SKILL_EVAL_WRITABLE_ROOT;
+  if (!writableRootValue) throw new Error("SKILL_EVAL_WRITABLE_ROOT is required");
+  const writableRoot = realpathSync(writableRootValue);
+  const allowedTools = new Set(parseStringArray(process.env.SKILL_EVAL_ALLOWED_TOOLS));
+  const declaredContext = loadDeclaredContext(process.env.SKILL_EVAL_CONTEXT_MANIFEST);
+
+  pi.on("before_agent_start", (event) => ({
+    systemPrompt: buildSubjectSystemPrompt(event.systemPromptOptions, declaredContext),
+  }));
+
+  pi.on("tool_call", (event, ctx) => {
+    if (!allowedTools.has(event.toolName)) {
+      return { block: true, reason: `Tool is not declared for this evaluation: ${event.toolName}` };
+    }
+    if (!PATH_TOOLS.has(event.toolName)) return undefined;
+    if (typeof event.input?.path !== "string") {
+      return { block: true, reason: `${event.toolName} requires a path inside the evaluation environment` };
+    }
+
+    const requested = event.input.path.startsWith("@") ? event.input.path.slice(1) : event.input.path;
+    let canonical;
+    try {
+      canonical =
+        event.toolName === "write"
+          ? resolveWritablePath(ctx.cwd, requested)
+          : realpathSync(path.resolve(ctx.cwd, requested));
+    } catch {
+      return {
+        block: true,
+        reason: `${event.toolName} path cannot be resolved inside the evaluation environment: ${requested}`,
+      };
+    }
+    const roots = event.toolName === "read" ? allowedRoots : [writableRoot];
+    if (!roots.some((root) => isContained(root, canonical))) {
+      return { block: true, reason: `${event.toolName} path escapes the evaluation environment: ${requested}` };
+    }
+    return undefined;
+  });
+}

package/skills/evaluate-skill/scripts/skill-eval.mjs

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+
+import { runEvaluation } from "./lib/runner.mjs";
+import { renderResult } from "./lib/view.mjs";
+
+const commands = new Set(["run", "view"]);
+const argv = process.argv.slice(2);
+const [command] = argv;
+
+function printUsage() {
+  process.stdout.write(
+    `Usage:
+  skill-eval run <suite-or-group-path> [options]
+  skill-eval view <result-id-or-directory> [options]
+
+Commands:
+  run    Execute selected evaluation groups
+  view   Render selected stored evidence
+
+Selectors:
+  --group <id-or-position>
+  --variant <baseline|candidate>
+
+Paths:
+  Definition paths resolve from the current working directory.
+  Suite group references resolve relative to the suite file.
+  Results are stored under <cwd>/.skill-eval/runs/<result-id>.
+  view accepts a stored result ID or an explicit result directory.
+
+Selection:
+  No selectors choose every suite group and both variants.
+  --group is invalid for a direct group definition or result.
+
+Current run support:
+  Description and explicit-body groups with prompt or ordered turns
+  Working-tree or Git-backed ordered skills/context; optional fresh fixture copies
+  Description tools: read; body tools: read, write, edit
+  Deterministic reads, paths, changed paths, text, JSON, and factual comparisons
+  Ordered suites run serially and continue after isolated variant, group, or post-processing failures
+  Grade-first views show compact criterion details, usage, and result-relative artifact paths
+`,
+  );
+}
+
+function fail(message, exitCode = 2) {
+  process.stderr.write(`${message}\n`);
+  process.exitCode = exitCode;
+}
+
+function parseOperation(values) {
+  const [target, ...options] = values;
+  if (!target) throw new Error(`skill-eval ${command} requires a target`);
+  let group = null;
+  let variant = null;
+  for (let index = 0; index < options.length; index += 1) {
+    const option = options[index];
+    const value = options[index + 1];
+    if (option !== "--group" && option !== "--variant") throw new Error(`Unknown option: ${option}`);
+    if (!value || value.startsWith("--")) throw new Error(`${option} requires a value`);
+    if (option === "--group") {
+      if (group !== null) throw new Error("--group may be supplied only once");
+      group = value;
+    } else {
+      if (variant !== null) throw new Error("--variant may be supplied only once");
+      variant = value;
+    }
+    index += 1;
+  }
+  return { target, selectors: { group, variant } };
+}
+
+if (!command || command === "--help" || command === "-h") {
+  printUsage();
+} else if (!commands.has(command)) {
+  fail(`Unknown command: ${command}`);
+} else {
+  try {
+    const operation = parseOperation(argv.slice(1));
+    if (command === "view") {
+      process.stdout.write(await renderResult(operation.target, operation.selectors, { root: process.cwd() }));
+    } else {
+      const controller = new AbortController();
+      const abort = () => controller.abort();
+      process.once("SIGINT", abort);
+      process.once("SIGTERM", abort);
+      try {
+        const result = await runEvaluation(operation.target, operation.selectors, {
+          root: process.cwd(),
+          signal: controller.signal,
+        });
+        process.stdout.write(`Result: ${result.id}\nPath: ${result.path}\nState: ${result.state}\n`);
+        if (result.state !== "complete") process.exitCode = 1;
+      } finally {
+        process.removeListener("SIGINT", abort);
+        process.removeListener("SIGTERM", abort);
+      }
+    }
+  } catch (error) {
+    fail(error instanceof Error ? error.message : String(error), 1);
+  }
+}