@hassangameryt/freeflow 0.3.0 → 0.5.0

package/skills/evaluate-skill/scripts/lib/grade-interaction.mjs

@@ -0,0 +1,148 @@
+import { realpath } from "node:fs/promises";
+import path from "node:path";
+
+import {
+  completedCheck,
+  hasOwn,
+  isContained,
+  isSafeRelativePath,
+  readPaths,
+  RESPONSE_EXPECTATIONS,
+  textMatches,
+  unavailableCheck,
+  validBodyExpectation,
+  validTurn,
+  VARIANTS,
+} from "./grade-support.mjs";
+
+const ACTIVATION_EXPECTATIONS = new Set(["never", "on-turn", "by-turn", "not-before-turn"]);
+
+export async function gradeSkillRead(group, runs, expectation) {
+  const turn = expectation.turn ?? null;
+  const timingExpectation = expectation.expect !== "never";
+  if (
+    group.type !== "description" ||
+    !VARIANTS.includes(expectation.variant) ||
+    !ACTIVATION_EXPECTATIONS.has(expectation.expect) ||
+    (timingExpectation ? turn === null || !validTurn(group, turn) : hasOwn(expectation, "turn"))
+  ) {
+    return { error: "invalid skill-read expectation" };
+  }
+
+  const expected = { read: expectation.expect, turn };
+  const run = runs[expectation.variant];
+  if (!run || run.state !== "complete") return unavailableCheck(expectation, expected);
+
+  const observed = {
+    targetRead: run.activation.targetRead,
+    firstReadTurn: run.activation.firstReadTurn,
+    readTurns: run.activation.readTurns,
+  };
+  return completedCheck(expectation, expected, observed, activationMatches(expectation, observed));
+}
+
+export async function gradeResourceRead(group, runs, expectation) {
+  const turn = expectation.turn ?? null;
+  const indexed = expectation.resource === "skill" || expectation.resource === "context";
+  if (
+    !VARIANTS.includes(expectation.variant) ||
+    !["workspace", "skill", "context"].includes(expectation.resource) ||
+    !["read", "not-read"].includes(expectation.expect) ||
+    typeof expectation.path !== "string" ||
+    !isSafeRelativePath(expectation.path) ||
+    !validTurn(group, turn) ||
+    (indexed && (!Number.isInteger(expectation.index) || expectation.index < 0)) ||
+    (!indexed && hasOwn(expectation, "index"))
+  ) {
+    return { error: "invalid resource-read expectation" };
+  }
+  const expected = {
+    expect: expectation.expect,
+    resource: expectation.resource,
+    index: indexed ? expectation.index : null,
+    path: expectation.path,
+    turn,
+  };
+  const run = runs[expectation.variant];
+  if (!run || run.state !== "complete") return unavailableCheck(expectation, expected);
+  const reads = readPaths(run, turn);
+  if (reads === null) return unavailableCheck(expectation, expected);
+
+  const resources = /** @type {any} */ (run.resources);
+  let resource = null;
+  if (expectation.resource === "skill") resource = resources?.skills?.[expectation.index];
+  if (expectation.resource === "context") resource = resources?.context?.[expectation.index];
+  const root = expectation.resource === "workspace" ? run.workspace : resource?.path;
+  if (typeof root !== "string") return { error: "declared resource is unavailable" };
+  if (resource !== null && !resource.files?.some((file) => file.path === expectation.path)) {
+    return { error: "declared resource path is unavailable" };
+  }
+
+  const canonicalRoot = await realpath(root);
+  const declaredPath = path.resolve(canonicalRoot, expectation.path);
+  const canonicalPath = await realpath(declaredPath).catch(() => declaredPath);
+  if (!isContained(canonicalRoot, canonicalPath)) return { error: "resource-read path escapes its declared root" };
+  if (
+    expectation.resource === "workspace" &&
+    !run.effects?.before?.some((file) => file.path === expectation.path) &&
+    !run.effects?.after?.some((file) => file.path === expectation.path) &&
+    !reads.includes(canonicalPath)
+  ) {
+    return { error: "declared workspace resource path is unavailable" };
+  }
+
+  const observed = { path: canonicalPath, read: reads.includes(canonicalPath), turn };
+  return completedCheck(
+    expectation,
+    expected,
+    observed,
+    expectation.expect === "read" ? observed.read : !observed.read,
+  );
+}
+
+export async function gradeResponseText(group, runs, expectation) {
+  const turn = expectation.turn ?? null;
+  if (
+    !validBodyExpectation(group, expectation, turn) ||
+    !RESPONSE_EXPECTATIONS.has(expectation.expect) ||
+    typeof expectation.value !== "string"
+  ) {
+    return { error: "invalid response-text expectation" };
+  }
+
+  const expected = { expect: expectation.expect, value: expectation.value, turn };
+  const run = runs[expectation.variant];
+  if (!run || run.state !== "complete") return unavailableCheck(expectation, expected);
+  const observed = responseObservation(run, turn);
+  if (observed === null) return unavailableCheck(expectation, expected);
+  return completedCheck(
+    expectation,
+    expected,
+    observed,
+    textMatches(expectation.expect, expectation.value, observed.response),
+  );
+}
+
+function activationMatches(expectation, observed) {
+  if (expectation.expect === "never") return !observed.targetRead;
+  if (!Number.isInteger(expectation.turn) || expectation.turn < 1) return false;
+  if (expectation.expect === "on-turn") return observed.readTurns.includes(expectation.turn);
+  if (expectation.expect === "by-turn") {
+    return observed.firstReadTurn !== null && observed.firstReadTurn <= expectation.turn;
+  }
+  return observed.firstReadTurn === null || observed.firstReadTurn >= expectation.turn;
+}
+
+function responseObservation(run, turn) {
+  if (turn === null) {
+    const observedTurn = Array.isArray(run.turns) && run.turns.length > 0 ? run.turns.at(-1).turn : 1;
+    return { response: typeof run.response === "string" ? run.response : "", turn: observedTurn };
+  }
+  if (!Array.isArray(run.turns)) {
+    if (turn !== 1) return null;
+    return { response: typeof run.response === "string" ? run.response : "", turn };
+  }
+  const observedTurn = run.turns.find((entry) => entry.turn === turn);
+  if (!observedTurn) return null;
+  return { response: observedTurn.response ?? "", turn };
+}

package/skills/evaluate-skill/scripts/lib/grade-support.mjs

@@ -0,0 +1,126 @@
+import { createHash } from "node:crypto";
+import { lstat, readFile, realpath } from "node:fs/promises";
+import path from "node:path";
+
+export const VARIANTS = ["baseline", "candidate"];
+export const RESPONSE_EXPECTATIONS = new Set(["contains", "not-contains", "equals"]);
+
+export function hasOwn(value, key) {
+  return Object.getOwnPropertyDescriptor(value, key) !== undefined;
+}
+
+export function validBodyExpectation(group, expectation, turn) {
+  return group.type === "body" && VARIANTS.includes(expectation.variant) && validTurn(group, turn);
+}
+
+export function validTurn(group, turn) {
+  if (turn === null) return true;
+  const declaredTurnCount = group.input.prompt === undefined ? group.input.turns.length : 1;
+  return Number.isInteger(turn) && turn >= 1 && turn <= declaredTurnCount;
+}
+
+export function completedCheck(expectation, expected, observed, passed) {
+  return {
+    id: expectation.id,
+    kind: expectation.kind,
+    variant: expectation.variant,
+    state: passed ? "pass" : "fail",
+    expected,
+    observed,
+  };
+}
+
+export function unavailableCheck(expectation, expected) {
+  return {
+    id: expectation.id,
+    kind: expectation.kind,
+    variant: expectation.variant,
+    state: "unavailable",
+    expected,
+    observed: null,
+  };
+}
+
+export function workspaceEvidence(run, turn) {
+  if (turn === null || (!Array.isArray(run.turns) && turn === 1)) {
+    if (!run.workspace || !Array.isArray(run.effects?.after)) return null;
+    return { root: run.workspace, files: run.effects.after, changes: run.effects.changes ?? null };
+  }
+  const observedTurn = run.turns?.find((entry) => entry.turn === turn);
+  if (!observedTurn?.workspace?.path || !Array.isArray(observedTurn.workspace.files)) return null;
+  return {
+    root: observedTurn.workspace.path,
+    files: observedTurn.workspace.files,
+    changes: observedTurn.workspace.changes ?? null,
+  };
+}
+
+export function readPaths(run, turn) {
+  if (turn === null) {
+    if (Array.isArray(run.turns)) {
+      return run.turns.flatMap((entry) => (Array.isArray(entry.successfulReadPaths) ? entry.successfulReadPaths : []));
+    }
+    return Array.isArray(run.successfulReadPaths) ? run.successfulReadPaths : [];
+  }
+  if (!Array.isArray(run.turns)) {
+    return turn === 1 && Array.isArray(run.successfulReadPaths) ? run.successfulReadPaths : null;
+  }
+  const observedTurn = run.turns.find((entry) => entry.turn === turn);
+  return observedTurn && Array.isArray(observedTurn.successfulReadPaths) ? observedTurn.successfulReadPaths : null;
+}
+
+export function textMatches(expect, value, text) {
+  if (expect === "contains") return text.includes(value);
+  if (expect === "not-contains") return !text.includes(value);
+  return text === value;
+}
+
+export function validPathArray(value) {
+  return (
+    Array.isArray(value) &&
+    value.every((entry) => typeof entry === "string" && isSafeRelativePath(entry)) &&
+    new Set(value).size === value.length
+  );
+}
+
+export async function readWorkspaceFile(workspace, relativePath) {
+  const entry = workspace.files.find((file) => file.path === relativePath);
+  if (!entry) return { state: "missing", sha256: null };
+  if (entry.type !== undefined && entry.type !== "file") {
+    throw new Error(`file assertion requires a regular file: ${relativePath}`);
+  }
+
+  const root = await realpath(workspace.root);
+  const declaredPath = path.resolve(root, relativePath);
+  const declaredStat = await lstat(declaredPath);
+  if (declaredStat.isSymbolicLink()) {
+    throw new Error(`file assertion requires a regular file: ${relativePath}`);
+  }
+  const file = await realpath(declaredPath);
+  if (!isContained(root, file)) throw new Error(`workspace evidence path escapes its root: ${relativePath}`);
+  const contents = await readFile(file);
+  const actualHash = createHash("sha256").update(contents).digest("hex");
+  if (actualHash !== entry.sha256) throw new Error(`workspace evidence changed after capture: ${relativePath}`);
+  return { state: "present", sha256: actualHash, text: contents.toString("utf8") };
+}
+
+export function isSafeRelativePath(value) {
+  return (
+    value.length > 0 &&
+    !path.isAbsolute(value) &&
+    !value.includes("\\") &&
+    !value.includes("\0") &&
+    !value.split("/").some((part) => part === "" || part === "." || part === "..")
+  );
+}
+
+export function isContained(root, target) {
+  const relative = path.relative(root, target);
+  return relative === "" || (!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative));
+}
+
+export function compareText(left, right) {
+  if (left < right) return -1;
+  if (left > right) return 1;
+  return 0;
+}

package/skills/evaluate-skill/scripts/lib/grade-workspace.mjs

@@ -0,0 +1,192 @@
+import { isDeepStrictEqual } from "node:util";
+
+import {
+  compareText,
+  completedCheck,
+  hasOwn,
+  isSafeRelativePath,
+  readWorkspaceFile,
+  RESPONSE_EXPECTATIONS,
+  textMatches,
+  unavailableCheck,
+  validBodyExpectation,
+  validPathArray,
+  workspaceEvidence,
+} from "./grade-support.mjs";
+
+const JSON_EXPECTATIONS = new Set([
+  "available",
+  "missing",
+  "valid",
+  "malformed",
+  "field-present",
+  "field-absent",
+  "field-equals",
+]);
+
+export async function gradePath(group, runs, expectation) {
+  const turn = expectation.turn ?? null;
+  if (
+    !validBodyExpectation(group, expectation, turn) ||
+    !["exists", "absent"].includes(expectation.expect) ||
+    typeof expectation.path !== "string" ||
+    !isSafeRelativePath(expectation.path)
+  ) {
+    return { error: "invalid path expectation" };
+  }
+  const expected = { expect: expectation.expect, path: expectation.path, turn };
+  const run = runs[expectation.variant];
+  if (!run || run.state !== "complete") return unavailableCheck(expectation, expected);
+  const workspace = workspaceEvidence(run, turn);
+  if (workspace === null) return unavailableCheck(expectation, expected);
+  const entry = workspace.files.find((file) => file.path === expectation.path);
+  const observed = {
+    path: expectation.path,
+    state: entry ? "present" : "missing",
+    type: entry ? (entry.type ?? "file") : null,
+    turn,
+  };
+  return completedCheck(
+    expectation,
+    expected,
+    observed,
+    expectation.expect === "exists" ? observed.state === "present" : observed.state === "missing",
+  );
+}
+
+export async function gradeChangedPaths(group, runs, expectation) {
+  const turn = expectation.turn ?? null;
+  if (
+    !validBodyExpectation(group, expectation, turn) ||
+    !["equals", "excludes"].includes(expectation.expect) ||
+    !validPathArray(expectation.paths)
+  ) {
+    return { error: "invalid changed-paths expectation" };
+  }
+  const expectedPaths = [...expectation.paths].sort(compareText);
+  const expected = { expect: expectation.expect, paths: expectedPaths, turn };
+  const run = runs[expectation.variant];
+  if (!run || run.state !== "complete") return unavailableCheck(expectation, expected);
+  const workspace = workspaceEvidence(run, turn);
+  if (workspace === null || workspace.changes === null) return unavailableCheck(expectation, expected);
+  const actual = [...workspace.changes.created, ...workspace.changes.modified, ...workspace.changes.deleted].sort(
+    compareText,
+  );
+  const matches =
+    expectation.expect === "equals"
+      ? isDeepStrictEqual(actual, expectedPaths)
+      : expectedPaths.every((entry) => !actual.includes(entry));
+  return completedCheck(expectation, expected, { paths: actual, turn }, matches);
+}
+
+export async function gradeFileText(group, runs, expectation) {
+  const turn = expectation.turn ?? null;
+  if (
+    !validBodyExpectation(group, expectation, turn) ||
+    !RESPONSE_EXPECTATIONS.has(expectation.expect) ||
+    typeof expectation.value !== "string" ||
+    typeof expectation.path !== "string" ||
+    !isSafeRelativePath(expectation.path)
+  ) {
+    return { error: "invalid file-text expectation" };
+  }
+  const expected = { expect: expectation.expect, path: expectation.path, value: expectation.value, turn };
+  const run = runs[expectation.variant];
+  if (!run || run.state !== "complete") return unavailableCheck(expectation, expected);
+  const workspace = workspaceEvidence(run, turn);
+  if (workspace === null) return unavailableCheck(expectation, expected);
+  const file = await readWorkspaceFile(workspace, expectation.path);
+  const observed = {
+    path: expectation.path,
+    file: file.state,
+    sha256: file.sha256,
+    turn,
+  };
+  const matches = file.state === "present" && textMatches(expectation.expect, expectation.value, file.text);
+  return completedCheck(expectation, expected, observed, matches);
+}
+
+export async function gradeJson(group, runs, expectation) {
+  const turn = expectation.turn ?? null;
+  if (!validJsonExpectation(group, expectation, turn)) return { error: "invalid json expectation" };
+
+  const expected = {
+    expect: expectation.expect,
+    path: expectation.path,
+    pointer: typeof expectation.pointer === "string" ? expectation.pointer : null,
+    value: hasOwn(expectation, "value") ? expectation.value : null,
+    turn,
+  };
+  const run = runs[expectation.variant];
+  if (!run || run.state !== "complete") return unavailableCheck(expectation, expected);
+  const workspace = workspaceEvidence(run, turn);
+  if (workspace === null) return unavailableCheck(expectation, expected);
+  const file = await readWorkspaceFile(workspace, expectation.path);
+  const observed = jsonObservation(file, expectation.pointer);
+  return completedCheck(expectation, expected, observed, jsonMatches(expectation, observed));
+}
+
+function validJsonExpectation(group, expectation, turn) {
+  const fieldExpectation = ["field-present", "field-absent", "field-equals"].includes(expectation.expect);
+  return (
+    validBodyExpectation(group, expectation, turn) &&
+    typeof expectation.path === "string" &&
+    isSafeRelativePath(expectation.path) &&
+    JSON_EXPECTATIONS.has(expectation.expect) &&
+    (!fieldExpectation || isJsonPointer(expectation.pointer)) &&
+    (expectation.expect !== "field-equals" || hasOwn(expectation, "value"))
+  );
+}
+
+function jsonObservation(file, pointer) {
+  if (file.state === "missing") {
+    return { file: "missing", sha256: null, parse: "unavailable", field: "unavailable", value: null };
+  }
+  let value;
+  try {
+    value = JSON.parse(file.text);
+  } catch {
+    return { file: "present", sha256: file.sha256, parse: "malformed", field: "unavailable", value: null };
+  }
+  if (typeof pointer !== "string") {
+    return { file: "present", sha256: file.sha256, parse: "valid", field: "unavailable", value: null };
+  }
+  const field = jsonPointer(value, pointer);
+  return {
+    file: "present",
+    sha256: file.sha256,
+    parse: "valid",
+    field: field.present ? "present" : "absent",
+    value: field.present ? field.value : null,
+  };
+}
+
+function jsonMatches(expectation, observed) {
+  if (expectation.expect === "available") return observed.file === "present";
+  if (expectation.expect === "missing") return observed.file === "missing";
+  if (expectation.expect === "valid") return observed.parse === "valid";
+  if (expectation.expect === "malformed") return observed.parse === "malformed";
+  if (expectation.expect === "field-present") return observed.field === "present";
+  if (expectation.expect === "field-absent") return observed.field === "absent";
+  return observed.field === "present" && isDeepStrictEqual(observed.value, expectation.value);
+}
+
+function jsonPointer(value, pointer) {
+  let current = value;
+  if (pointer === "") return { present: true, value: current };
+  for (const token of pointer.slice(1).split("/").map(decodePointerToken)) {
+    if (typeof current !== "object" || current === null || !hasOwn(current, token)) {
+      return { present: false, value: null };
+    }
+    current = current[token];
+  }
+  return { present: true, value: current };
+}
+
+function decodePointerToken(token) {
+  return token.replaceAll("~1", "/").replaceAll("~0", "~");
+}
+
+function isJsonPointer(value) {
+  return typeof value === "string" && (value === "" || (value.startsWith("/") && !/~(?:[^01]|$)/.test(value)));
+}

package/skills/evaluate-skill/scripts/lib/grade.mjs

@@ -0,0 +1,56 @@
+import { gradeComparisons } from "./grade-comparison.mjs";
+import { gradeResourceRead, gradeResponseText, gradeSkillRead } from "./grade-interaction.mjs";
+import { gradeChangedPaths, gradeFileText, gradeJson, gradePath } from "./grade-workspace.mjs";
+
+const GRADERS = new Map([
+  ["skill-read", gradeSkillRead],
+  ["resource-read", gradeResourceRead],
+  ["path", gradePath],
+  ["changed-paths", gradeChangedPaths],
+  ["file-text", gradeFileText],
+  ["json", gradeJson],
+  ["response-text", gradeResponseText],
+]);
+
+/**
+ * @param {any} group
+ * @param {any} runs
+ * @param {any} evidence
+ * @returns {Promise<any>}
+ */
+export async function gradeDeterministic(group, runs, evidence) {
+  const checks = /** @type {Array<any>} */ ([]);
+  const errors = /** @type {Array<any>} */ ([]);
+
+  for (const expectation of group.expectations) {
+    const grader = GRADERS.get(expectation.kind);
+    if (!grader) {
+      errors.push({ id: expectation.id, reason: `unsupported expectation kind: ${expectation.kind}` });
+      continue;
+    }
+    try {
+      const result = await grader(group, runs, expectation);
+      if (result.error) errors.push({ id: expectation.id, reason: result.error });
+      else checks.push(result);
+    } catch (error) {
+      errors.push({ id: expectation.id, reason: error instanceof Error ? error.message : String(error) });
+    }
+  }
+
+  const compared = gradeComparisons(group.expectations, checks);
+  errors.push(...compared.errors);
+  return {
+    schema_version: 1,
+    state: reportState(checks, errors),
+    evidence,
+    checks,
+    comparisons: compared.comparisons,
+    errors,
+  };
+}
+
+function reportState(checks, errors) {
+  if (errors.length > 0) return "grade-error";
+  if (checks.length > 0 && checks.every((check) => check.state === "unavailable")) return "unavailable";
+  return "complete";
+}