@hassangameryt/freeflow 0.8.0 → 0.9.0

package/skills/evaluate-skill/scripts/lib/definitions.mjs

@@ -1,448 +0,0 @@
-import { readFile, realpath } from "node:fs/promises";
-import path from "node:path";
-
-const ID = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/;
-const EVALUATION_TYPES = new Set(["description", "body"]);
-const VARIANTS = ["baseline", "candidate"];
-const EVALUATION_HOSTS = new Set(["pi"]);
-const PROMPT_MODES = new Set(["isolated", "host"]);
-export const COMMAND_TOOL = "run_command";
-const MAX_COMMAND_TIMEOUT_MS = 10 * 60 * 1000;
-const RESERVED_ENVIRONMENT_KEYS = new Set([
-  "PATH",
-  "HOME",
-  "PWD",
-  "OLDPWD",
-  "SHELL",
-  "BASH_ENV",
-  "NODE_OPTIONS",
-  "NODE_PATH",
-  "PI_PACKAGE_DIR",
-]);
-
-export class DefinitionError extends Error {
-  constructor(code, message, definitionPath = null) {
-    super(message);
-    this.name = "DefinitionError";
-    this.code = code;
-    this.definitionPath = definitionPath;
-  }
-}
-
-function fail(code, message, definitionPath) {
-  throw new DefinitionError(code, message, definitionPath);
-}
-
-function isPlainObject(value) {
-  return value !== null && typeof value === "object" && !Array.isArray(value);
-}
-
-function requireObject(value, label, definitionPath) {
-  if (!isPlainObject(value)) fail("invalid-definition", `${label} must be an object`, definitionPath);
-  return value;
-}
-
-function requireExactKeys(value, required, optional, label, definitionPath) {
-  const object = requireObject(value, label, definitionPath);
-  const allowed = new Set([...required, ...optional]);
-  for (const key of required) {
-    if (!Object.hasOwn(object, key)) fail("invalid-definition", `${label} is missing ${key}`, definitionPath);
-  }
-  for (const key of Object.keys(object)) {
-    if (!allowed.has(key)) fail("invalid-definition", `${label} contains unsupported field ${key}`, definitionPath);
-  }
-  return object;
-}
-
-function requireId(value, label, definitionPath) {
-  if (typeof value !== "string" || value.length > 64 || !ID.test(value)) {
-    fail(
-      "invalid-definition",
-      `${label} must start with a lowercase letter and use lowercase letters, numbers, or single hyphens up to 64 characters`,
-      definitionPath,
-    );
-  }
-  return value;
-}
-
-function requireString(value, label, definitionPath) {
-  if (typeof value !== "string" || value.trim() === "") {
-    fail("invalid-definition", `${label} must be a non-empty string`, definitionPath);
-  }
-  return value;
-}
-
-function requireStringArray(value, label, definitionPath, { paths = false, unique = false } = {}) {
-  if (!Array.isArray(value)) fail("invalid-definition", `${label} must be an array`, definitionPath);
-  const seen = new Set();
-  return value.map((entry, index) => {
-    const string = requireString(entry, `${label}[${index}]`, definitionPath);
-    if (paths) requireSafeRelativePath(string, `${label}[${index}]`, definitionPath);
-    if (unique && seen.has(string)) {
-      fail("invalid-definition", `${label} contains duplicate ${string}`, definitionPath);
-    }
-    seen.add(string);
-    return string;
-  });
-}
-
-function requireSafeRelativePath(value, label, definitionPath) {
-  const string = requireString(value, label, definitionPath);
-  const normalized = path.normalize(string);
-  if (path.isAbsolute(string) || normalized === "." || normalized === ".." || normalized.startsWith(`..${path.sep}`)) {
-    fail("unsafe-path", `${label} must stay inside the definition root`, definitionPath);
-  }
-  return string;
-}
-
-function requireSuiteReference(value, label, definitionPath) {
-  const string = requireString(value, label, definitionPath);
-  if (path.isAbsolute(string) || path.normalize(string) === ".") {
-    fail("unsafe-path", `${label} must be relative to the suite file`, definitionPath);
-  }
-  return string;
-}
-
-function validateInput(value, definitionPath) {
-  const input = requireExactKeys(value, [], ["prompt", "turns"], "group.input", definitionPath);
-  const hasPrompt = Object.hasOwn(input, "prompt");
-  const hasTurns = Object.hasOwn(input, "turns");
-  if (hasPrompt === hasTurns) {
-    fail("invalid-definition", "group.input must contain exactly one of prompt or turns", definitionPath);
-  }
-  if (hasPrompt) requireString(input.prompt, "group.input.prompt", definitionPath);
-  if (hasTurns) {
-    const turns = requireStringArray(input.turns, "group.input.turns", definitionPath);
-    if (turns.length === 0) fail("invalid-definition", "group.input.turns must not be empty", definitionPath);
-  }
-}
-
-function validateSource(value, label, definitionPath) {
-  const source = requireObject(value, label, definitionPath);
-  if (source.kind === "working-tree") {
-    requireExactKeys(source, ["kind"], [], label, definitionPath);
-    return;
-  }
-  if (source.kind === "git") {
-    requireExactKeys(source, ["kind", "ref"], [], label, definitionPath);
-    requireString(source.ref, `${label}.ref`, definitionPath);
-    return;
-  }
-  fail("invalid-definition", `${label}.kind must be working-tree or git`, definitionPath);
-}
-
-function validateEnvironment(value, label, definitionPath) {
-  const environment = requireExactKeys(value, ["source", "skills", "target", "context"], [], label, definitionPath);
-  validateSource(environment.source, `${label}.source`, definitionPath);
-  const skills = requireStringArray(environment.skills, `${label}.skills`, definitionPath, { paths: true });
-  requireStringArray(environment.context, `${label}.context`, definitionPath, { paths: true });
-  if (
-    environment.target !== null &&
-    (!Number.isInteger(environment.target) || environment.target < 0 || environment.target >= skills.length)
-  ) {
-    fail("invalid-definition", `${label}.target must be null or an index into skills`, definitionPath);
-  }
-}
-
-function validateRuntime(value, definitionPath) {
-  const runtime = requireExactKeys(
-    value,
-    ["host", "extensions", "environment", "session"],
-    ["prompt"],
-    "group.runtime",
-    definitionPath,
-  );
-  if (!EVALUATION_HOSTS.has(runtime.host)) {
-    fail("invalid-definition", "group.runtime.host must be pi", definitionPath);
-  }
-  if (Object.hasOwn(runtime, "prompt") && !PROMPT_MODES.has(runtime.prompt)) {
-    fail("invalid-definition", "group.runtime.prompt must be isolated or host", definitionPath);
-  }
-  if (!Array.isArray(runtime.extensions)) {
-    fail("invalid-definition", "group.runtime.extensions must be an array", definitionPath);
-  }
-  const bundleIds = new Set();
-  for (const [index, entry] of runtime.extensions.entries()) {
-    const bundle = requireExactKeys(
-      entry,
-      ["entry", "resources"],
-      [],
-      `group.runtime.extensions[${index}]`,
-      definitionPath,
-    );
-    requireSafeRelativePath(bundle.entry, `group.runtime.extensions[${index}].entry`, definitionPath);
-    const resources = requireStringArray(
-      bundle.resources,
-      `group.runtime.extensions[${index}].resources`,
-      definitionPath,
-      { paths: true, unique: true },
-    );
-    if (resources.length === 0) {
-      fail("invalid-definition", `group.runtime.extensions[${index}].resources must not be empty`, definitionPath);
-    }
-    const bundleId = `${bundle.entry}\u0000${resources.join("\u0000")}`;
-    if (bundleIds.has(bundleId)) {
-      fail("invalid-definition", "group.runtime.extensions contains duplicate bundles", definitionPath);
-    }
-    bundleIds.add(bundleId);
-  }
-  if (typeof runtime.session !== "boolean") {
-    fail("invalid-definition", "group.runtime.session must be a boolean", definitionPath);
-  }
-  const environment = requireExactKeys(
-    runtime.environment,
-    ["literal", "inherit"],
-    [],
-    "group.runtime.environment",
-    definitionPath,
-  );
-  const literal = requireObject(environment.literal, "group.runtime.environment.literal", definitionPath);
-  const inherited = requireStringArray(environment.inherit, "group.runtime.environment.inherit", definitionPath, {
-    unique: true,
-  });
-  const literalKeys = new Set(Object.keys(literal).map((key) => key.toUpperCase()));
-  for (const [key, value] of Object.entries(literal)) {
-    const normalizedKey = key.toUpperCase();
-    if (
-      !/^[A-Za-z_][A-Za-z0-9_]*$/.test(key) ||
-      isBlockedEnvironmentKey(normalizedKey) ||
-      /TOKEN|SECRET|PASSWORD|API[_-]?KEY|PRIVATE[_-]?KEY|CREDENTIAL/i.test(key)
-    ) {
-      fail(
-        "invalid-definition",
-        `group.runtime.environment.literal contains a reserved or secret-like key: ${key}`,
-        definitionPath,
-      );
-    }
-    if (typeof value !== "string") {
-      fail("invalid-definition", `group.runtime.environment.literal.${key} must be a string`, definitionPath);
-    }
-  }
-  for (const key of inherited) {
-    const normalizedKey = key.toUpperCase();
-    if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key) || isBlockedEnvironmentKey(normalizedKey)) {
-      fail("invalid-definition", `group.runtime.environment.inherit contains an invalid key: ${key}`, definitionPath);
-    }
-    if (literalKeys.has(normalizedKey)) {
-      fail(
-        "invalid-definition",
-        `group.runtime.environment key is declared as both literal and inherited: ${key}`,
-        definitionPath,
-      );
-    }
-  }
-}
-
-function isBlockedEnvironmentKey(normalizedKey) {
-  return (
-    RESERVED_ENVIRONMENT_KEYS.has(normalizedKey) ||
-    normalizedKey.startsWith("SKILL_EVAL_") ||
-    normalizedKey.startsWith("PI_") ||
-    normalizedKey.startsWith("LD_") ||
-    normalizedKey.startsWith("DYLD_")
-  );
-}
-
-function validateExpectations(value, definitionPath) {
-  if (!Array.isArray(value)) fail("invalid-definition", "group.expectations must be an array", definitionPath);
-  const ids = new Set();
-  for (const [index, entry] of value.entries()) {
-    const expectation = requireObject(entry, `group.expectations[${index}]`, definitionPath);
-    requireId(expectation.id, `group.expectations[${index}].id`, definitionPath);
-    requireString(expectation.kind, `group.expectations[${index}].kind`, definitionPath);
-    if (ids.has(expectation.id)) {
-      fail("invalid-definition", `group.expectations contains duplicate ID ${expectation.id}`, definitionPath);
-    }
-    ids.add(expectation.id);
-  }
-}
-
-// Declared commands are fixed by the definition author; the subject may only choose which one runs.
-function validateCommands(group, definitionPath) {
-  const toolDeclared = group.tools.includes(COMMAND_TOOL);
-  if (!Object.hasOwn(group, "commands")) {
-    if (toolDeclared) fail("invalid-definition", `${COMMAND_TOOL} requires declared group.commands`, definitionPath);
-    return;
-  }
-  if (!toolDeclared) fail("invalid-definition", `group.commands requires the ${COMMAND_TOOL} tool`, definitionPath);
-  if (group.type !== "body")
-    fail("invalid-definition", "group.commands are supported only by body groups", definitionPath);
-  if (!Array.isArray(group.commands) || group.commands.length === 0) {
-    fail("invalid-definition", "group.commands must be a non-empty array", definitionPath);
-  }
-  const ids = new Set();
-  for (const [index, entry] of group.commands.entries()) {
-    const label = `group.commands[${index}]`;
-    const command = requireExactKeys(entry, ["id", "argv"], ["timeout_ms"], label, definitionPath);
-    requireId(command.id, `${label}.id`, definitionPath);
-    if (ids.has(command.id))
-      fail("invalid-definition", `group.commands contains duplicate ID ${command.id}`, definitionPath);
-    ids.add(command.id);
-    const argv = requireStringArray(command.argv, `${label}.argv`, definitionPath);
-    if (argv.length === 0) fail("invalid-definition", `${label}.argv must not be empty`, definitionPath);
-    if (
-      Object.hasOwn(command, "timeout_ms") &&
-      (!Number.isInteger(command.timeout_ms) || command.timeout_ms < 1 || command.timeout_ms > MAX_COMMAND_TIMEOUT_MS)
-    ) {
-      fail(
-        "invalid-definition",
-        `${label}.timeout_ms must be an integer from 1 to ${MAX_COMMAND_TIMEOUT_MS}`,
-        definitionPath,
-      );
-    }
-  }
-}
-
-function validateModel(value, definitionPath) {
-  const model = requireExactKeys(value, ["model"], ["thinking"], "group.model", definitionPath);
-  requireString(model.model, "group.model.model", definitionPath);
-  if (Object.hasOwn(model, "thinking")) requireString(model.thinking, "group.model.thinking", definitionPath);
-}
-
-function validateGroup(value, definitionPath) {
-  const group = requireExactKeys(
-    value,
-    [
-      "schema_version",
-      "kind",
-      "id",
-      "type",
-      "input",
-      "fixture",
-      "tools",
-      "variants",
-      "expectations",
-      "review_questions",
-    ],
-    ["model", "runtime", "commands"],
-    "group",
-    definitionPath,
-  );
-  if (group.schema_version !== 1) fail("unsupported-schema", "group.schema_version must be 1", definitionPath);
-  if (group.kind !== "group") fail("invalid-definition", "group.kind must be group", definitionPath);
-  requireId(group.id, "group.id", definitionPath);
-  if (!EVALUATION_TYPES.has(group.type)) {
-    fail("invalid-definition", "group.type must be description or body", definitionPath);
-  }
-  validateInput(group.input, definitionPath);
-  if (group.fixture !== null) requireSafeRelativePath(group.fixture, "group.fixture", definitionPath);
-  requireStringArray(group.tools, "group.tools", definitionPath, { unique: true });
-  validateCommands(group, definitionPath);
-  if (Object.hasOwn(group, "runtime")) validateRuntime(group.runtime, definitionPath);
-  const variants = requireExactKeys(group.variants, VARIANTS, [], "group.variants", definitionPath);
-  for (const variant of VARIANTS) validateEnvironment(variants[variant], `group.variants.${variant}`, definitionPath);
-  if (variants.candidate.target === null && variants.candidate.skills.length > 0) {
-    fail(
-      "invalid-definition",
-      "group.variants.candidate.target must identify the candidate target skill when candidate skills are declared",
-      definitionPath,
-    );
-  }
-  validateExpectations(group.expectations, definitionPath);
-  requireStringArray(group.review_questions, "group.review_questions", definitionPath);
-  if (Object.hasOwn(group, "model")) validateModel(group.model, definitionPath);
-  return group;
-}
-
-function validateSuite(value, definitionPath) {
-  const suite = requireExactKeys(value, ["schema_version", "kind", "id", "groups"], [], "suite", definitionPath);
-  if (suite.schema_version !== 1) fail("unsupported-schema", "suite.schema_version must be 1", definitionPath);
-  if (suite.kind !== "suite") fail("invalid-definition", "suite.kind must be suite", definitionPath);
-  requireId(suite.id, "suite.id", definitionPath);
-  const groups = requireStringArray(suite.groups, "suite.groups", definitionPath, { unique: true });
-  for (const [index, reference] of groups.entries()) {
-    requireSuiteReference(reference, `suite.groups[${index}]`, definitionPath);
-  }
-  if (groups.length === 0) fail("invalid-definition", "suite.groups must not be empty", definitionPath);
-  return suite;
-}
-
-function isContained(root, target) {
-  const relative = path.relative(root, target);
-  return relative === "" || (!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative));
-}
-
-async function readJsonFile(file, root) {
-  const lexicalFile = path.resolve(file);
-  if (!isContained(root, lexicalFile)) fail("unsafe-path", `Definition escapes root: ${file}`, lexicalFile);
-
-  let canonicalRoot;
-  let canonicalFile;
-  try {
-    [canonicalRoot, canonicalFile] = await Promise.all([realpath(root), realpath(lexicalFile)]);
-  } catch {
-    fail("unreadable-definition", `Cannot resolve definition: ${lexicalFile}`, lexicalFile);
-  }
-  if (!isContained(canonicalRoot, canonicalFile)) {
-    fail("unsafe-path", `Definition symlink escapes root: ${lexicalFile}`, lexicalFile);
-  }
-
-  let contents;
-  try {
-    contents = await readFile(canonicalFile, "utf8");
-  } catch {
-    fail("unreadable-definition", `Cannot read definition: ${lexicalFile}`, lexicalFile);
-  }
-  try {
-    return { value: JSON.parse(contents), file: lexicalFile };
-  } catch {
-    fail("invalid-json", `Definition is not valid JSON: ${lexicalFile}`, lexicalFile);
-  }
-}
-
-export async function loadDefinition(file, { root = process.cwd() } = {}) {
-  const definitionRoot = path.resolve(root);
-  const loaded = await readJsonFile(file, definitionRoot);
-  const object = requireObject(loaded.value, "definition", loaded.file);
-  if (object.kind === "group") return validateGroup(object, loaded.file);
-  if (object.kind !== "suite") fail("invalid-definition", "definition.kind must be group or suite", loaded.file);
-
-  const suite = validateSuite(object, loaded.file);
-  const groups = [];
-  const ids = new Set();
-  for (const reference of suite.groups) {
-    const groupFile = path.resolve(path.dirname(loaded.file), reference);
-    const groupLoaded = await readJsonFile(groupFile, definitionRoot);
-    const group = validateGroup(groupLoaded.value, groupLoaded.file);
-    if (ids.has(group.id)) fail("invalid-definition", `suite contains duplicate group ID ${group.id}`, loaded.file);
-    ids.add(group.id);
-    groups.push(group);
-  }
-  return { ...suite, groups };
-}
-
-function variantSelection(selected) {
-  return {
-    baseline: selected === null || selected === "baseline" ? "selected" : "not-selected",
-    candidate: selected === null || selected === "candidate" ? "selected" : "not-selected",
-  };
-}
-
-export function selectDefinition(definition, { group = null, variant = null } = {}) {
-  if (variant !== null && !VARIANTS.includes(variant)) {
-    fail("invalid-selector", "variant must be baseline or candidate", null);
-  }
-  if (definition.kind === "group" && group !== null) {
-    fail("invalid-selector", "--group cannot be used with a direct group definition", null);
-  }
-
-  const available = definition.kind === "group" ? [definition] : definition.groups;
-  let selected = available;
-  if (group !== null) {
-    const position = typeof group === "number" || /^\d+$/.test(group) ? Number(group) : null;
-    selected =
-      position === null ? available.filter((entry) => entry.id === group) : [available[position - 1]].filter(Boolean);
-    if (selected.length === 0) fail("unknown-group", `Unknown group selector: ${group}`, null);
-  }
-
-  return {
-    kind: "selection",
-    definitionKind: definition.kind,
-    id: definition.id,
-    groups: selected.map((entry) => ({
-      position: available.indexOf(entry) + 1,
-      group: entry,
-      variants: variantSelection(variant),
-    })),
-  };
-}

package/skills/evaluate-skill/scripts/lib/grade-comparison.mjs

@@ -1,60 +0,0 @@
-import { isDeepStrictEqual } from "node:util";
-
-import { hasOwn } from "./grade-support.mjs";
-
-export function gradeComparisons(expectations, checks) {
-  /** @type {Map<string, any[]>} */
-  const grouped = new Map();
-  const errors = /** @type {any[]} */ ([]);
-  for (const expectation of expectations) {
-    if (!hasOwn(expectation, "comparison")) continue;
-    if (typeof expectation.comparison !== "string" || !/^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/.test(expectation.comparison)) {
-      errors.push({ id: expectation.id, reason: "invalid comparison identity" });
-      continue;
-    }
-    const entries = grouped.get(expectation.comparison) ?? [];
-    entries.push(expectation);
-    grouped.set(expectation.comparison, entries);
-  }
-
-  /** @type {Map<string, any>} */
-  const byCheck = new Map(checks.map((check) => [check.id, check]));
-  const comparisons = /** @type {any[]} */ ([]);
-  for (const [id, entries] of grouped) {
-    const baselineExpectation = entries.find((entry) => entry.variant === "baseline");
-    const candidateExpectation = entries.find((entry) => entry.variant === "candidate");
-    if (entries.length !== 2 || !baselineExpectation || !candidateExpectation) {
-      errors.push({ id, reason: "comparison requires one baseline and one candidate expectation" });
-      continue;
-    }
-    const baseline = byCheck.get(baselineExpectation.id);
-    const candidate = byCheck.get(candidateExpectation.id);
-    if (!sameCheck(baseline, candidate)) {
-      errors.push({ id, reason: "comparison expectations must describe the same deterministic check" });
-      continue;
-    }
-    comparisons.push({
-      id,
-      kind: baseline.kind,
-      baseline: { check: baseline.id, state: baseline.state },
-      candidate: { check: candidate.id, state: candidate.state },
-      transition: transition(baseline.state, candidate.state),
-    });
-  }
-  return { comparisons, errors };
-}
-
-function sameCheck(baseline, candidate) {
-  return (
-    baseline !== undefined &&
-    candidate !== undefined &&
-    baseline.kind === candidate.kind &&
-    isDeepStrictEqual(baseline.expected, candidate.expected)
-  );
-}
-
-function transition(baseline, candidate) {
-  return [baseline, candidate].every((state) => state === "pass" || state === "fail")
-    ? `${baseline}-to-${candidate}`
-    : "unavailable";
-}

package/skills/evaluate-skill/scripts/lib/grade.mjs

@@ -1,64 +0,0 @@
-import { gradeComparisons } from "./grade-comparison.mjs";
-import {
-  gradeContextText,
-  gradeResourceRead,
-  gradeResponseText,
-  gradeSkillRead,
-  gradeToolCall,
-} from "./grade-interaction.mjs";
-import { gradeChangedPaths, gradeFileText, gradeJson, gradePath } from "./grade-workspace.mjs";
-
-const GRADERS = new Map([
-  ["skill-read", gradeSkillRead],
-  ["resource-read", gradeResourceRead],
-  ["path", gradePath],
-  ["changed-paths", gradeChangedPaths],
-  ["file-text", gradeFileText],
-  ["json", gradeJson],
-  ["response-text", gradeResponseText],
-  ["context-text", gradeContextText],
-  ["tool-call", gradeToolCall],
-]);
-
-/**
- * @param {any} group
- * @param {any} runs
- * @param {any} evidence
- * @returns {Promise<any>}
- */
-export async function gradeDeterministic(group, runs, evidence) {
-  const checks = /** @type {Array<any>} */ ([]);
-  const errors = /** @type {Array<any>} */ ([]);
-
-  for (const expectation of group.expectations) {
-    const grader = GRADERS.get(expectation.kind);
-    if (!grader) {
-      errors.push({ id: expectation.id, reason: `unsupported expectation kind: ${expectation.kind}` });
-      continue;
-    }
-    try {
-      const result = await grader(group, runs, expectation);
-      if (result.error) errors.push({ id: expectation.id, reason: result.error });
-      else checks.push(result);
-    } catch (error) {
-      errors.push({ id: expectation.id, reason: error instanceof Error ? error.message : String(error) });
-    }
-  }
-
-  const compared = gradeComparisons(group.expectations, checks);
-  errors.push(...compared.errors);
-  return {
-    schema_version: 1,
-    state: reportState(checks, errors),
-    evidence,
-    checks,
-    comparisons: compared.comparisons,
-    errors,
-  };
-}
-
-function reportState(checks, errors) {
-  if (errors.length > 0) return "grade-error";
-  if (checks.length > 0 && checks.every((check) => check.state === "unavailable")) return "unavailable";
-  return "complete";
-}