@hassangameryt/freeflow 0.7.3 → 0.9.0

package/skills/evaluate-behavior/scripts/lib/grade-comparison.mjs

@@ -0,0 +1,90 @@
+import { isDeepStrictEqual } from "node:util";
+
+import { DEFAULT_ARMS, DEFAULT_REFERENCE, isClassicPair } from "./arms.mjs";
+import { hasOwn } from "./grade-support.mjs";
+
+/**
+ * A comparison ID names exactly one expectation per arm, all of one kind and expected value. Each non-reference
+ * arm's outcome is a factual transition from the reference's state to its own. A baseline/candidate group keeps
+ * its original shape (`baseline`, `candidate`, `transition`); any other group records `reference`, `arms` and
+ * `transitions` keyed by arm.
+ *
+ * @param {any[]} expectations
+ * @param {any[]} checks
+ * @param {{arms?: string[], reference?: string}} [options]
+ */
+export function gradeComparisons(expectations, checks, { arms = DEFAULT_ARMS, reference = DEFAULT_REFERENCE } = {}) {
+  /** @type {Map<string, any[]>} */
+  const grouped = new Map();
+  const errors = /** @type {any[]} */ ([]);
+  for (const expectation of expectations) {
+    if (!hasOwn(expectation, "comparison")) continue;
+    if (typeof expectation.comparison !== "string" || !/^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/.test(expectation.comparison)) {
+      errors.push({ id: expectation.id, reason: "invalid comparison identity" });
+      continue;
+    }
+    const entries = grouped.get(expectation.comparison) ?? [];
+    entries.push(expectation);
+    grouped.set(expectation.comparison, entries);
+  }
+
+  const classic = isClassicPair(arms, reference);
+  const others = arms.filter((arm) => arm !== reference);
+  /** @type {Map<string, any>} */
+  const byCheck = new Map(checks.map((check) => [check.id, check]));
+  const comparisons = /** @type {any[]} */ ([]);
+  for (const [id, entries] of grouped) {
+    const expectationFor = Object.fromEntries(
+      arms.map((arm) => [arm, entries.filter((entry) => entry.variant === arm)]),
+    );
+    if (entries.length !== arms.length || arms.some((arm) => expectationFor[arm].length !== 1)) {
+      errors.push({
+        id,
+        reason: classic
+          ? "comparison requires one baseline and one candidate expectation"
+          : `comparison requires exactly one expectation for each arm: ${arms.join(", ")}`,
+      });
+      continue;
+    }
+    const checkFor = Object.fromEntries(arms.map((arm) => [arm, byCheck.get(expectationFor[arm][0].id)]));
+    if (!others.every((arm) => sameCheck(checkFor[reference], checkFor[arm]))) {
+      errors.push({ id, reason: "comparison expectations must describe the same deterministic check" });
+      continue;
+    }
+    if (classic) {
+      comparisons.push({
+        id,
+        kind: checkFor.baseline.kind,
+        baseline: { check: checkFor.baseline.id, state: checkFor.baseline.state },
+        candidate: { check: checkFor.candidate.id, state: checkFor.candidate.state },
+        transition: transition(checkFor.baseline.state, checkFor.candidate.state),
+      });
+      continue;
+    }
+    comparisons.push({
+      id,
+      kind: checkFor[reference].kind,
+      reference,
+      arms: Object.fromEntries(arms.map((arm) => [arm, { check: checkFor[arm].id, state: checkFor[arm].state }])),
+      transitions: Object.fromEntries(
+        others.map((arm) => [arm, transition(checkFor[reference].state, checkFor[arm].state)]),
+      ),
+    });
+  }
+  return { comparisons, errors };
+}
+
+function sameCheck(reference, other) {
+  return (
+    reference !== undefined &&
+    other !== undefined &&
+    reference.kind === other.kind &&
+    isDeepStrictEqual(reference.expected, other.expected)
+  );
+}
+
+function transition(reference, other) {
+  return [reference, other].every((state) => state === "pass" || state === "fail")
+    ? `${reference}-to-${other}`
+    : "unavailable";
+}

package/skills/{evaluate-skill → evaluate-behavior}/scripts/lib/grade-interaction.mjs

@@ -5,15 +5,15 @@ import path from "node:path";
 import {
   completedCheck,
   hasOwn,
+  invalidExpectation,
   isContained,
   isSafeRelativePath,
+  placementProblem,
   readPaths,
   RESPONSE_EXPECTATIONS,
   textMatches,
   unavailableCheck,
-  validBodyExpectation,
   validTurn,
-  VARIANTS,
 } from "./grade-support.mjs";
 
 const ACTIVATION_EXPECTATIONS = new Set(["never", "on-turn", "by-turn", "not-before-turn"]);
@@ -21,17 +21,19 @@ const TOOL_CALL_EXPECTATIONS = new Set(["called", "succeeded", "failed", "not-ca
 const CONTEXT_TEXT_EXPECTATIONS = new Set(["contains", "not-contains", "equals"]);
 const CONTEXT_SURFACES = new Set(["system-prompt", "provider-context"]);
 const CONTEXT_REQUEST_SELECTORS = new Set(["first", "last"]);
+const BODY = ["body"];
 
 export async function gradeSkillRead(group, runs, expectation) {
   const turn = expectation.turn ?? null;
   const timingExpectation = expectation.expect !== "never";
+  // The turn is checked here rather than as placement: a timing expectation needs one, and `never` takes none.
+  const placement = placementProblem(group, expectation, null, ["description"]);
   if (
-    group.type !== "description" ||
-    !VARIANTS.includes(expectation.variant) ||
+    placement !== null ||
     !ACTIVATION_EXPECTATIONS.has(expectation.expect) ||
     (timingExpectation ? turn === null || !validTurn(group, turn) : hasOwn(expectation, "turn"))
   ) {
-    return { error: "invalid skill-read expectation" };
+    return invalidExpectation(expectation, placement);
   }
 
   const expected = { read: expectation.expect, turn };
@@ -49,17 +51,17 @@ export async function gradeSkillRead(group, runs, expectation) {
 export async function gradeResourceRead(group, runs, expectation) {
   const turn = expectation.turn ?? null;
   const indexed = expectation.resource === "skill" || expectation.resource === "context";
+  const placement = placementProblem(group, expectation, turn, ["description", "body"]);
   if (
-    !VARIANTS.includes(expectation.variant) ||
+    placement !== null ||
     !["workspace", "skill", "context"].includes(expectation.resource) ||
     !["read", "not-read"].includes(expectation.expect) ||
     typeof expectation.path !== "string" ||
     !isSafeRelativePath(expectation.path) ||
-    !validTurn(group, turn) ||
     (indexed && (!Number.isInteger(expectation.index) || expectation.index < 0)) ||
     (!indexed && hasOwn(expectation, "index"))
   ) {
-    return { error: "invalid resource-read expectation" };
+    return invalidExpectation(expectation, placement);
   }
   const expected = {
     expect: expectation.expect,
@@ -109,15 +111,16 @@ export async function gradeContextText(group, runs, expectation) {
   const turn = expectation.turn ?? null;
   const surface = expectation.surface ?? "system-prompt";
   const request = expectation.request ?? "last";
+  const placement = placementProblem(group, expectation, turn, BODY);
   if (
-    !validBodyExpectation(group, expectation, turn) ||
+    placement !== null ||
     !CONTEXT_SURFACES.has(surface) ||
     (!CONTEXT_REQUEST_SELECTORS.has(request) && !Number.isInteger(request)) ||
     (Number.isInteger(request) && request < 1) ||
     !CONTEXT_TEXT_EXPECTATIONS.has(expectation.expect) ||
     typeof expectation.value !== "string"
   ) {
-    return { error: "invalid context-text expectation" };
+    return invalidExpectation(expectation, placement);
   }
 
   const expected = { expect: expectation.expect, value: expectation.value, surface, request, turn };
@@ -133,8 +136,9 @@ export async function gradeToolCall(group, runs, expectation) {
   const turn = expectation.turn ?? null;
   const contains = expectation.argumentContains ?? [];
   const notContains = expectation.argumentNotContains ?? [];
+  const placement = placementProblem(group, expectation, turn, BODY);
   if (
-    !validBodyExpectation(group, expectation, turn) ||
+    placement !== null ||
     typeof expectation.tool !== "string" ||
     expectation.tool.trim() === "" ||
     !TOOL_CALL_EXPECTATIONS.has(expectation.expect) ||
@@ -143,7 +147,7 @@ export async function gradeToolCall(group, runs, expectation) {
     !Array.isArray(notContains) ||
     !notContains.every((value) => typeof value === "string")
   ) {
-    return { error: "invalid tool-call expectation" };
+    return invalidExpectation(expectation, placement);
   }
 
   const expected = {
@@ -196,12 +200,9 @@ export async function gradeToolCall(group, runs, expectation) {
 
 export async function gradeResponseText(group, runs, expectation) {
   const turn = expectation.turn ?? null;
-  if (
-    !validBodyExpectation(group, expectation, turn) ||
-    !RESPONSE_EXPECTATIONS.has(expectation.expect) ||
-    typeof expectation.value !== "string"
-  ) {
-    return { error: "invalid response-text expectation" };
+  const placement = placementProblem(group, expectation, turn, BODY);
+  if (placement !== null || !RESPONSE_EXPECTATIONS.has(expectation.expect) || typeof expectation.value !== "string") {
+    return invalidExpectation(expectation, placement);
   }
 
   const expected = { expect: expectation.expect, value: expectation.value, turn };

package/skills/{evaluate-skill → evaluate-behavior}/scripts/lib/grade-support.mjs

@@ -2,15 +2,35 @@ import { createHash } from "node:crypto";
 import { lstat, readFile, realpath } from "node:fs/promises";
 import path from "node:path";
 
-export const VARIANTS = ["baseline", "candidate"];
+import { armNames } from "./arms.mjs";
+
 export const RESPONSE_EXPECTATIONS = new Set(["contains", "not-contains", "equals"]);
 
 export function hasOwn(value, key) {
   return Object.getOwnPropertyDescriptor(value, key) !== undefined;
 }
 
-export function validBodyExpectation(group, expectation, turn) {
-  return group.type === "body" && VARIANTS.includes(expectation.variant) && validTurn(group, turn);
+/** Whether an expectation's `variant` names one of the group's arms. */
+export function isArm(group, name) {
+  return typeof name === "string" && armNames(group).includes(name);
+}
+
+/**
+ * Why an expectation cannot be graded in this group, decided by the group alone: its group type, its arm or its
+ * turn; null when it fits. Graders and definition loading share it, so a definition that loads is placed right.
+ */
+export function placementProblem(group, expectation, turn, types) {
+  if (!types.includes(group.type)) return `it needs a ${types.join(" or ")} group`;
+  if (!isArm(group, expectation.variant)) {
+    return `its variant ${JSON.stringify(expectation.variant ?? null)} is not an arm of the group`;
+  }
+  if (!validTurn(group, turn)) return `its turn ${JSON.stringify(turn)} is not a declared turn`;
+  return null;
+}
+
+/** A grader's refusal of an expectation: its kind, and the placement problem when there is one. */
+export function invalidExpectation(expectation, problem = null) {
+  return { error: `invalid ${expectation.kind} expectation${problem === null ? "" : `: ${problem}`}` };
 }
 
 export function validTurn(group, turn) {
@@ -60,10 +80,10 @@ export function readPaths(run, turn) {
     if (Array.isArray(run.turns)) {
       return run.turns.flatMap((entry) => (Array.isArray(entry.successfulReadPaths) ? entry.successfulReadPaths : []));
     }
-    return Array.isArray(run.successfulReadPaths) ? run.successfulReadPaths : [];
+    return singleTurnReadPaths(run) ?? [];
   }
   if (!Array.isArray(run.turns)) {
-    return turn === 1 && Array.isArray(run.successfulReadPaths) ? run.successfulReadPaths : null;
+    return turn === 1 ? singleTurnReadPaths(run) : null;
   }
   const observedTurn = run.turns.find((entry) => entry.turn === turn);
   return observedTurn && Array.isArray(observedTurn.successfulReadPaths) ? observedTurn.successfulReadPaths : null;
@@ -124,3 +144,10 @@ export function compareText(left, right) {
   if (left > right) return 1;
   return 0;
 }
+
+// One-shot description runs record their reads under activation rather than at the top level.
+function singleTurnReadPaths(run) {
+  if (Array.isArray(run.successfulReadPaths)) return run.successfulReadPaths;
+  if (Array.isArray(run.activation?.successfulReadPaths)) return run.activation.successfulReadPaths;
+  return null;
+}

package/skills/{evaluate-skill → evaluate-behavior}/scripts/lib/grade-workspace.mjs

@@ -4,16 +4,18 @@ import {
   compareText,
   completedCheck,
   hasOwn,
+  invalidExpectation,
   isSafeRelativePath,
+  placementProblem,
   readWorkspaceFile,
   RESPONSE_EXPECTATIONS,
   textMatches,
   unavailableCheck,
-  validBodyExpectation,
   validPathArray,
   workspaceEvidence,
 } from "./grade-support.mjs";
 
+const BODY = ["body"];
 const JSON_EXPECTATIONS = new Set([
   "available",
   "missing",
@@ -26,13 +28,14 @@ const JSON_EXPECTATIONS = new Set([
 
 export async function gradePath(group, runs, expectation) {
   const turn = expectation.turn ?? null;
+  const placement = placementProblem(group, expectation, turn, BODY);
   if (
-    !validBodyExpectation(group, expectation, turn) ||
+    placement !== null ||
     !["exists", "absent"].includes(expectation.expect) ||
     typeof expectation.path !== "string" ||
     !isSafeRelativePath(expectation.path)
   ) {
-    return { error: "invalid path expectation" };
+    return invalidExpectation(expectation, placement);
   }
   const expected = { expect: expectation.expect, path: expectation.path, turn };
   const run = runs[expectation.variant];
@@ -56,12 +59,13 @@ export async function gradePath(group, runs, expectation) {
 
 export async function gradeChangedPaths(group, runs, expectation) {
   const turn = expectation.turn ?? null;
+  const placement = placementProblem(group, expectation, turn, BODY);
   if (
-    !validBodyExpectation(group, expectation, turn) ||
+    placement !== null ||
     !["equals", "excludes"].includes(expectation.expect) ||
     !validPathArray(expectation.paths)
   ) {
-    return { error: "invalid changed-paths expectation" };
+    return invalidExpectation(expectation, placement);
   }
   const expectedPaths = [...expectation.paths].sort(compareText);
   const expected = { expect: expectation.expect, paths: expectedPaths, turn };
@@ -81,16 +85,20 @@ export async function gradeChangedPaths(group, runs, expectation) {
 
 export async function gradeFileText(group, runs, expectation) {
   const turn = expectation.turn ?? null;
+  const line = expectation.line ?? null;
+  const placement = placementProblem(group, expectation, turn, BODY);
   if (
-    !validBodyExpectation(group, expectation, turn) ||
+    placement !== null ||
     !RESPONSE_EXPECTATIONS.has(expectation.expect) ||
     typeof expectation.value !== "string" ||
     typeof expectation.path !== "string" ||
-    !isSafeRelativePath(expectation.path)
+    !isSafeRelativePath(expectation.path) ||
+    (line !== null && !(Number.isInteger(line) && line >= 1))
   ) {
-    return { error: "invalid file-text expectation" };
+    return invalidExpectation(expectation, placement);
   }
   const expected = { expect: expectation.expect, path: expectation.path, value: expectation.value, turn };
+  if (line !== null) expected.line = line;
   const run = runs[expectation.variant];
   if (!run || run.state !== "complete") return unavailableCheck(expectation, expected);
   const workspace = workspaceEvidence(run, turn);
@@ -102,13 +110,22 @@ export async function gradeFileText(group, runs, expectation) {
     sha256: file.sha256,
     turn,
   };
-  const matches = file.state === "present" && textMatches(expectation.expect, expectation.value, file.text);
+  let text = file.state === "present" ? file.text : null;
+  if (line !== null) {
+    // One line, without its line ending. A line that does not exist fails every expectation,
+    // so an empty or short file cannot pass a not-contains check by having nothing to read.
+    const lines = text === null || text === "" ? [] : text.split(/\r?\n/);
+    text = line <= lines.length ? lines[line - 1] : null;
+    observed.line = text === null ? "missing" : "present";
+  }
+  const matches = text !== null && textMatches(expectation.expect, expectation.value, text);
   return completedCheck(expectation, expected, observed, matches);
 }
 
 export async function gradeJson(group, runs, expectation) {
   const turn = expectation.turn ?? null;
-  if (!validJsonExpectation(group, expectation, turn)) return { error: "invalid json expectation" };
+  const placement = placementProblem(group, expectation, turn, BODY);
+  if (placement !== null || !validJsonExpectation(expectation)) return invalidExpectation(expectation, placement);
 
   const expected = {
     expect: expectation.expect,
@@ -126,10 +143,9 @@ export async function gradeJson(group, runs, expectation) {
   return completedCheck(expectation, expected, observed, jsonMatches(expectation, observed));
 }
 
-function validJsonExpectation(group, expectation, turn) {
+function validJsonExpectation(expectation) {
   const fieldExpectation = ["field-present", "field-absent", "field-equals"].includes(expectation.expect);
   return (
-    validBodyExpectation(group, expectation, turn) &&
     typeof expectation.path === "string" &&
     isSafeRelativePath(expectation.path) &&
     JSON_EXPECTATIONS.has(expectation.expect) &&

package/skills/evaluate-behavior/scripts/lib/grade.mjs

@@ -0,0 +1,101 @@
+import { armNames, referenceArm } from "./arms.mjs";
+import { gradeCommand } from "./grade-command.mjs";
+import { gradeComparisons } from "./grade-comparison.mjs";
+import {
+  gradeContextText,
+  gradeResourceRead,
+  gradeResponseText,
+  gradeSkillRead,
+  gradeToolCall,
+} from "./grade-interaction.mjs";
+import { gradeChangedPaths, gradeFileText, gradeJson, gradePath } from "./grade-workspace.mjs";
+
+const GRADERS = new Map([
+  ["skill-read", gradeSkillRead],
+  ["resource-read", gradeResourceRead],
+  ["path", gradePath],
+  ["changed-paths", gradeChangedPaths],
+  ["file-text", gradeFileText],
+  ["json", gradeJson],
+  ["response-text", gradeResponseText],
+  ["context-text", gradeContextText],
+  ["tool-call", gradeToolCall],
+  ["command", gradeCommand],
+]);
+
+/**
+ * @param {any} group
+ * @param {any} runs
+ * @param {any} evidence
+ * @returns {Promise<any>}
+ */
+export async function gradeDeterministic(group, runs, evidence) {
+  const { checks, errors } = await gradeEach(group, runs, group.expectations);
+  const compared = gradeComparisons(group.expectations, checks, {
+    arms: armNames(group),
+    reference: referenceArm(group),
+  });
+  errors.push(...compared.errors);
+  return {
+    schema_version: 1,
+    state: reportState(checks, errors),
+    evidence,
+    checks,
+    comparisons: compared.comparisons,
+    errors,
+  };
+}
+
+/**
+ * What grading would refuse in a loaded group's expectations that the definition alone decides: a kind in a group
+ * type it cannot grade, an arm or turn the group lacks, field values its grader rejects, or comparison members
+ * that do not describe one check in every arm. Each grader runs with no run evidence, so the check is the
+ * grader's own and runs before anything is spent. An unknown kind, and any comparison it belongs to, is left to
+ * grading, where it is a grade error.
+ *
+ * @returns {Promise<Array<{ expectation?: string, comparison?: string, reason: string }>>}
+ */
+export async function expectationProblems(group) {
+  const known = group.expectations.filter((expectation) => GRADERS.has(expectation.kind));
+  const { checks, errors } = await gradeEach(group, {}, known);
+  const unknownComparisons = new Set(
+    group.expectations
+      .filter((expectation) => !GRADERS.has(expectation.kind) && Object.hasOwn(expectation, "comparison"))
+      .map((expectation) => expectation.comparison),
+  );
+  const compared = gradeComparisons(
+    group.expectations.filter((expectation) => !unknownComparisons.has(expectation.comparison)),
+    checks,
+    { arms: armNames(group), reference: referenceArm(group) },
+  );
+  return [
+    ...errors.map(({ id, reason }) => ({ expectation: id, reason })),
+    ...compared.errors.map(({ id, reason }) => ({ comparison: id, reason })),
+  ];
+}
+
+async function gradeEach(group, runs, expectations) {
+  const checks = /** @type {Array<any>} */ ([]);
+  const errors = /** @type {Array<any>} */ ([]);
+  for (const expectation of expectations) {
+    const grader = GRADERS.get(expectation.kind);
+    if (!grader) {
+      errors.push({ id: expectation.id, reason: `unsupported expectation kind: ${expectation.kind}` });
+      continue;
+    }
+    try {
+      const result = await grader(group, runs, expectation);
+      if (result.error) errors.push({ id: expectation.id, reason: result.error });
+      else checks.push(result);
+    } catch (error) {
+      errors.push({ id: expectation.id, reason: error instanceof Error ? error.message : String(error) });
+    }
+  }
+  return { checks, errors };
+}
+
+function reportState(checks, errors) {
+  if (errors.length > 0) return "grade-error";
+  if (checks.length > 0 && checks.every((check) => check.state === "unavailable")) return "unavailable";
+  return "complete";
+}

package/skills/evaluate-behavior/scripts/lib/hooks.mjs

@@ -0,0 +1,98 @@
+// Extension points for later evaluator features. Every registry starts empty, and with nothing registered the
+// evaluator writes exactly what it wrote before. A hook that throws is recorded as `{ error }` in its own slot;
+// it never fails the run, the grade or the aggregate.
+
+const HOOK_NAME = /^[a-z][A-Za-z0-9]*$/;
+// Fields an aggregate group entry already has; a section may not replace one.
+const AGGREGATE_GROUP_FIELDS = new Set([
+  "id",
+  "trials",
+  "unreadableGrades",
+  "gradeErrors",
+  "runStates",
+  "comparisons",
+  "checks",
+]);
+
+function createRegistry(kind, reserved = new Set()) {
+  const hooks = new Map();
+  return {
+    register(name, hook) {
+      if (typeof name !== "string" || !HOOK_NAME.test(name) || reserved.has(name)) {
+        throw new Error(`${kind} name must be a camelCase identifier that is not reserved: ${name}`);
+      }
+      if (typeof hook !== "function") throw new Error(`${kind} ${name} must be a function`);
+      if (hooks.has(name)) throw new Error(`${kind} ${name} is already registered`);
+      hooks.set(name, hook);
+      return () => {
+        if (hooks.get(name) === hook) hooks.delete(name);
+      };
+    },
+    entries() {
+      return [...hooks.entries()];
+    },
+  };
+}
+
+function failure(error) {
+  return { error: error instanceof Error ? error.message : String(error) };
+}
+
+const aggregators = createRegistry("aggregator", AGGREGATE_GROUP_FIELDS);
+const postRunHooks = createRegistry("post-run hook");
+const runMetricsHooks = createRegistry("run-metrics hook");
+
+/** Aggregator: `hook({ id, arms, reference, trials: [{ id, directory, state, grade, runs }] })`, where `directory` is the trial's group directory; its result is stored on the aggregate group entry under `<name>`. */
+export const registerAggregator = aggregators.register;
+
+/** Post-run hook: `hook({ group, arm, run, root, directory, signal })` for each executed run, before run.json is written; its result is stored in run.json under `postRun.<name>`. */
+export const registerPostRunHook = postRunHooks.register;
+
+/** Run-metrics hook: synchronous `hook({ group, arm, subject, observation, run })` where pi.mjs assembles run.json; its returned object is merged into `run.metrics`. */
+export const registerRunMetricsHook = runMetricsHooks.register;
+
+export function hasAggregators() {
+  return aggregators.entries().length > 0;
+}
+
+/** Each registered aggregator's section for one aggregate group, by name. */
+export async function aggregateSections(input) {
+  const sections = {};
+  for (const [name, hook] of aggregators.entries()) {
+    try {
+      sections[name] = await hook(input);
+    } catch (error) {
+      sections[name] = failure(error);
+    }
+  }
+  return sections;
+}
+
+/** Each registered post-run hook's record for one run, by name; a hook that returns undefined records nothing, and null means nothing was recorded. */
+export async function postRunRecords(context) {
+  const records = {};
+  for (const [name, hook] of postRunHooks.entries()) {
+    try {
+      const record = await hook(context);
+      if (record !== undefined) records[name] = record;
+    } catch (error) {
+      records[name] = failure(error);
+    }
+  }
+  return Object.keys(records).length === 0 ? null : records;
+}
+
+/** The registered run-metrics hooks' fields merged in registration order; null when none is registered. */
+export function runMetrics(context) {
+  const entries = runMetricsHooks.entries();
+  if (entries.length === 0) return null;
+  const metrics = {};
+  for (const [name, hook] of entries) {
+    try {
+      Object.assign(metrics, hook(context));
+    } catch (error) {
+      metrics[name] = failure(error);
+    }
+  }
+  return metrics;
+}