@hassangameryt/freeflow 0.8.0 → 0.9.0

package/skills/discuss-work/scripts/lib/choices.mjs

@@ -0,0 +1,159 @@
+// Discuss Work's decision list as a record file: the task's sources/decisions.md, or a list given on stdin or as a file
+// when there is no Working Record. One line per item, "- P1 [state] choice -> default ...", other lines ignored
+// (Discuss Work, "Build The Decision List").
+//   validate   IDs, states, T -> P links, a default on every plain item, the user's words on agreed items, a reason on
+//              changed ones;
+//   agreement  the reply sent before building: What I'll do / Where I stop / Your decisions / Technical: N;
+//   handover   the items a hand-over shows: new, changed and default;
+//   catch-up   after an unattended run: defaults taken, choices waiting, changes, and what ran (the record's current
+//              Slice Material updates).
+
+export const CHOICE_STATES = new Set(["proposed", "agreed", "default", "changed", "new", "dropped"]);
+const ITEM = /^\s*[-*]\s+([A-Za-z]+\d+)\s+\[([^\]]*)\]\s+(.*)$/;
+
+export function parseChoices(text) {
+  const items = [];
+  const errors = [];
+  const warnings = [];
+  text.split(/\r?\n/).forEach((raw, index) => {
+    const match = ITEM.exec(raw);
+    if (!match) return;
+    const [, id, state, rest] = match;
+    const arrow = rest.indexOf("->");
+    const choice = (arrow === -1 ? rest : rest.slice(0, arrow)).trim();
+    const tail = arrow === -1 ? "" : rest.slice(arrow + 2).trim();
+    // A technical item states its choice and the plain item it serves: "T1 [default] FIFO array -> serves P1".
+    const serves = /^(?:serves\s+)?(P\d+)\b/i.exec(tail);
+    items.push({
+      line: index + 1,
+      id,
+      kind: id[0] === "P" ? "plain" : id[0] === "T" ? "technical" : "unknown",
+      state,
+      choice,
+      default: serves ? choice : tail,
+      serves: serves?.[1] ?? null,
+      quote: /\bYou:\s*["“]/.test(rest),
+      reason: /\b(because|why|was|since|instead|reason)\b/i.test(tail),
+      text: rest.trim(),
+    });
+  });
+
+  const seen = new Map();
+  for (const item of items) {
+    const where = `line ${item.line} ${item.id}`;
+    if (!/^[PT][1-9]\d*$/.test(item.id)) errors.push(`${where}: ID must be P<n> or T<n>`);
+    if (seen.has(item.id)) errors.push(`${where}: duplicate ID (first at line ${seen.get(item.id)})`);
+    else seen.set(item.id, item.line);
+    if (!CHOICE_STATES.has(item.state))
+      errors.push(`${where}: state "${item.state}" is not one of ${[...CHOICE_STATES].join(", ")}`);
+    if (!item.choice) errors.push(`${where}: names no choice`);
+    if (item.kind === "plain" && !item.default)
+      errors.push(`${where}: has no default after "->" (what matches existing behavior and is easiest to reverse)`);
+    if (item.kind === "technical" && !item.serves)
+      errors.push(`${where}: a technical item names the plain item it serves ("-> serves P1")`);
+    if (item.state === "agreed" && !item.quote) warnings.push(`${where}: agreed without the user's words (You: "...")`);
+    if (item.state === "changed" && !item.reason) warnings.push(`${where}: changed without saying why`);
+  }
+  for (const item of items.filter((each) => each.serves)) {
+    const target = items.find((each) => each.id === item.serves);
+    if (!target) errors.push(`line ${item.line} ${item.id}: serves ${item.serves}, which is not in the list`);
+    else if (target.state === "dropped" && item.state !== "dropped")
+      warnings.push(`line ${item.line} ${item.id}: serves ${item.serves}, which is dropped`);
+  }
+  if (!items.length) errors.push('no items found: each item is a line "- P1 [state] choice -> default"');
+  const firstTechnical = items.findIndex((item) => item.kind === "technical");
+  if (firstTechnical !== -1 && items.slice(firstTechnical).some((item) => item.kind === "plain"))
+    warnings.push("plain items come first; a P item follows a T item");
+  return { items, errors, warnings, ok: errors.length === 0 };
+}
+
+const itemLine = (item) => `- ${item.id} [${item.state}] ${item.text}`;
+const byState = (items, states) => items.filter((item) => states.includes(item.state));
+
+// The current Slice's Material updates: what ran or changed while no one was answering.
+function currentSliceUpdates(recordText) {
+  const lines = recordText.split(/\r?\n/);
+  const start = lines.findIndex((each) => /^### Current Slice\s*$/.test(each));
+  if (start === -1) return { slice: "no Current Slice", updates: [] };
+  const end = lines.findIndex((each, index) => index > start && /^#{2,3} /.test(each));
+  const slice = lines.slice(start, end === -1 ? undefined : end);
+  const title = slice.find((each) => /^#### S-\d+/.test(each));
+  const at = slice.findIndex((each) => /^##### Material updates\s*$/.test(each));
+  const updates = at === -1 ? [] : slice.slice(at + 1).filter((each) => /^\s*-\s+\S/.test(each));
+  return {
+    slice: title ? title.replace(/^####\s+/, "") : "no Current Slice",
+    updates: updates.map((each) => each.trim().replace(/^-\s+/, "")),
+  };
+}
+
+// Renders one view; returns { text, data }. The caller validates first.
+export function renderChoices(view, parsed, { outcome, stop, showTechnical = false, recordText = null } = {}) {
+  const { items } = parsed;
+  const plain = items.filter((item) => item.kind === "plain" && item.state !== "dropped");
+  const technical = items.filter((item) => item.kind === "technical" && item.state !== "dropped");
+  if (view === "validate")
+    return { text: `Decision list valid: ${items.length} item(s).`, data: { items: items.length } };
+  if (view === "agreement") {
+    const open = plain.filter((item) => item.state !== "agreed");
+    const pad = "                ";
+    const decisions = open.length
+      ? open.map(
+          (item, index) =>
+            `${index ? pad : ""}${index + 1}. ${item.choice}${/[?.!]$/.test(item.choice) ? "" : "?"} I'd go with ${item.default}`,
+        )
+      : ["none: the request and the repository settle every choice the user would see."];
+    const technicalLine = showTechnical
+      ? [
+          `Technical:      ${technical.map((item, index) => `${index ? pad : ""}${item.id} ${item.choice} (serves ${item.serves})`).join("\n") || "none"}`,
+        ]
+      : technical.length
+        ? [
+            `Technical:      ${technical.length} more choice${technical.length === 1 ? "" : "s"}; say "show technical" to see them.`,
+          ]
+        : [];
+    const text = [
+      `What I'll do:   ${outcome}`,
+      `Where I stop:   ${stop}`,
+      `Your decisions: ${decisions.join("\n")}`,
+      ...technicalLine,
+    ].join("\n");
+    return { text, data: { decisions: open.map((item) => item.id), technical: technical.length } };
+  }
+  if (view === "handover") {
+    const shown = byState(items, ["new", "changed", "default"]);
+    return {
+      text: shown.length
+        ? `Choices you have not seen (${shown.length}):\n${shown.map(itemLine).join("\n")}`
+        : "No new, changed or default choices: every item was agreed or dropped.",
+      data: { shown: shown.map((item) => item.id) },
+    };
+  }
+  // catch-up
+  const defaults = byState(items, ["default"]);
+  const waiting = byState(items, ["proposed", "new"]);
+  const changed = byState(items, ["changed"]);
+  const out = [`Defaults taken (${defaults.length}):`, ...(defaults.length ? defaults.map(itemLine) : ["- none"])];
+  out.push(
+    "",
+    `Choices waiting for you (${waiting.length}):`,
+    ...(waiting.length ? waiting.map(itemLine) : ["- none"]),
+  );
+  if (changed.length) out.push("", `Changed while you were away (${changed.length}):`, ...changed.map(itemLine));
+  const ran = recordText === null ? null : currentSliceUpdates(recordText);
+  if (ran)
+    out.push(
+      "",
+      `What ran (${ran.slice}, Material updates: ${ran.updates.length}):`,
+      ...(ran.updates.length ? ran.updates.map((each) => `- ${each}`) : ["- nothing recorded"]),
+    );
+  else out.push("", "What ran: no Working Record; account for it from the conversation.");
+  return {
+    text: out.join("\n"),
+    data: {
+      defaults: defaults.map((item) => item.id),
+      waiting: waiting.map((item) => item.id),
+      changed: changed.map((item) => item.id),
+      ...(ran ? { slice: ran.slice, ran: ran.updates } : {}),
+    },
+  };
+}

package/skills/evaluate-behavior/SKILL.md

@@ -0,0 +1,107 @@
+---
+name: evaluate-behavior
+description: "Use when testing, evaluating or comparing how an agent behaves: whether a skill triggers, whether a change to a prompt, skill, model, setting or extension changes outcomes, measuring a reviewer or checker, or benchmarking agents on coding tasks, conversations or pipelines. Also use when writing or checking an eval's JSON files, running an eval and reading its results, and deciding what a result can claim, how many cases and trials it needs and what it costs."
+---
+
+# Evaluate Behavior
+
+Determine whether a declared change to an agent's setup changes its behavior as intended under declared pressure. Evidence outranks confident prose.
+
+A **configuration** is everything that can shape the agent's behavior: host, model, skills, instruction files, and host extensions and settings. A **group** asks one behavioral question by running named **arms** of one configuration on the same task, usually `baseline` (the **reference**: the setup without the change, such as no target skill, the previous commit, or the setting off) and `candidate` (the same setup with exactly the declared change, its **component**); a single arm measures one setup alone. The **subject** is the agent performing the task in one arm. The **evaluator** isolates subjects and preserves canonical evidence; the **deterministic grader** derives fixed facts after the run is saved; a **judge** is a separate model whose answers are **judged evidence**. A **suite** runs ordered groups and may declare a **decision** rule that turns their results into a verdict.
+
+Evaluate Behavior does not change what it evaluates, claim readiness, or choose adoption. The component's owner revises it: [Write Skill](../write-skill/SKILL.md) for a skill or other instruction text; extension code goes to its owner, and a model, setting or extension choice to the user.
+
+The **Rules** below are binding; if one cannot be met, stop and say why. Everything else is **judgment**: follow its stated goal, and adapt when a different choice serves that goal better.
+
+## Routes
+
+| Situation | Go to |
+| --- | --- |
+| The question, evidence class, arms, pressure or component; what the result can claim, the decision rule, cases and trials, noise, judge counts, cost | [design and measurement](references/design-and-measurement.md) |
+| Measuring a reviewer, checker, or other detector | its [Defect Suites](references/design-and-measurement.md#defect-suites) |
+| Writing or checking group, suite, or configuration JSON | [authoring](references/authoring.md) |
+| A coding task (hidden tests, declared commands, a shell, a container, network), or the Claude Code host | [coding tasks](references/coding-tasks.md) |
+| A conversation, a pipeline, a rubric and judges, or a runtime profile | [conversations, pipelines and judges](references/conversations-pipelines-judges.md) |
+| Running the evaluator and reading its results; evidence that conflicts, the failed boundary, a revision target, or rerun scope | [running and evidence](references/running-and-evidence.md), and `skill-eval --help` |
+| Reviewing a definition before paying for its runs | Review Work's [Eval Definition Review](../review-work/references/evals-and-judges.md#eval-definition-review) |
+| Choosing a judge or relying on its answers | Review Work's [Judge Contract](../review-work/references/evals-and-judges.md#judge-contract) |
+
+## Rules
+
+- Never mutate canonical evidence or erase an unfavorable result. Preserve partial and infrastructure evidence without converting unavailable behavior into pass or fail.
+- Never fabricate behavioral evidence. Missing evidence stays unavailable.
+- Compare a change only against an exact immutable baseline, such as a commit. If none exists, preserve the current state first (for a skill or instruction text, through Write Skill); never create a baseline by editing what is being evaluated.
+- Keep deterministic criteria, grader files, review questions and rubric references out of subject prompts, the fixture and declared subject context.
+- Declare every part of the configuration that can change behavior. Ambient installation, settings, or context are not declared composition.
+- Claim only what the evidence class observed; one class never proves another.
+- Never write that one arm beats another without paired runs on the same cases and a suite `decision` declared before the run; the evaluator reports "beats" only from that rule. Below that, report counts.
+- Never cite a dry run as evidence. When a run is refused for uncommitted inputs, commit the candidate and the definition, or keep the run as a dry run for your own checking only.
+- Never change the component under test while evaluating it. Return one measured revision target to its owner.
+- After a behavioral change, rerun the complete fixed group with every arm. Never hand-pick an earlier run as the fresh counterpart or rerun only the preferred arm; an earlier run may stand in only through `--reuse`, which matches the arm's exact inputs, and the report names it as reused.
+- Judged evidence is its own class. Judge answers are stored apart from run and deterministic grade files, with each judge's identity and calibration status, and count toward a claim only through a decision rule declared before the run. They never change run or grade files and never establish readiness.
+- Installing anything an eval needs, such as the VM runtime behind a confined shell, is the user's decision. Relay the evaluator's exact refusal and ask before installing; never install silently.
+- Never invoke archived evaluators.
+- Never clear a run's transcript-scan flag on your own judgment. Show the user the flag and its evidence, and run `skill-eval clear` only with their decision and their reason.
+
+## Design The Smallest Valid Comparison
+
+Goal: the arms differ only in the component, under pressure strong enough to show it, in a run no larger than the claim needs.
+
+Answer from saved evidence or an existing group when they still fit (the same question, pressure, arms and fixed criteria); for a draft-only request, use Write Skill. Do not rerun merely to demonstrate process. Name one behavioral question, its evidence class, and the strongest claim the result must support: can do X, a rate on this suite, or A beats B; readiness is beyond Evaluate Behavior. Activation, first-read body and nearby behavior are questions about skills; for other components, check delivery, behavior and artifact outcome first.
+
+Choose the component:
+
+| Change under test | Component | Baseline |
+| --- | --- | --- |
+| A new or revised skill | `subject.skills`, `subject.target`, `subject.source` | no target, or the previous commit |
+| An instruction or context file | `subject.context` | without the file, or its previous commit |
+| A model or thinking level | `model.model`, `model.thinking` | the model in use |
+| An extension or plugin, or a host setting | `host.extensions`, `host.settings` | without it, or its current version or value |
+
+The task's fixture and the runtime are shared by every arm and cannot be the component.
+
+Choose the mode by where the behavior happens; modes combine:
+
+- **Compare:** arms run the same task once per trial; the default.
+- **Noise check:** `component: []` with identical arms; their gap is the floor a real difference must clear.
+- **Measure:** one arm; a rate on this suite, never a comparison.
+- **Conversation:** a simulated user who knows only its card answers back.
+- **Pipeline:** ordered stages, each a subject that sees only what it declares.
+- **Coding task:** hidden tests or other commands grade the changed repository.
+- **Judged evaluation:** judges answer a rubric for each run alone, blind to its arm.
+
+Before subject output exists:
+
+1. fix one behavioral question;
+2. fix deterministic expectations, rubric and review questions;
+3. preserve the earliest natural prompt or strongest pressure capable of distinguishing behavior;
+4. declare each arm's exact identity, and the suite's decision rule when the claim is a comparison;
+5. keep prompt or turns, fixture, tools, and every configuration part other than the component fixed, along with the criteria;
+6. run one group or an ordered suite;
+7. inspect canonical evidence and deterministic facts before judging meaning;
+8. classify the failed boundary before proposing revision.
+
+If every arm passes, the pressure may be weak or the baseline already sufficient. If every arm fails, distinguish component, fixture, dependency, environment, host, isolation, and criterion failures before revising anything.
+
+## Run And Read The Evidence
+
+Run the canonical [skill evaluator](scripts/skill-eval.mjs) from the definition root: `node <evaluate-behavior-directory>/scripts/skill-eval.mjs <run|check-tasks|view|review|clear> ...`. Run `check-tasks`, which calls no model, before spending on a coding task; use `--trials <n>` before claiming a difference, and `--max-cost` when a batch could spend more than intended. Canonical results live under `<definition-root>/.skill-eval/runs/<result-id>`; a view is a projection of them. A failed check is behavioral evidence; a transition such as `fail-to-pass` is a fact, not a verdict.
+
+## Report And Stop
+
+Report:
+
+- what ran;
+- each arm's exact identity, and which is the reference;
+- selected groups and arms, and any run reused from an earlier result;
+- deterministic results, with case and trial counts and pass^k;
+- metrics that bear on the question, such as cost, turns or duration per arm;
+- the decision's verdict, with its split, p and rule, when the suite declares one, and otherwise the claim class reached;
+- judged evidence apart from deterministic results, with each judge's identity and calibration status;
+- infrastructure failures, invalid runs with their reasons, runs held for uncleared transcript flags, cancellation, or unavailable evidence;
+- viewed scope and canonical result paths;
+- what the evidence proves;
+- what it does not prove;
+- any supported failed boundary or revision target.
+
+Stop rather than overclaim when required delivery, activation, first-read, composition, retained-use, isolation, artifact, or host evidence is unavailable. The user decides whether to revise, adopt, or reject the change.