feed-the-machine 1.5.0 → 1.6.0

package/ftm-brainstorm/evals/evals.json

@@ -1,100 +1,100 @@
-{
-  "skill_name": "ftm-brainstorm",
-  "evals": [
-    {
-      "id": 0,
-      "name": "fresh-idea-intake",
-      "prompt": "I'm thinking about building an app that helps people find study buddies at their university. Like Tinder but for studying.",
-      "expected_output": "Phase 0 repo scan launched in background, 1-2 intake questions, hard STOP",
-      "files": [],
-      "assertions": [
-        {"name": "one_or_two_questions", "description": "Asks 1-2 questions max, not a wall of questions"},
-        {"name": "no_research_sprint_turn1", "description": "Does NOT dispatch research agents on the very first turn — intake only"},
-        {"name": "hard_stop", "description": "Ends with a question and waits — does NOT proceed to synthesize or generate suggestions"},
-        {"name": "repo_scan_launched", "description": "Mentions or silently launches a repo/codebase scan agent in background"}
-      ]
-    },
-    {
-      "id": 1,
-      "name": "fresh-idea-turn2-research",
-      "prompt": "It's for college students who want to find people in the same classes to study with. Matching based on courses, study style, and schedule availability.",
-      "expected_output": "First research sprint dispatched (3 agents), synthesized suggestions with citations, challenge, 1-2 questions, STOP",
-      "files": [],
-      "multi_turn_context": "Turn 2. Turn 1: user said 'building study buddy app like Tinder for studying', skill asked intake questions, user now responds with details.",
-      "assertions": [
-        {"name": "three_agents_dispatched", "description": "Dispatches 3 parallel research agents (web, github, competitive) — not fewer"},
-        {"name": "real_citations", "description": "At least 3 unique URLs to real repos/articles/products in the synthesis"},
-        {"name": "suggestions_with_evidence", "description": "Presents numbered suggestions (3-5) with real-world evidence backing each one"},
-        {"name": "recommendation_labeled", "description": "Suggestion #1 is labeled RECOMMENDED with rationale"},
-        {"name": "challenge_present", "description": "Includes at least one challenge/pushback after suggestions"},
-        {"name": "ends_with_question", "description": "Ends with 1-2 targeted questions to drive next research sprint"},
-        {"name": "hard_stop", "description": "Does NOT continue past the questions — waits for user response"},
-        {"name": "depth_is_broad", "description": "Research queries are landscape-level (major approaches, who's done this) not implementation-specific"}
-      ]
-    },
-    {
-      "id": 2,
-      "name": "turn3-deeper-research",
-      "prompt": "I like option 2 — the React Native approach with Firebase. But I'm worried about the matching algorithm complexity. Also we need to handle the cold-start problem when a new university joins.",
-      "expected_output": "Second research sprint (deeper, focused on RN+Firebase+matching), new suggestions building on prior, new challenges, new questions",
-      "files": [],
-      "multi_turn_context": "Turn 3. Prior turns: user described study buddy app, first research sprint found 5 approaches, user now picks one and raises two specific concerns.",
-      "assertions": [
-        {"name": "three_agents_dispatched", "description": "Dispatches 3 parallel research agents again — every turn gets a research sprint"},
-        {"name": "research_is_deeper", "description": "Search queries target matching algorithms and cold-start specifically, NOT broad 'study buddy app' landscape again"},
-        {"name": "builds_on_prior", "description": "References prior turn's findings — does not re-present the same 5 approaches"},
-        {"name": "new_citations", "description": "At least 2 URLs not seen in prior turns — fresh research, not recycled"},
-        {"name": "addresses_user_concerns", "description": "Suggestions specifically address matching algorithm complexity AND cold-start problem"},
-        {"name": "challenge_present", "description": "Challenges the user on at least one assumption or pushes back on scope"},
-        {"name": "ends_with_question", "description": "Ends with 1-2 questions that unlock the NEXT research vector"},
-        {"name": "hard_stop", "description": "Does NOT continue past the questions"}
-      ]
-    },
-    {
-      "id": 3,
-      "name": "brain-dump-intake",
-      "prompt": "help me build all the suggestions in this chat: [brain dump about eng-buddy]",
-      "expected_output": "Path B structured extraction with repo context, confirmation gate, no research yet",
-      "files": ["brain-dump-input.md"],
-      "assertions": [
-        {"name": "decided_section", "description": "Contains a 'Decided' or 'Decisions already made' section"},
-        {"name": "open_questions_section", "description": "Contains an 'Open questions' or 'Gaps' section"},
-        {"name": "no_basic_questions", "description": "Does NOT ask basic 'what are you building?' questions already answered by the paste"},
-        {"name": "confirmation_gate", "description": "Ends with a confirmation question before proceeding to research"},
-        {"name": "no_research_sprint", "description": "Does NOT dispatch research agents or present suggestions on this turn"},
-        {"name": "hard_stop", "description": "Stops after asking for confirmation — does not proceed"}
-      ]
-    },
-    {
-      "id": 4,
-      "name": "brain-dump-turn2-research",
-      "prompt": "Yeah that looks right, go ahead and research it",
-      "expected_output": "First research sprint in brain dump mode: novelty map, suggestions with citations, challenges",
-      "files": ["brain-dump-input.md"],
-      "multi_turn_context": "Turn 2 of brain dump. Turn 1: user pasted eng-buddy brain dump, skill extracted structured summary, user now confirms.",
-      "assertions": [
-        {"name": "three_agents_dispatched", "description": "Dispatches 3 parallel research agents searching for specific brain dump claims"},
-        {"name": "novelty_map_present", "description": "Contains a Novelty Map table with solved/partially solved/novel verdicts"},
-        {"name": "real_citations", "description": "At least 5 unique URLs to real repos/articles/products"},
-        {"name": "brain_dump_claims_researched", "description": "Agents searched for specific architectural claims from the dump, not just broad topic searches"},
-        {"name": "challenge_present", "description": "At least one challenge/pushback raised"},
-        {"name": "ends_with_question", "description": "Ends with 1-2 targeted questions"},
-        {"name": "hard_stop", "description": "Does NOT proceed past questions"}
-      ]
-    },
-    {
-      "id": 5,
-      "name": "phase3-gate",
-      "prompt": "Ok I think I know what I want. Let's turn this into a plan.",
-      "expected_output": "Vision summary for approval, NOT the full plan yet",
-      "files": [],
-      "multi_turn_context": "Turn 6+ of brainstorming. Previous turns explored study-buddy app, settled on React Native + Firebase, matching algorithm, cold-start solution. User now wants to move to planning.",
-      "assertions": [
-        {"name": "vision_summary", "description": "Presents a brief 'here's what we've landed on' summary before generating the full plan"},
-        {"name": "approval_gate", "description": "Asks for explicit confirmation before proceeding to full plan generation"},
-        {"name": "does_not_dump_full_plan", "description": "Does NOT generate the entire task breakdown, agent assignments, and wave structure in this message"},
-        {"name": "references_plan_template", "description": "Reads or references references/plan-template.md for plan generation"}
-      ]
-    }
-  ]
-}
+{
+  "skill_name": "ftm-brainstorm",
+  "evals": [
+    {
+      "id": 0,
+      "name": "fresh-idea-intake",
+      "prompt": "I'm thinking about building an app that helps people find study buddies at their university. Like Tinder but for studying.",
+      "expected_output": "Phase 0 repo scan launched in background, 1-2 intake questions, hard STOP",
+      "files": [],
+      "assertions": [
+        {"name": "one_or_two_questions", "description": "Asks 1-2 questions max, not a wall of questions"},
+        {"name": "no_research_sprint_turn1", "description": "Does NOT dispatch research agents on the very first turn — intake only"},
+        {"name": "hard_stop", "description": "Ends with a question and waits — does NOT proceed to synthesize or generate suggestions"},
+        {"name": "repo_scan_launched", "description": "Mentions or silently launches a repo/codebase scan agent in background"}
+      ]
+    },
+    {
+      "id": 1,
+      "name": "fresh-idea-turn2-research",
+      "prompt": "It's for college students who want to find people in the same classes to study with. Matching based on courses, study style, and schedule availability.",
+      "expected_output": "First research sprint dispatched (3 agents), synthesized suggestions with citations, challenge, 1-2 questions, STOP",
+      "files": [],
+      "multi_turn_context": "Turn 2. Turn 1: user said 'building study buddy app like Tinder for studying', skill asked intake questions, user now responds with details.",
+      "assertions": [
+        {"name": "three_agents_dispatched", "description": "Dispatches 3 parallel research agents (web, github, competitive) — not fewer"},
+        {"name": "real_citations", "description": "At least 3 unique URLs to real repos/articles/products in the synthesis"},
+        {"name": "suggestions_with_evidence", "description": "Presents numbered suggestions (3-5) with real-world evidence backing each one"},
+        {"name": "recommendation_labeled", "description": "Suggestion #1 is labeled RECOMMENDED with rationale"},
+        {"name": "challenge_present", "description": "Includes at least one challenge/pushback after suggestions"},
+        {"name": "ends_with_question", "description": "Ends with 1-2 targeted questions to drive next research sprint"},
+        {"name": "hard_stop", "description": "Does NOT continue past the questions — waits for user response"},
+        {"name": "depth_is_broad", "description": "Research queries are landscape-level (major approaches, who's done this) not implementation-specific"}
+      ]
+    },
+    {
+      "id": 2,
+      "name": "turn3-deeper-research",
+      "prompt": "I like option 2 — the React Native approach with Firebase. But I'm worried about the matching algorithm complexity. Also we need to handle the cold-start problem when a new university joins.",
+      "expected_output": "Second research sprint (deeper, focused on RN+Firebase+matching), new suggestions building on prior, new challenges, new questions",
+      "files": [],
+      "multi_turn_context": "Turn 3. Prior turns: user described study buddy app, first research sprint found 5 approaches, user now picks one and raises two specific concerns.",
+      "assertions": [
+        {"name": "three_agents_dispatched", "description": "Dispatches 3 parallel research agents again — every turn gets a research sprint"},
+        {"name": "research_is_deeper", "description": "Search queries target matching algorithms and cold-start specifically, NOT broad 'study buddy app' landscape again"},
+        {"name": "builds_on_prior", "description": "References prior turn's findings — does not re-present the same 5 approaches"},
+        {"name": "new_citations", "description": "At least 2 URLs not seen in prior turns — fresh research, not recycled"},
+        {"name": "addresses_user_concerns", "description": "Suggestions specifically address matching algorithm complexity AND cold-start problem"},
+        {"name": "challenge_present", "description": "Challenges the user on at least one assumption or pushes back on scope"},
+        {"name": "ends_with_question", "description": "Ends with 1-2 questions that unlock the NEXT research vector"},
+        {"name": "hard_stop", "description": "Does NOT continue past the questions"}
+      ]
+    },
+    {
+      "id": 3,
+      "name": "brain-dump-intake",
+      "prompt": "help me build all the suggestions in this chat: [brain dump about eng-buddy]",
+      "expected_output": "Path B structured extraction with repo context, confirmation gate, no research yet",
+      "files": ["brain-dump-input.md"],
+      "assertions": [
+        {"name": "decided_section", "description": "Contains a 'Decided' or 'Decisions already made' section"},
+        {"name": "open_questions_section", "description": "Contains an 'Open questions' or 'Gaps' section"},
+        {"name": "no_basic_questions", "description": "Does NOT ask basic 'what are you building?' questions already answered by the paste"},
+        {"name": "confirmation_gate", "description": "Ends with a confirmation question before proceeding to research"},
+        {"name": "no_research_sprint", "description": "Does NOT dispatch research agents or present suggestions on this turn"},
+        {"name": "hard_stop", "description": "Stops after asking for confirmation — does not proceed"}
+      ]
+    },
+    {
+      "id": 4,
+      "name": "brain-dump-turn2-research",
+      "prompt": "Yeah that looks right, go ahead and research it",
+      "expected_output": "First research sprint in brain dump mode: novelty map, suggestions with citations, challenges",
+      "files": ["brain-dump-input.md"],
+      "multi_turn_context": "Turn 2 of brain dump. Turn 1: user pasted eng-buddy brain dump, skill extracted structured summary, user now confirms.",
+      "assertions": [
+        {"name": "three_agents_dispatched", "description": "Dispatches 3 parallel research agents searching for specific brain dump claims"},
+        {"name": "novelty_map_present", "description": "Contains a Novelty Map table with solved/partially solved/novel verdicts"},
+        {"name": "real_citations", "description": "At least 5 unique URLs to real repos/articles/products"},
+        {"name": "brain_dump_claims_researched", "description": "Agents searched for specific architectural claims from the dump, not just broad topic searches"},
+        {"name": "challenge_present", "description": "At least one challenge/pushback raised"},
+        {"name": "ends_with_question", "description": "Ends with 1-2 targeted questions"},
+        {"name": "hard_stop", "description": "Does NOT proceed past questions"}
+      ]
+    },
+    {
+      "id": 5,
+      "name": "phase3-gate",
+      "prompt": "Ok I think I know what I want. Let's turn this into a plan.",
+      "expected_output": "Vision summary for approval, NOT the full plan yet",
+      "files": [],
+      "multi_turn_context": "Turn 6+ of brainstorming. Previous turns explored study-buddy app, settled on React Native + Firebase, matching algorithm, cold-start solution. User now wants to move to planning.",
+      "assertions": [
+        {"name": "vision_summary", "description": "Presents a brief 'here's what we've landed on' summary before generating the full plan"},
+        {"name": "approval_gate", "description": "Asks for explicit confirmation before proceeding to full plan generation"},
+        {"name": "does_not_dump_full_plan", "description": "Does NOT generate the entire task breakdown, agent assignments, and wave structure in this message"},
+        {"name": "references_plan_template", "description": "Reads or references references/plan-template.md for plan generation"}
+      ]
+    }
+  ]
+}

package/ftm-brainstorm/evals/promptfoo.yaml

@@ -1,109 +1,109 @@
-description: "ftm-brainstorm behavior across multi-turn brainstorming sessions"
-
-prompts:
-  - "{{input}}"
-
-providers:
-  - id: "exec:claude --print"
-    label: "claude-code"
-
-defaultTest:
-  options:
-    transformVars: "vars"
-
-tests:
-  # Eval 0: fresh-idea-intake
-  - description: "fresh idea intake — asks 1-2 questions, no research sprint, hard stop"
-    vars:
-      input: "I'm thinking about building an app that helps people find study buddies at their university. Like Tinder but for studying."
-    assert:
-      - type: regex
-        value: "\\?"
-        description: "Response must contain at least one question"
-      - type: not-contains
-        value: "sprint"
-        description: "Does not dispatch research sprint on first turn"
-      - type: not-contains
-        value: "dispatch"
-        description: "Does not dispatch agents on first turn"
-
-  # Eval 1: fresh-idea-turn2-research
-  - description: "turn 2 — dispatches research agents, citations, labeled recommendation, challenge, ends with question"
-    vars:
-      input: "It's for college students who want to find people in the same classes to study with. Matching based on courses, study style, and schedule availability."
-    assert:
-      - type: regex
-        value: "https?://"
-        description: "Response includes at least one URL citation"
-      - type: regex
-        value: "RECOMMENDED|recommended|#1"
-        description: "At least one suggestion is labeled as recommended"
-      - type: regex
-        value: "\\?"
-        description: "Ends with a question"
-
-  # Eval 2: turn3-deeper-research
-  - description: "turn 3 — deeper research on specific concerns, builds on prior, fresh citations, challenge"
-    vars:
-      input: "I like option 2 — the React Native approach with Firebase. But I'm worried about the matching algorithm complexity. Also we need to handle the cold-start problem when a new university joins."
-    assert:
-      - type: regex
-        value: "matching|algorithm|cold.start"
-        description: "Response addresses the specific concerns raised"
-      - type: regex
-        value: "https?://"
-        description: "Response includes at least one URL citation"
-      - type: regex
-        value: "\\?"
-        description: "Ends with a question"
-
-  # Eval 3: brain-dump-intake
-  - description: "brain dump intake — extracts structure, confirmation gate, no research yet"
-    vars:
-      input: "help me build all the suggestions in this chat: [brain dump about eng-buddy]"
-    assert:
-      - type: regex
-        value: "Decided|decided|Decision|decision"
-        description: "Contains a 'Decided' section"
-      - type: regex
-        value: "open question|Open question|gap|Gap"
-        description: "Contains an open questions or gaps section"
-      - type: regex
-        value: "\\?"
-        description: "Ends with a confirmation question"
-      - type: not-contains
-        value: "dispatch"
-        description: "Does not dispatch research agents on intake turn"
-
-  # Eval 4: brain-dump-turn2-research
-  - description: "brain dump turn 2 — novelty map, research citations, challenge, question"
-    vars:
-      input: "Yeah that looks right, go ahead and research it"
-    assert:
-      - type: regex
-        value: "Novelty|novelty|novel|Novel"
-        description: "Contains a Novelty Map"
-      - type: regex
-        value: "https?://"
-        description: "Response includes at least one URL citation"
-      - type: regex
-        value: "\\?"
-        description: "Ends with a question"
-
-  # Eval 5: phase3-gate
-  - description: "phase 3 gate — vision summary and approval before generating full plan"
-    vars:
-      input: "Ok I think I know what I want. Let's turn this into a plan."
-    assert:
-      - type: regex
-        value: "summary|Summary|we've landed|landed on|here's what"
-        description: "Presents a vision summary before generating the full plan"
-      - type: regex
-        value: "confirm|Confirm|proceed|Proceed|ready|Ready|go ahead|approve"
-        description: "Asks for confirmation before proceeding to full plan"
-      - type: not-contains
-        value: "Wave 1"
-        description: "Does not dump the full plan structure prematurely"
-      - type: not-contains
-        value: "Wave 2"
-        description: "Does not dump the full plan structure prematurely"
+description: "ftm-brainstorm behavior across multi-turn brainstorming sessions"
+
+prompts:
+  - "{{input}}"
+
+providers:
+  - id: "exec:claude --print"
+    label: "claude-code"
+
+defaultTest:
+  options:
+    transformVars: "vars"
+
+tests:
+  # Eval 0: fresh-idea-intake
+  - description: "fresh idea intake — asks 1-2 questions, no research sprint, hard stop"
+    vars:
+      input: "I'm thinking about building an app that helps people find study buddies at their university. Like Tinder but for studying."
+    assert:
+      - type: regex
+        value: "\\?"
+        description: "Response must contain at least one question"
+      - type: not-contains
+        value: "sprint"
+        description: "Does not dispatch research sprint on first turn"
+      - type: not-contains
+        value: "dispatch"
+        description: "Does not dispatch agents on first turn"
+
+  # Eval 1: fresh-idea-turn2-research
+  - description: "turn 2 — dispatches research agents, citations, labeled recommendation, challenge, ends with question"
+    vars:
+      input: "It's for college students who want to find people in the same classes to study with. Matching based on courses, study style, and schedule availability."
+    assert:
+      - type: regex
+        value: "https?://"
+        description: "Response includes at least one URL citation"
+      - type: regex
+        value: "RECOMMENDED|recommended|#1"
+        description: "At least one suggestion is labeled as recommended"
+      - type: regex
+        value: "\\?"
+        description: "Ends with a question"
+
+  # Eval 2: turn3-deeper-research
+  - description: "turn 3 — deeper research on specific concerns, builds on prior, fresh citations, challenge"
+    vars:
+      input: "I like option 2 — the React Native approach with Firebase. But I'm worried about the matching algorithm complexity. Also we need to handle the cold-start problem when a new university joins."
+    assert:
+      - type: regex
+        value: "matching|algorithm|cold.start"
+        description: "Response addresses the specific concerns raised"
+      - type: regex
+        value: "https?://"
+        description: "Response includes at least one URL citation"
+      - type: regex
+        value: "\\?"
+        description: "Ends with a question"
+
+  # Eval 3: brain-dump-intake
+  - description: "brain dump intake — extracts structure, confirmation gate, no research yet"
+    vars:
+      input: "help me build all the suggestions in this chat: [brain dump about eng-buddy]"
+    assert:
+      - type: regex
+        value: "Decided|decided|Decision|decision"
+        description: "Contains a 'Decided' section"
+      - type: regex
+        value: "open question|Open question|gap|Gap"
+        description: "Contains an open questions or gaps section"
+      - type: regex
+        value: "\\?"
+        description: "Ends with a confirmation question"
+      - type: not-contains
+        value: "dispatch"
+        description: "Does not dispatch research agents on intake turn"
+
+  # Eval 4: brain-dump-turn2-research
+  - description: "brain dump turn 2 — novelty map, research citations, challenge, question"
+    vars:
+      input: "Yeah that looks right, go ahead and research it"
+    assert:
+      - type: regex
+        value: "Novelty|novelty|novel|Novel"
+        description: "Contains a Novelty Map"
+      - type: regex
+        value: "https?://"
+        description: "Response includes at least one URL citation"
+      - type: regex
+        value: "\\?"
+        description: "Ends with a question"
+
+  # Eval 5: phase3-gate
+  - description: "phase 3 gate — vision summary and approval before generating full plan"
+    vars:
+      input: "Ok I think I know what I want. Let's turn this into a plan."
+    assert:
+      - type: regex
+        value: "summary|Summary|we've landed|landed on|here's what"
+        description: "Presents a vision summary before generating the full plan"
+      - type: regex
+        value: "confirm|Confirm|proceed|Proceed|ready|Ready|go ahead|approve"
+        description: "Asks for confirmation before proceeding to full plan"
+      - type: not-contains
+        value: "Wave 1"
+        description: "Does not dump the full plan structure prematurely"
+      - type: not-contains
+        value: "Wave 2"
+        description: "Does not dump the full plan structure prematurely"