claude-dev-env 1.16.0 → 1.16.1

package/hooks/HOOK_SPECS_PROMPT_WORKFLOW.md

@@ -54,3 +54,9 @@ These two signals are runtime-checked by the Stop guard whenever a prompt-workfl
 ## Deterministic Boundary
 
 These hooks enforce only structural/runtime checks. Semantic quality remains in auditor layer.
+
+## Reviewing Flattened Transcript Exports
+
+- Live prompt-workflow responses still require an explicit `Audit:` line plus one outer `xml` fence. The Stop guard and clipboard path continue to evaluate that literal boundary.
+- Saved transcript exports can flatten blocked retry turns and omit the outer fence lines. Normalize those files with `prompt_workflow_gate_core.normalize_prompt_workflow_export(...)`, then evaluate the rebuilt message with `extract_fenced_xml_content(...)` or `extract_fenced_xml_content_from_export(...)`.
+- Fence-relative evals review the **last successful Audit + artifact pair** after normalization. Earlier blocked retries in the flattened transcript remain diagnostic evidence and do not count as extra delivered artifacts.

package/hooks/blocking/prompt_workflow_gate_core.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 import re
+import textwrap
 from typing import Iterable
 
 from prompt_workflow_gate_config import (
@@ -18,25 +19,57 @@ from prompt_workflow_gate_config import (
     REQUIRED_XML_SECTIONS,
 )
 
+TRIPLE_BACKTICK = "```"
+AUDIT_LINE_PATTERN = re.compile(r"^\s*[●•]?\s*(Audit:\s*.+?)\s*$")
 
 def _line_opens_xml_fence(line: str) -> bool:
     stripped = line.strip()
-    if not stripped.startswith("```"):
+    if not stripped.startswith(TRIPLE_BACKTICK):
         return False
-    remainder = stripped[3:].strip()
+    fence_marker_length = len(TRIPLE_BACKTICK)
+    remainder = stripped[fence_marker_length:].strip()
     return remainder == "xml" or remainder.startswith("xml ")
 
-
 def _line_is_bare_fence_close(line: str) -> bool:
-    return line.strip() == "```"
-
+    return line.strip() == TRIPLE_BACKTICK
 
 def _line_opens_inner_markdown_fence(line: str) -> bool:
     stripped = line.strip()
-    if not stripped.startswith("```"):
+    if not stripped.startswith(TRIPLE_BACKTICK):
         return False
-    return stripped != "```"
-
+    return stripped != TRIPLE_BACKTICK
+
+def _collect_inner_markdown_fence(
+    lines: list[str],
+    start_index: int,
+) -> tuple[list[str], int]:
+    inner_lines: list[str] = []
+    index = start_index
+    while index < len(lines):
+        current_line = lines[index]
+        inner_lines.append(current_line)
+        index += 1
+        if _line_is_bare_fence_close(current_line):
+            break
+    return inner_lines, index
+
+def _collect_xml_fence_body(
+    lines: list[str],
+    start_index: int,
+) -> tuple[list[str], int]:
+    body_lines: list[str] = []
+    index = start_index
+    while index < len(lines):
+        current_line = lines[index]
+        if _line_is_bare_fence_close(current_line):
+            return body_lines, index + 1
+        if _line_opens_inner_markdown_fence(current_line):
+            inner_lines, index = _collect_inner_markdown_fence(lines, index)
+            body_lines.extend(inner_lines)
+            continue
+        body_lines.append(current_line)
+        index += 1
+    return body_lines, index
 
 def extract_fenced_xml_content(text: str) -> str:
     """Extract bodies of ```xml fenced blocks.
@@ -50,31 +83,104 @@ def extract_fenced_xml_content(text: str) -> str:
     lines = text.splitlines()
     index = 0
     while index < len(lines):
-        if _line_opens_xml_fence(lines[index]):
+        if not _line_opens_xml_fence(lines[index]):
             index += 1
-            body_lines: list[str] = []
-            while index < len(lines):
-                line = lines[index]
-                if _line_is_bare_fence_close(line):
-                    index += 1
-                    break
-                if _line_opens_inner_markdown_fence(line):
-                    body_lines.append(line)
-                    index += 1
-                    while index < len(lines):
-                        inner_line = lines[index]
-                        body_lines.append(inner_line)
-                        index += 1
-                        if _line_is_bare_fence_close(inner_line):
-                            break
-                    continue
-                body_lines.append(line)
-                index += 1
-            results.append("\n".join(body_lines))
             continue
-        index += 1
+        body_lines, index = _collect_xml_fence_body(lines, index + 1)
+        results.append("\n".join(body_lines))
     return "\n".join(results)
 
+def _line_is_audit_line(line: str) -> bool:
+    return AUDIT_LINE_PATTERN.match(line) is not None
+
+def _normalize_audit_line(line: str) -> str:
+    match = AUDIT_LINE_PATTERN.match(line)
+    if match:
+        return match.group(1).strip()
+    return line.strip()
+
+def _line_starts_exported_artifact(line: str) -> bool:
+    stripped = line.strip()
+    if not stripped:
+        return False
+    if _line_opens_xml_fence(stripped):
+        return True
+    exported_artifact_pattern = re.compile(
+        r"^<(\?xml\b|prompt\b|runtime_context\b|role\b|background\b|instructions\b|constraints\b|output_format\b|illustrations\b|open_question\b)",
+    )
+    return exported_artifact_pattern.match(stripped) is not None
+
+def _trim_trailing_blank_lines(lines: list[str]) -> list[str]:
+    trimmed = list(lines)
+    while trimmed and not trimmed[-1].strip():
+        trimmed.pop()
+    return trimmed
+
+def _trim_flattened_export_tail(lines: list[str]) -> list[str]:
+    trimmed = _trim_trailing_blank_lines(lines)
+    while trimmed and trimmed[-1].lstrip().startswith("✻ "):
+        trimmed.pop()
+        trimmed = _trim_trailing_blank_lines(trimmed)
+    return trimmed
+
+def _find_last_audit_index(lines: list[str]) -> int | None:
+    last_audit_index: int | None = None
+    for index, line in enumerate(lines):
+        if _line_is_audit_line(line):
+            last_audit_index = index
+    return last_audit_index
+
+def _find_first_artifact_index(lines: list[str]) -> int | None:
+    for index, line in enumerate(lines):
+        if _line_starts_exported_artifact(line):
+            return index
+    return None
+
+def _rebuild_from_existing_fence(audit_line: str, artifact_text: str) -> str:
+    fenced_body = extract_fenced_xml_content(artifact_text).strip()
+    if not fenced_body:
+        return audit_line
+    return f"{audit_line}\n```xml\n{fenced_body}\n```"
+
+def _rebuild_from_flattened_body(audit_line: str, artifact_text: str) -> str:
+    dedented_body = textwrap.dedent(artifact_text).strip("\n")
+    if not dedented_body:
+        return audit_line
+    return f"{audit_line}\n```xml\n{dedented_body}\n```"
+
+def _rebuild_canonical_export(audit_line: str, artifact_lines: list[str]) -> str:
+    if not artifact_lines:
+        return audit_line
+    artifact_text = "\n".join(artifact_lines).rstrip()
+    if _line_opens_xml_fence(artifact_lines[0]):
+        return _rebuild_from_existing_fence(audit_line, artifact_text)
+    return _rebuild_from_flattened_body(audit_line, artifact_text)
+
+def normalize_prompt_workflow_export(text: str) -> str:
+    """Return the last successful Audit + fenced XML pair from a message or export.
+
+    Saved transcript exports can flatten blocked retry turns and strip the outer
+    ``xml`` fence. This helper keeps only the last successful ``Audit:`` attempt
+    and rebuilds the canonical audit-plus-fence shape used by prompt-workflow
+    hooks and reviewers.
+    """
+    lines = text.splitlines()
+    last_audit_index = _find_last_audit_index(lines)
+    if last_audit_index is None:
+        return text.strip()
+    audit_line = _normalize_audit_line(lines[last_audit_index])
+    artifact_index = _find_first_artifact_index(lines[last_audit_index + 1 :])
+    if artifact_index is None:
+        return audit_line
+    artifact_lines = _trim_flattened_export_tail(
+        lines[last_audit_index + 1 + artifact_index :],
+    )
+    return _rebuild_canonical_export(audit_line, artifact_lines)
+
+def extract_fenced_xml_content_from_export(text: str) -> str:
+    """Extract fenced XML from a canonical message or flattened transcript export."""
+    normalized = normalize_prompt_workflow_export(text)
+    return extract_fenced_xml_content(normalized)
 
 def missing_required_xml_sections(text: str) -> list[str]:
     fenced_body = extract_fenced_xml_content(text)
@@ -88,6 +194,30 @@ def missing_required_xml_sections(text: str) -> list[str]:
             missing_sections.append(section_name)
     return missing_sections
 
+def _build_negative_keyword_violation(
+    match: re.Match[str],
+    line_number: int,
+    line_text: str,
+) -> dict[str, str | int]:
+    return {
+        "keyword": match.group(),
+        "line_number": line_number,
+        "line_text": line_text.strip(),
+    }
+
+def _find_pattern_violations(
+    patterns: Iterable[re.Pattern[str]],
+    line_text: str,
+    line_number: int,
+) -> list[dict[str, str | int]]:
+    violations: list[dict[str, str | int]] = []
+    for pattern in patterns:
+        match = pattern.search(line_text)
+        if match:
+            violations.append(
+                _build_negative_keyword_violation(match, line_number, line_text),
+            )
+    return violations
 
 def find_negative_keywords_in_fenced_xml(
     text: str,
@@ -95,45 +225,37 @@ def find_negative_keywords_in_fenced_xml(
     fenced_content = extract_fenced_xml_content(text)
     if not fenced_content:
         return []
-    fenced_lines = fenced_content.splitlines()
     all_violations: list[dict[str, str | int]] = []
-    for line_index, each_line in enumerate(fenced_lines):
-        for each_pattern in COMPILED_NEGATIVE_KEYWORD_PATTERNS:
-            each_match = each_pattern.search(each_line)
-            if each_match:
-                all_violations.append({
-                    "keyword": each_match.group(),
-                    "line_number": line_index + 1,
-                    "line_text": each_line.strip(),
-                })
-        for each_pattern in COMPILED_NEGATIVE_INDIRECT_PATTERNS:
-            each_match = each_pattern.search(each_line)
-            if each_match:
-                all_violations.append({
-                    "keyword": each_match.group(),
-                    "line_number": line_index + 1,
-                    "line_text": each_line.strip(),
-                })
+    for line_index, each_line in enumerate(fenced_content.splitlines(), start=1):
+        all_violations.extend(
+            _find_pattern_violations(
+                COMPILED_NEGATIVE_KEYWORD_PATTERNS,
+                each_line,
+                line_index,
+            ),
+        )
+        all_violations.extend(
+            _find_pattern_violations(
+                COMPILED_NEGATIVE_INDIRECT_PATTERNS,
+                each_line,
+                line_index,
+            ),
+        )
     return all_violations
 
-
 def _contains_any_marker(text: str, markers: Iterable[str]) -> bool:
     lower_text = text.lower()
     return any(marker.lower() in lower_text for marker in markers)
 
-
 def has_debug_intent(text: str) -> bool:
     return _contains_any_marker(text, DEBUG_INTENT_MARKERS)
 
-
 def has_internal_object_leak(text: str) -> bool:
     return _contains_any_marker(text, INTERNAL_OBJECT_MARKERS)
 
-
 def missing_scope_anchors(text: str) -> list[str]:
     return [anchor for anchor in REQUIRED_SCOPE_ANCHORS if anchor not in text]
 
-
 def find_ambiguous_scope_terms(text: str) -> list[str]:
     if "scope" not in text.lower():
         return []
@@ -144,16 +266,13 @@ def find_ambiguous_scope_terms(text: str) -> list[str]:
             matches.append(term)
     return matches
 
-
 def has_checklist_container(text: str) -> bool:
     lower_text = text.lower()
     return "checklist_results" in lower_text or "checklist:" in lower_text
 
-
 def missing_checklist_rows(text: str) -> list[str]:
     return [row for row in REQUIRED_CHECKLIST_ROWS if row not in text]
 
-
 def is_prompt_workflow_response(text: str) -> bool:
     lower_text = text.lower()
     matched_markers = [
@@ -161,7 +280,6 @@ def is_prompt_workflow_response(text: str) -> bool:
     ]
     return len(matched_markers) >= 2
 
-
 def missing_context_control_signals(text: str) -> list[str]:
     required_signals: tuple[str, ...] = (
         "base_minimal_instruction_layer: true",

package/hooks/blocking/test_prompt_workflow_gate_core.py

@@ -2,6 +2,7 @@
 
 from prompt_workflow_gate_core import (
     extract_fenced_xml_content,
+    extract_fenced_xml_content_from_export,
     find_ambiguous_scope_terms,
     has_checklist_container,
     has_internal_object_leak,
@@ -10,31 +11,27 @@ from prompt_workflow_gate_core import (
     missing_checklist_rows,
     missing_required_xml_sections,
     missing_scope_anchors,
+    normalize_prompt_workflow_export,
 )
 
-
 def test_internal_object_leak_detected() -> None:
     text = '{"pipeline_mode": "internal_section_refinement_with_final_audit"}'
     assert has_internal_object_leak(text)
 
-
 def test_missing_scope_anchors_returns_expected_rows() -> None:
     text = "target_local_roots only."
     missing = missing_scope_anchors(text)
     assert "target_canonical_roots" in missing
     assert "completion_boundary" in missing
 
-
 def test_missing_checklist_rows_detected() -> None:
     text = "checklist_results: structured_scoped_instructions only"
     missing = missing_checklist_rows(text)
     assert "completion_boundary_measurable" in missing
 
-
 def test_checklist_container_detection() -> None:
     assert has_checklist_container("checklist_results:\n- structured_scoped_instructions")
 
-
 def test_prompt_workflow_response_detection() -> None:
     message = (
         "overall_status: pass\n"
@@ -43,22 +40,36 @@ def test_prompt_workflow_response_detection() -> None:
     )
     assert is_prompt_workflow_response(message)
 
-
 def test_missing_context_control_signals_detected() -> None:
     missing = missing_context_control_signals("base_minimal_instruction_layer: true")
     assert "on_demand_skill_loading: true" in missing
 
-
 def test_ambiguous_scope_terms_detected() -> None:
     text = "Scope applies to this session and current files."
     terms = find_ambiguous_scope_terms(text)
     assert "this session" in terms
     assert "current files" in terms
 
-
 def _fenced_xml(body: str) -> str:
     return f"```xml\n{body}\n```"
 
+def _runtime_context_lines() -> tuple[str, ...]:
+    return (
+        "<runtime_context>",
+        "base_minimal_instruction_layer: true",
+        "on_demand_skill_loading: true",
+        "</runtime_context>",
+        "",
+    )
+
+def _flattened_transcript(*lines: str) -> str:
+    return "\n".join(lines) + "\n"
+
+def _flattened_attempt(*body_lines: str, audit_line: str = "Audit: pass 15/15") -> str:
+    flattened_lines = [audit_line, ""]
+    for line in body_lines:
+        flattened_lines.append(f"  {line}" if line else "")
+    return "\n".join(flattened_lines)
 
 def test_missing_required_xml_sections_all_present_returns_empty() -> None:
     body = (
@@ -70,7 +81,6 @@ def test_missing_required_xml_sections_all_present_returns_empty() -> None:
     )
     assert missing_required_xml_sections(_fenced_xml(body)) == []
 
-
 def test_missing_required_xml_sections_missing_background() -> None:
     body = (
         "<role>R.</role>\n"
@@ -80,7 +90,6 @@ def test_missing_required_xml_sections_missing_background() -> None:
     )
     assert missing_required_xml_sections(_fenced_xml(body)) == ["background"]
 
-
 def test_missing_required_xml_sections_missing_role_and_output_format() -> None:
     body = (
         "<background>C.</background>\n"
@@ -90,11 +99,9 @@ def test_missing_required_xml_sections_missing_role_and_output_format() -> None:
     missing = missing_required_xml_sections(_fenced_xml(body))
     assert missing == ["role", "output_format"]
 
-
 def test_missing_required_xml_sections_no_fence_returns_empty() -> None:
     assert missing_required_xml_sections("no fenced xml here") == []
 
-
 def test_missing_required_xml_sections_prose_without_tags_counts_as_missing() -> None:
     body = (
         "<role>R.</role>\n"
@@ -105,7 +112,6 @@ def test_missing_required_xml_sections_prose_without_tags_counts_as_missing() ->
     )
     assert missing_required_xml_sections(_fenced_xml(body)) == ["background"]
 
-
 def test_extract_fenced_xml_preserves_content_after_nested_inner_fence() -> None:
     message = (
         "```xml\n"
@@ -122,3 +128,68 @@ def test_extract_fenced_xml_preserves_content_after_nested_inner_fence() -> None
     extracted = extract_fenced_xml_content(message)
     assert "</illustrations>" in extracted
     assert "<background>B</background>" in extracted
+
+def test_normalize_prompt_workflow_export_rebuilds_fence_from_flattened_transcript() -> None:
+    transcript = _flattened_transcript(
+        _flattened_attempt(
+            *_runtime_context_lines(),
+            "<role>R</role>",
+            "<background>B</background>",
+            "<instructions>I</instructions>",
+            "<constraints>C</constraints>",
+            "<output_format>O</output_format>",
+            "✻ Worked for 1m 7s",
+            audit_line="● Audit: pass 15/15",
+        ),
+    )
+    normalized = normalize_prompt_workflow_export(transcript)
+    assert normalized.startswith("Audit: pass 15/15\n```xml\n")
+    assert normalized.endswith("\n```")
+    assert "<runtime_context>" in normalized
+    assert "✻ Worked for 1m 7s" not in normalized
+
+def test_normalize_prompt_workflow_export_uses_last_audit_attempt() -> None:
+    first_attempt = _flattened_attempt(
+        "<role>FIRST</role>",
+        "<background>Old</background>",
+        "<instructions>Old</instructions>",
+        "<constraints>Old</constraints>",
+        "<output_format>Old</output_format>",
+        audit_line="● Audit: pass 15/15",
+    )
+    second_attempt = _flattened_attempt(
+        *_runtime_context_lines(),
+        "<role>FINAL</role>",
+        "<background>Fresh</background>",
+        "<instructions>I</instructions>",
+        "<constraints>C</constraints>",
+        "<output_format>O</output_format>",
+        "✻ Worked for 2m 8s",
+    )
+    transcript = _flattened_transcript(
+        first_attempt,
+        "",
+        "● Re-emitting the full artifact with the runtime signals added.",
+        "",
+        second_attempt,
+    )
+    normalized = normalize_prompt_workflow_export(transcript)
+    assert "<role>FINAL</role>" in normalized
+    assert "<role>FIRST</role>" not in normalized
+
+def test_extract_fenced_xml_content_from_export_supports_flattened_transcript() -> None:
+    transcript = _flattened_transcript(
+        _flattened_attempt(
+            "<role>R</role>",
+            "<background>B</background>",
+            "<instructions>I</instructions>",
+            "<constraints>C</constraints>",
+            "<output_format>O</output_format>",
+            "✻ Worked for 31s",
+            audit_line="● Audit: pass 15/15",
+        ),
+    )
+    extracted = extract_fenced_xml_content_from_export(transcript)
+    assert extracted.startswith("<role>R</role>")
+    assert "<output_format>O</output_format>" in extracted
+    assert "Worked for" not in extracted

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "claude-dev-env",
-    "version": "1.16.0",
+    "version": "1.16.1",
     "description": "Claude Code development standards — rules, hooks, agents, commands, and skills",
     "type": "module",
     "bin": {

package/skills/prompt-generator/evals/prompt-generator.json

@@ -72,10 +72,10 @@
       "prompt": "/prompt-generator Create a prompt for an agent that traces a routing bug across shared_utils/export_handler.py, orchestrator.py, and download_manager.py — find where extract_apk is called and whether it handles APK signature check failures",
       "files": ["packages/samsung-automation/shared_utils/export_handler.py"],
       "expected_behavior": [
-        "No tool_use blocks appear after the first fence marker of the prompt artifact",
+        "No tool_use blocks appear after the first fence marker of the canonical prompt artifact",
         "All Glob/Grep discovery calls precede the AskUserQuestion",
         "All AskUserQuestion interactions precede the fenced block",
-        "Prompt artifact emits in a single uninterrupted response"
+        "Review the last successful Audit + fenced xml pair; blocked retry attempts preserved by flattened transcript exports do not count as additional delivered artifacts"
       ]
     },
     {
@@ -85,7 +85,7 @@
       "prompt": "/prompt-generator Write a detailed agent-harness prompt for a TDD bug-fix workflow that traces a routing error across 5+ files, with state management for multi-window execution and structured test tracking",
       "files": [],
       "expected_behavior": [
-        "Opening fence has a matching closing fence",
+        "The canonical prompt artifact has one opening xml fence and one matching closing fence; flattened transcript exports are normalized to that same boundary before review",
         "Every XML tag properly opened and closed",
         "No truncation at numbered-list bullets (the Issue #41 failure mode)",
         "No mid-sentence cuts or incomplete sections",
@@ -102,8 +102,8 @@
         "Discovery tool calls attempt to locate scoring logic before prompt generation",
         "If resolved: prompt references concrete file paths from discovery",
         "If unresolved: prompt contains <open_question> in <background> for downstream agent",
-        "No re-entry to discovery after fenced block starts",
-        "AskUserQuestion may surface the uncertainty if discovery was inconclusive"
+        "No re-entry to discovery after the canonical artifact fence starts",
+        "AskUserQuestion may surface the uncertainty if discovery was inconclusive; when discovery resolves concrete paths before the artifact, absence of <open_question> is expected"
       ]
     },
     {

package/skills/skill-builder/SKILL.md

@@ -0,0 +1,87 @@
+---
+name: skill-builder
+description: >-
+  Orchestrates the complete skill-building lifecycle using evaluation-driven
+  development. Routes through gap analysis, eval creation, skill writing (via
+  skill-writer), subagent testing (via skill-creator infrastructure), and
+  iterative refinement. Use when creating new skills, improving existing skills,
+  or optimizing skill descriptions. Triggers: 'build a skill', 'new skill
+  workflow', 'improve this skill', 'optimize skill description', 'skill
+  development lifecycle'.
+---
+
+@${CLAUDE_SKILL_DIR}/references/eval-driven-flow.md
+
+# Skill Builder
+
+**Core principle:** Evaluation-driven development. Build evals BEFORE writing extensive documentation. This ensures skills solve real problems rather than documenting imagined ones.
+
+Source: [Anthropic Skill Best Practices - Evaluation and Iteration](https://platform.claude.com/docs/en/agents-and-tools/agent-skills/best-practices#evaluation-and-iteration)
+
+## When this skill applies
+
+Trigger for requests to **build**, **improve**, or **polish** a skill through the full evaluation-driven lifecycle. This skill orchestrates the process -- it delegates writing to `/skill-writer` and evaluation infrastructure to the `skill-creator` plugin.
+
+For quick skill syntax questions or one-off SKILL.md edits, use `/skill-writer` directly instead.
+
+## Routing
+
+Assess the user's intent from conversation context and existing artifacts. Route directly:
+
+**Creating a new skill?**
+Read `${CLAUDE_SKILL_DIR}/workflows/new-skill.md` and follow it.
+
+**Improving an existing skill?**
+Read `${CLAUDE_SKILL_DIR}/workflows/improve-skill.md` and follow it.
+
+**Final polish only (description optimization, trigger eval)?**
+Read `${CLAUDE_SKILL_DIR}/workflows/polish-skill.md` and follow it.
+
+**Ambiguous?** Ask: "Are you creating a new skill, improving an existing one, or doing a final polish pass?"
+
+## The Claude A / Claude B Pattern
+
+You and the user are **Claude A** -- the expert who designs and refines the skill. Subagents running the built skill on eval tasks are **Claude B** -- the agent using the skill to perform real work.
+
+> "Work with one instance of Claude ('Claude A') to create a Skill that is used by other instances ('Claude B'). Claude A helps you design and refine instructions, while Claude B tests them in real tasks."
+
+The feedback loop: observe Claude B's behavior, bring insights back, refine the skill, test again.
+
+## Phase Overview
+
+| Phase | Purpose | Delegated To |
+|-------|---------|-------------|
+| 1. Identify gaps | Document what fails without the skill | This skill (guided conversation) |
+| 2. Build evals | Create 3+ scenarios testing the gaps | This skill (templates + user input) |
+| 3. Write skill | Minimal instructions addressing gaps | `/skill-writer` |
+| 4. Test | Subagent runs with/without skill, grade, benchmark | `skill-creator` eval infrastructure |
+| 5. Iterate | Review results, refine, re-test | This skill + `/skill-writer` + Phase 4 |
+| 6. Polish | Description optimization, trigger eval, final check | `skill-creator` description optimizer |
+
+## Principles (apply across all phases)
+
+1. **Evals before documentation.** Never write extensive skill content without evaluation scenarios to validate it.
+
+2. **Minimal instructions first.** Write just enough to pass evaluations. Resist the urge to over-document.
+
+3. **Generalize from feedback.** The skill will be used across many prompts. Do not overfit to test cases.
+
+4. **Explain the why.** Theory of mind beats rigid rules. Help the model understand reasoning, not just constraints.
+
+5. **Observe, do not assume.** Iterate based on what Claude B actually does, not what you think it should do.
+
+## Delegation Details
+
+See `${CLAUDE_SKILL_DIR}/references/delegation-map.md` for exact invocation patterns and integration points between this orchestrator, `/skill-writer`, and `skill-creator`.
+
+## File Index
+
+| File | Purpose |
+|------|---------|
+| `workflows/new-skill.md` | Full lifecycle for new skills (6 phases) |
+| `workflows/improve-skill.md` | Observation-first flow for existing skills |
+| `workflows/polish-skill.md` | Description optimization and final validation |
+| `references/eval-driven-flow.md` | Official Anthropic methodology with citations |
+| `references/delegation-map.md` | Integration map for skill-writer and skill-creator |
+| `templates/gap-analysis.md` | Template for Phase 1 gap documentation |
+| `templates/eval-scenario.json` | Eval template matching skill-creator schema |

package/skills/skill-builder/references/delegation-map.md

@@ -0,0 +1,151 @@
+# Delegation Map
+
+How the skill-builder orchestrator integrates with external skills at each phase.
+
+## Phase 1: Identify Gaps -- This Orchestrator
+
+No external delegation. The orchestrator guides a conversation with the user to document what fails without a skill.
+
+**Output:** `[skill-name]-workspace/gap-analysis.md` using the template at `templates/gap-analysis.md`.
+
+## Phase 2: Build Evals -- This Orchestrator
+
+No external delegation. The orchestrator helps the user transform gaps into eval scenarios.
+
+**Output:** `[skill-name]-workspace/evals/evals.json` using the template at `templates/eval-scenario.json`.
+
+**Baseline runs:** Spawn subagents WITHOUT any skill for each eval scenario. These run as background Agent tasks.
+
+## Phase 3: Write Skill -- Delegate to `/skill-writer`
+
+Invoke `/skill-writer` with the following context in your prompt:
+
+```
+Create a skill based on this gap analysis and eval scenarios.
+
+Gap analysis: [paste or reference gap-analysis.md]
+Eval scenarios: [paste or reference evals.json expected_output fields]
+Baseline failures: [summarize what Claude got wrong without the skill]
+
+Constraint: Write the minimum instructions needed to address these specific gaps.
+Do not over-document. Every line must serve a documented gap.
+```
+
+skill-writer handles: type classification, degree of freedom, frontmatter, body structure, progressive disclosure, self-check.
+
+**Output:** The skill's SKILL.md (and optional REFERENCE.md, scripts, etc.)
+
+## Phase 4: Test -- Delegate to skill-creator Infrastructure
+
+The skill-creator plugin provides the eval infrastructure. Reference its components directly:
+
+### Spawning test runs
+
+For each eval, spawn TWO subagents in the SAME turn (parallel):
+
+**With-skill subagent:**
+```
+Execute this task:
+- Read the skill at [path-to-skill]/SKILL.md and follow its instructions
+- Task: [eval prompt from evals.json]
+- Input files: [eval files if any]
+- Save all output files to: [workspace]/iteration-N/eval-[name]/with_skill/outputs/
+- Save a transcript of your complete work to: [workspace]/iteration-N/eval-[name]/with_skill/transcript.md
+- At the end, write a metrics.json with tool call counts and file list
+```
+
+**Without-skill subagent** (baseline):
+For iteration-1, reuse baseline results from Phase 2 (iteration-0). For later iterations, the original baseline persists.
+
+### Grading
+
+Read the grading agent instructions from the skill-creator plugin:
+`[skill-creator-plugin-path]/agents/grader.md`
+
+Spawn a grader subagent for each run with:
+- The expectations from evals.json
+- The transcript path
+- The outputs directory
+
+**Output:** `grading.json` in each run directory.
+
+### Benchmarking
+
+Run the aggregation script from the skill-creator plugin directory:
+```bash
+cd [skill-creator-plugin-path] && python -m scripts.aggregate_benchmark [workspace]/iteration-N --skill-name [name]
+```
+
+**Output:** `benchmark.json` and `benchmark.md` in the iteration directory.
+
+### Eval Viewer
+
+Launch the viewer from the skill-creator plugin:
+```bash
+python [skill-creator-plugin-path]/eval-viewer/generate_review.py \
+  [workspace]/iteration-N \
+  --skill-name "[name]" \
+  --benchmark [workspace]/iteration-N/benchmark.json
+```
+
+For iteration 2+, add: `--previous-workspace [workspace]/iteration-[N-1]`
+
+If no browser/display available, add: `--static [workspace]/iteration-N/review.html`
+
+**Output:** Browser-based reviewer where the user inspects outputs and leaves feedback.
+
+### Finding the skill-creator plugin path
+
+The skill-creator plugin is installed at a path like:
+`~/.claude/plugins/marketplaces/claude-plugins-official/plugins/skill-creator/skills/skill-creator/`
+
+To find it dynamically, search for the skill-creator SKILL.md:
+```bash
+find ~/.claude/plugins -name "SKILL.md" -path "*/skill-creator/*" 2>/dev/null | head -1
+```
+
+Then derive the plugin root from that path.
+
+## Phase 5: Iterate -- This Orchestrator + `/skill-writer`
+
+The orchestrator reads feedback.json and transcripts, synthesizes observations, then delegates refinement to `/skill-writer`:
+
+```
+Refine this existing skill based on these observations from testing.
+
+Current SKILL.md: [paste or reference]
+User feedback: [from feedback.json]
+Behavioral observations: [from transcript analysis]
+
+Specific issues to address:
+1. [Issue from feedback]
+2. [Issue from observation]
+
+Constraint: Only change what the feedback demands. Do not reorganize working content.
+```
+
+Then return to Phase 4 with the refined skill.
+
+## Phase 6: Polish -- Delegate to skill-creator Description Optimizer
+
+The skill-creator plugin includes a description optimization loop:
+
+### Trigger eval generation
+
+Generate 20 realistic eval queries (10 should-trigger, 10 should-not-trigger). Use the HTML review template from:
+`[skill-creator-plugin-path]/assets/eval_review.html`
+
+### Optimization loop
+
+```bash
+cd [skill-creator-plugin-path] && python -m scripts.run_loop \
+  --eval-set [path-to-trigger-eval.json] \
+  --skill-path [path-to-skill] \
+  --model [current-model-id] \
+  --max-iterations 5 \
+  --verbose
+```
+
+### Final validation
+
+Run the skill-writer self-check rubric (from skill-writer's Step 9) against the finished skill. All items must pass.

package/skills/skill-builder/templates/gap-analysis.md

@@ -0,0 +1,41 @@
+# Gap Analysis: [Skill Name]
+
+## Task Description
+
+[What the user is trying to accomplish -- the capability this skill should provide]
+
+## Gaps Identified
+
+### Gap 1: [Descriptive Name]
+
+- **What happened:** [Description of the failure or missing context when working without a skill]
+- **What was needed:** [The specific context, instruction, or knowledge that would fix it]
+- **Frequency:** [How often this comes up in real usage]
+- **Example task:** [A concrete task that exposes this gap]
+
+### Gap 2: [Descriptive Name]
+
+- **What happened:** [Description]
+- **What was needed:** [Context/instruction needed]
+- **Frequency:** [Frequency]
+- **Example task:** [Concrete example]
+
+### Gap 3: [Descriptive Name]
+
+- **What happened:** [Description]
+- **What was needed:** [Context/instruction needed]
+- **Frequency:** [Frequency]
+- **Example task:** [Concrete example]
+
+## Patterns
+
+- [Recurring themes across gaps -- e.g., "Claude consistently lacks knowledge about X"]
+- [Common failure modes -- e.g., "Without guidance, Claude chooses library A when library B is required"]
+- [Context that was repeatedly provided manually]
+
+## Candidate Eval Scenarios
+
+- [Task that would expose Gap 1 -- becomes the seed for an eval]
+- [Task that would expose Gap 2]
+- [Task that would expose multiple gaps simultaneously]
+- [Edge case that tests boundary behavior]

package/skills/skill-builder/workflows/improve-skill.md

@@ -0,0 +1,88 @@
+# Improve Skill Workflow
+
+Observation-first flow for iterating on an existing skill.
+
+## Prerequisites
+
+- An existing skill that needs improvement
+- The skill has been used at least once (or the user has observed specific issues)
+
+---
+
+## Phase 1: Observe
+
+**Goal:** Document the existing skill's current behavior by running it on real tasks.
+
+> "Use the Skill in real workflows: Give Claude B (with the Skill loaded) actual tasks, not test scenarios"
+
+### Process
+
+1. Identify the skill to improve. Read its current SKILL.md and any reference files.
+
+2. Ask the user what prompted the improvement:
+   - "What specific issue did you observe?"
+   - "Can you give me a concrete task where the skill underperformed?"
+   - "Is this a triggering issue (skill does not activate), a quality issue (skill activates but produces poor results), or a scope issue (skill does the wrong thing)?"
+
+3. Run the existing skill on 2-3 real tasks. For each, spawn a subagent:
+
+   ```
+   Execute this task using the skill at [path-to-existing-skill]:
+   - Read the skill at [path]/SKILL.md and follow its instructions
+   - Task: [realistic task from user]
+   - Save outputs to: [skill-name]-workspace/observation/task-[N]/outputs/
+   - Save transcript to: [skill-name]-workspace/observation/task-[N]/transcript.md
+   ```
+
+4. Analyze the transcripts. Document observations:
+   - Where did the skill work well?
+   - Where did it fail or produce subpar results?
+   - Did Claude B follow the skill's instructions as written?
+   - Did Claude B ignore any sections or files?
+   - Did Claude B explore in unexpected directions?
+
+5. Generate a gap analysis (same template as new-skill Phase 1) focused on the delta between current behavior and desired behavior.
+
+**Output:** `[skill-name]-workspace/gap-analysis.md` with observation-based gaps
+
+---
+
+## Phase 2-6: Follow the New Skill Workflow
+
+From here, follow the same phases as `${CLAUDE_SKILL_DIR}/workflows/new-skill.md`, starting at Phase 2 (Build Evals).
+
+Key differences from the new-skill flow:
+
+- **Phase 2 (Build Evals):** Evals should test the specific issues observed in Phase 1, not hypothetical gaps.
+
+- **Phase 3 (Write Skill):** Instead of writing from scratch, invoke `/skill-writer` with:
+
+  ```
+  Refine this existing skill based on observation findings.
+
+  Current SKILL.md: [reference or paste current skill]
+  Gap analysis: [reference observation-based gaps]
+  Eval scenarios: [reference evals]
+
+  Constraint: Preserve what works. Only change what the observations demand.
+  ```
+
+- **Phase 4 (Test):** The baseline is the CURRENT skill (snapshot it before editing). Compare old-skill vs new-skill, not with-skill vs without-skill.
+
+  Before making any changes, snapshot the existing skill:
+  ```bash
+  cp -r [skill-path] [workspace]/skill-snapshot/
+  ```
+
+  Then for baseline runs, point subagents at the snapshot:
+  ```
+  Execute this task using the ORIGINAL skill at [workspace]/skill-snapshot/:
+  - Read the skill and follow its instructions
+  - Task: [eval prompt]
+  - Save outputs to: [workspace]/iteration-N/eval-[name]/old_skill/outputs/
+  - Save transcript to: [workspace]/iteration-N/eval-[name]/old_skill/transcript.md
+  ```
+
+- **Phase 5 (Iterate):** Same process. The improvement loop compares new version against the snapshot.
+
+- **Phase 6 (Polish):** Same process. Run description optimization if triggering was an issue.

package/skills/skill-builder/workflows/new-skill.md

@@ -0,0 +1,223 @@
+# New Skill Workflow
+
+Full evaluation-driven lifecycle for building a new skill from scratch.
+
+## Prerequisites
+
+- The user has a task or domain they want to capture as a skill
+- No existing skill for this capability (or intentionally starting fresh)
+
+---
+
+## Phase 1: Identify Gaps
+
+**Goal:** Document what fails or requires repeated context when working without a skill.
+
+### Process
+
+1. Have a guided conversation to uncover gaps. Explore these areas:
+   - "What task were you doing when you realized you needed a skill?"
+   - "What context did you repeatedly provide to Claude?"
+   - "Where did Claude fail or produce subpar results without guidance?"
+   - "What domain knowledge was missing?"
+   - "What specific format or structure did you need?"
+   - "Were there tools or scripts that needed to be used in a particular way?"
+   - "What rules or constraints did Claude violate?"
+
+2. As patterns emerge, probe for eval-worthy scenarios:
+   - "Can you give me a concrete example of a task where this failed?"
+   - "What would success look like for that specific task?"
+   - "Are there edge cases where the right approach changes?"
+
+3. Generate `gap-analysis.md` from the conversation using the template at `${CLAUDE_SKILL_DIR}/templates/gap-analysis.md`. Fill in all sections from what was discussed.
+
+4. Review the gap analysis with the user. Confirm completeness before moving to Phase 2.
+
+**Output:** `[skill-name]-workspace/gap-analysis.md`
+
+---
+
+## Phase 2: Build Evals
+
+**Goal:** Create 3+ evaluation scenarios that test the identified gaps. Establish a baseline.
+
+### Process
+
+1. Transform each gap into at least one eval scenario. Each scenario needs:
+   - A realistic user prompt (detailed and specific, like a real request)
+   - A description of what success looks like
+   - Objectively verifiable expectations (assertions)
+
+2. Draft evals using the schema at `${CLAUDE_SKILL_DIR}/templates/eval-scenario.json`. Ensure:
+   - Minimum 3 scenarios (official requirement)
+   - Every identified gap has at least one scenario testing it
+   - Expectations are objectively verifiable, not subjective
+   - Prompts sound like things a real user would say
+
+3. Review eval scenarios with the user. Adjust until both sides are satisfied.
+
+4. Save to `[skill-name]-workspace/evals/evals.json`.
+
+5. **Establish baseline.** For each eval, spawn a subagent WITHOUT any skill:
+
+   ```
+   Execute this task with NO skill loaded:
+   - Task: [eval prompt]
+   - Input files: [eval files if any, or "none"]
+   - Save all output files to: [workspace]/iteration-0/eval-[name]/without_skill/outputs/
+   - Save a complete transcript of your work to: [workspace]/iteration-0/eval-[name]/without_skill/transcript.md
+   ```
+
+   Spawn all baseline runs in parallel. Capture timing data when each completes.
+
+6. Grade baseline results using the skill-creator grading agent. See `${CLAUDE_SKILL_DIR}/references/delegation-map.md` for exact grading invocation.
+
+**Output:** `[skill-name]-workspace/evals/evals.json` and baseline results in `iteration-0/`
+
+---
+
+## Phase 3: Write Minimal Skill
+
+**Goal:** Create just enough skill content to address the documented gaps and pass evaluations.
+
+### Process
+
+1. Invoke `/skill-writer` with this context:
+
+   ```
+   Create a skill based on this gap analysis and eval scenarios.
+
+   Gap analysis: [reference or paste gap-analysis.md]
+   Eval scenarios: [reference or paste evals.json expected_output and expectations]
+   Baseline failures: [summarize what Claude got wrong in iteration-0]
+
+   Constraint: Write the minimum instructions needed to address these specific gaps.
+   Every line must serve a documented gap. Do not over-document.
+   ```
+
+2. `/skill-writer` will run its workflow: classify type, set degree of freedom, ask clarifying questions, produce the SKILL.md artifact.
+
+3. Review the draft with the user:
+   - "Does this address all the gaps we identified?"
+   - "Is anything here unnecessary or over-engineered?"
+   - "Would this pass our eval scenarios?"
+
+4. Save the skill to its target directory.
+
+**Output:** The skill's SKILL.md (and optional reference files)
+
+---
+
+## Phase 4: Test (Feedback Loop)
+
+**Goal:** Run the skill on eval scenarios, compare against baseline, identify remaining gaps.
+
+### Process
+
+1. **Spawn all runs in parallel.** For each eval scenario, launch a with-skill subagent:
+
+   ```
+   Execute this task:
+   - Read the skill at [path-to-skill]/SKILL.md and follow its instructions
+   - Task: [eval prompt from evals.json]
+   - Input files: [eval files if any, or "none"]
+   - Save all output files to: [workspace]/iteration-N/eval-[name]/with_skill/outputs/
+   - Save a complete transcript of your work to: [workspace]/iteration-N/eval-[name]/with_skill/transcript.md
+   ```
+
+   For iteration-1, the without-skill baseline already exists from Phase 2.
+
+2. **While runs are in progress**, review and refine assertions if needed based on what was learned from the baseline.
+
+3. **When runs complete**, immediately capture timing data (`total_tokens`, `duration_ms`) to `timing.json` in each run directory. This data is only available in the task completion notification.
+
+4. **Grade each run** using the skill-creator grading agent. See `${CLAUDE_SKILL_DIR}/references/delegation-map.md` for the grading process.
+
+5. **Aggregate into benchmark** using skill-creator's aggregation script. See delegation-map.md for the exact command.
+
+6. **Launch the eval viewer** using skill-creator's generate_review.py. See delegation-map.md for the exact command. For iteration 2+, include `--previous-workspace` to show diffs.
+
+7. Tell the user to review in the viewer:
+   - "Outputs" tab: click through each test case, leave feedback
+   - "Benchmark" tab: quantitative comparison (pass rates, timing, tokens)
+
+8. Wait for the user to complete their review.
+
+**Output:** `grading.json`, `benchmark.json`, `feedback.json` in the iteration directory
+
+---
+
+## Phase 5: Iterate
+
+**Goal:** Refine the skill based on observed Claude B behavior and user feedback.
+
+### Process
+
+1. Read `feedback.json` from the viewer. Empty feedback means the user was satisfied with that test case.
+
+2. Read transcripts from Phase 4 runs. Watch for the signals the official docs highlight:
+   - **Unexpected exploration paths** -- Claude B read files in an order you did not anticipate
+   - **Missed connections** -- Claude B did not follow references to important files
+   - **Overreliance on certain sections** -- content that should be promoted to SKILL.md
+   - **Ignored content** -- files Claude B never accessed (may be unnecessary or poorly signaled)
+   - **Repeated work across test cases** -- all subagents wrote similar helper scripts (bundle them into the skill)
+
+3. Synthesize observations into actionable improvements. For each piece of feedback, identify the specific skill change that would fix it.
+
+4. Apply improvements. For significant changes, re-invoke `/skill-writer` with:
+
+   ```
+   Refine this existing skill based on testing observations.
+
+   Current SKILL.md: [reference or paste]
+   User feedback: [from feedback.json -- only non-empty entries]
+   Behavioral observations: [from transcript analysis]
+
+   Specific issues to address:
+   1. [Issue]
+   2. [Issue]
+
+   Constraint: Only change what the feedback demands. Do not reorganize working content.
+   ```
+
+5. Key principles for this phase (from the official docs):
+   - **Generalize from feedback** -- the skill will be used across many different prompts, not just these test cases
+   - **Keep the prompt lean** -- remove instructions that are not pulling their weight
+   - **Explain the why** -- theory of mind beats rigid MUSTs
+   - **Bundle repeated work** -- if subagents all wrote similar scripts, add them to the skill
+
+6. Return to Phase 4 with the refined skill. Continue iterating until:
+   - User feedback is all empty (satisfied with every test case)
+   - Pass rates meet acceptable thresholds
+   - No meaningful progress between iterations
+
+---
+
+## Phase 6: Polish
+
+**Goal:** Optimize the skill description for triggering accuracy and run final validation.
+
+### Process
+
+1. **Description optimization.** Follow the process in `${CLAUDE_SKILL_DIR}/workflows/polish-skill.md`.
+
+2. **Final validation.** Run the skill-writer self-check rubric against the finished skill:
+   - [ ] Description is third person with trigger phrases
+   - [ ] Under 500 lines
+   - [ ] States what to do in positive terms (not prohibition-heavy)
+   - [ ] Degree of freedom matches task fragility
+   - [ ] Progressive disclosure used (heavy content in separate files)
+   - [ ] Examples are concrete, not abstract
+   - [ ] Frontmatter fields are valid
+   - [ ] One skill = one capability
+
+3. **Final checklist** from the official Anthropic docs:
+   - [ ] At least 3 evaluation scenarios created and passing
+   - [ ] Tested with real usage scenarios
+   - [ ] Skill solves documented gaps (not imagined requirements)
+   - [ ] Iterative refinement based on observed behavior (not assumptions)
+
+4. Present the finished skill to the user with:
+   - Final benchmark comparison (latest iteration vs baseline)
+   - Summary of gaps addressed
+   - Any remaining limitations or known edge cases

package/skills/skill-builder/workflows/polish-skill.md

@@ -0,0 +1,83 @@
+# Polish Skill Workflow
+
+Final optimization pass for a skill that is functionally complete.
+
+## Prerequisites
+
+- The skill passes its evaluation scenarios
+- The user is satisfied with output quality
+- This is the final step before the skill is considered done
+
+---
+
+## Step 1: Description Optimization
+
+Optimize the skill's description for triggering accuracy using the skill-creator's trigger eval system.
+
+### Generate trigger eval queries
+
+Create 20 eval queries: 10 should-trigger and 10 should-not-trigger.
+
+**Should-trigger queries (10):** Different phrasings of the same intent. Include:
+- Formal and casual variations
+- Cases where the user does not explicitly name the skill but clearly needs it
+- Uncommon use cases
+- Cases where this skill competes with another but should win
+
+**Should-not-trigger queries (10):** Near-misses that share keywords but need something different. Include:
+- Adjacent domains with overlapping terminology
+- Ambiguous phrasing where naive keyword matching would falsely trigger
+- Tasks that touch the skill's domain but in a context where another tool is better
+
+All queries must be realistic -- detailed, specific, with file paths, personal context, casual speech. Not abstract one-liners.
+
+### Review with user
+
+Present the eval set using the skill-creator's HTML review template. See `${CLAUDE_SKILL_DIR}/references/delegation-map.md` for the exact process.
+
+The user can edit queries, toggle should-trigger, and add/remove entries.
+
+### Run optimization loop
+
+See `${CLAUDE_SKILL_DIR}/references/delegation-map.md` for the exact command. The loop:
+1. Splits eval set into 60% train / 40% held-out test
+2. Evaluates current description (3 runs per query for reliability)
+3. Proposes improvements based on failures
+4. Re-evaluates on both train and test
+5. Iterates up to 5 times
+6. Selects best description by test score (avoids overfitting)
+
+### Apply result
+
+Update the skill's SKILL.md frontmatter with the optimized description. Show the user before/after with scores.
+
+---
+
+## Step 2: Final Validation
+
+Run the skill-writer self-check rubric:
+
+- [ ] Description is third person with trigger phrases
+- [ ] SKILL.md body under 500 lines
+- [ ] States what to do in positive terms (not prohibition-heavy)
+- [ ] Degree of freedom matches task fragility
+- [ ] Progressive disclosure used (heavy content in separate files)
+- [ ] No time-sensitive claims unless clearly dated
+- [ ] Examples are concrete, not abstract
+- [ ] Frontmatter fields are valid per official docs
+- [ ] One skill = one capability
+- [ ] Consistent terminology throughout
+- [ ] File references are one level deep from SKILL.md
+- [ ] Files over 100 lines have a table of contents
+
+---
+
+## Step 3: Final Summary
+
+Present the finished skill to the user:
+
+1. **Benchmark summary:** Final pass rate vs baseline, with delta
+2. **Gaps addressed:** Map each original gap to the skill content that addresses it
+3. **Description optimization:** Before/after trigger accuracy scores
+4. **Known limitations:** Anything the skill does not handle (scope boundaries)
+5. **Maintenance notes:** What to watch for in future usage that might warrant re-iteration