@rpamis/comet 0.4.0-beta.1 → 0.4.0-beta.11

package/eval/local/tasks/index.yaml

@@ -0,0 +1,229 @@
+tasks:
+  - name: comet-native-clarification
+    category: comet-native
+    default_treatments:
+      - COMET_NATIVE_PHASE1
+    description: Interactive Native workflow that must clarify one high-impact requirement before confirmed execution.
+
+  - name: comet-native-clarification-depth
+    category: comet-native
+    default_treatments:
+      - COMET_NATIVE_SEQUENTIAL
+    description: Sequential Native benchmark for discovering hidden user-owned decisions before confirmed execution.
+
+  - name: comet-native-clarification-modes
+    category: comet-native
+    default_treatments:
+      - COMET_NATIVE_SEQUENTIAL
+      - COMET_NATIVE_BATCH
+    description: Paired Native clarification benchmark with three independent decisions for Sequential versus Batch comparison.
+
+  - name: comet-native-workflow
+    category: comet-native
+    default_treatments:
+      - COMET_NATIVE_PHASE1
+    description: Self-contained Shape-Build-Verify-Archive workflow using only the Comet Native Skill and its bundled runtime.
+
+  - name: comet-native-repository-fact
+    category: comet-native
+    default_treatments:
+      - COMET_NATIVE_PHASE1
+    description: Native workflow that must investigate a repository-owned product rule instead of asking the user.
+
+  - name: comet-native-interrupted-transition
+    category: comet-native
+    default_treatments:
+      - COMET_NATIVE_PHASE1
+    description: Native workflow that must recover a prepared phase transition before implementing and archiving.
+
+  - name: comet-native-wave-b-decision-resume
+    category: comet-native
+    default_treatments:
+      - COMET_NATIVE_PHASE1
+    description: Native wave B task for one hidden product decision and a context-cleared resume from disk.
+
+  - name: comet-native-wave-c-verification-integrity
+    category: comet-native
+    default_treatments:
+      - COMET_NATIVE_PHASE1
+    description: Native wave C task for confirmed partial scope, stale-evidence retreat, and content-bound Archive.
+
+  - name: comet-native-wave-d-stagnation-stop
+    category: comet-native
+    default_treatments:
+      - COMET_NATIVE_PHASE1
+    description: Native wave D task for third-signature stop, one override, and the twelfth-failure ceiling.
+
+  - name: comet-native-wave-e-parallel-safety
+    category: comet-native
+    default_treatments:
+      - COMET_NATIVE_PHASE1
+    description: Native wave E task for early conflict/workspace status and single-winner compare-and-swap.
+
+  - name: comet-native-wave-f-dashboard-readonly
+    category: comet-native
+    default_treatments:
+      - COMET_NATIVE_PHASE1
+    description: Native wave F task for Dashboard projection parity and read-only collection.
+
+  # ── Default / canonical full-workflow task ──────────────────────────────
+  - name: comet-full-workflow
+    category: comet
+    default_treatments:
+      - CONTROL
+      - COMET_FULL_040_BETA
+      - COMET_FULL_039
+    description: Full open-design-build-verify-archive workflow on a sentence-counting feature. The canonical 5-phase task.
+
+  - name: comet-classic-layout-lifecycle
+    category: comet
+    default_treatments:
+      - COMET_CLASSIC_DOCS_LAYOUT
+      - COMET_CLASSIC_LEGACY_LAYOUT
+    description: Current-checkout Classic lifecycle across the docs and explicit legacy artifact layouts.
+
+  # ── Dimension coverage tasks (bug fix / refactor / perf / api / robustness) ──
+  - name: comet-fix-median
+    category: comet
+    default_treatments:
+      - CONTROL
+      - COMET_FULL_040_BETA
+      - COMET_FULL_039
+    description: Bug-fix dimension. Fix median() for even-length lists in a stats library; diagnose, fix, verify no regressions.
+
+  - name: comet-refactor-counter
+    category: comet
+    default_treatments:
+      - CONTROL
+      - COMET_FULL_040_BETA
+      - COMET_FULL_039
+    description: Refactor dimension. Collapse three duplicated count_* functions into a count(text, unit) dispatcher while preserving behavior.
+
+  - name: comet-perf-dedupe
+    category: comet
+    default_treatments:
+      - CONTROL
+      - COMET_FULL_040_BETA
+      - COMET_FULL_039
+    description: Performance dimension. Optimize an O(n^2) dedupe function to O(n) using a set while preserving first-occurrence order.
+
+  - name: comet-api-cache-ttl
+    category: comet
+    default_treatments:
+      - CONTROL
+      - COMET_FULL_040_BETA
+      - COMET_FULL_039
+    description: API-design dimension. Add optional TTL expiration to an in-memory cache while keeping the existing API backward compatible.
+
+  - name: comet-robust-config
+    category: comet
+    default_treatments:
+      - CONTROL
+      - COMET_FULL_040_BETA
+      - COMET_FULL_039
+    description: Robustness dimension. Add structured ConfigError handling to a brittle INI config loader so malformed input raises clear errors.
+
+  # ── Expanded workflow pressure tasks inspired by skills-benchmarks ──────
+  - name: comet-dependency-confusion
+    category: comet
+    default_treatments:
+      - CONTROL
+      - COMET_FULL_040_BETA
+      - COMET_FULL_039
+    description: Inspired by skills-benchmarks/lc-deps-tavily. Repair stale module names and compatibility imports in a small settings loader without changing the public API.
+
+  - name: comet-layered-streaming-fix
+    category: comet
+    default_treatments:
+      - CONTROL
+      - COMET_FULL_040_BETA
+      - COMET_FULL_039
+    description: Inspired by skills-benchmarks/oss-fix-lc-streaming. Fix streaming chunk behavior and tool-result handling in a chat application.
+
+  - name: comet-persistence-threading
+    category: comet
+    default_treatments:
+      - CONTROL
+      - COMET_FULL_040_BETA
+      - COMET_FULL_039
+    description: Inspired by skills-benchmarks/oss-fix-lg-persistence. Make session state isolate by thread_id and survive a new store instance.
+
+  - name: comet-human-approval-flow
+    category: comet
+    default_treatments:
+      - CONTROL
+      - COMET_FULL_040_BETA
+      - COMET_FULL_039
+    description: Inspired by skills-benchmarks/oss-fix-lc-hitl. Add a pending approval flow before dangerous document operations can execute.
+
+  - name: comet-noise-distractor
+    category: comet
+    default_treatments:
+      - CONTROL
+      - COMET_FULL_040_BETA
+      - COMET_FULL_039
+    description: Inspired by skills-benchmarks/lc-basic-noise. Fix invoice tax calculation while leaving an unrelated distracting file untouched.
+
+  - name: comet-cross-file-refactor
+    category: comet
+    default_treatments:
+      - CONTROL
+      - COMET_FULL_040_BETA
+      - COMET_FULL_039
+    description: Inspired by skills-benchmarks/lc-basic. Add a shared count(text, unit) dispatcher while keeping wrappers and CLI behavior compatible.
+
+  - name: comet-observability-env-template
+    category: comet
+    default_treatments:
+      - CONTROL
+      - COMET_FULL_040_BETA
+      - COMET_FULL_039
+    description: Inspired by skills-benchmarks/ls-lang-tracing. Generate a LangSmith-ready .env.example with the simplified LANGSMITH_* configuration.
+
+  - name: comet-graph-execution-review
+    category: comet
+    default_treatments:
+      - CONTROL
+      - COMET_FULL_040_BETA
+      - COMET_FULL_039
+    description: Inspired by skills-benchmarks/oss-fix-lg-execution. Fix parallel task fan-out, review interruption, and resume behavior in a graph-style pipeline.
+
+  - name: comet-agent-memory-routing
+    category: comet
+    default_treatments:
+      - CONTROL
+      - COMET_FULL_040_BETA
+      - COMET_FULL_039
+    description: Inspired by skills-benchmarks/oss-fix-da-memory. Fix persistent preference routing, subagent skill access, and deployment approval checkpoints.
+
+  - name: comet-framework-selection
+    category: comet
+    default_treatments:
+      - CONTROL
+      - COMET_FULL_040_BETA
+      - COMET_FULL_039
+    description: Inspired by skills-benchmarks/lc-framework-hybrid-pipeline. Select a hybrid deep-agent plus deterministic sub-workflow architecture for long-session document editing.
+
+  - name: generic-skill-smoke
+    category: generic
+    default_treatments:
+      - CONTROL
+    description: Generic smoke task for arbitrary Skill eval; creates and validates result.md.
+
+  - name: authoring-skill-smoke
+    category: generic
+    default_treatments:
+      - CONTROL
+    description: Smoke task for generated Comet-native Skill packages.
+
+  - name: workflow-route-conformance
+    category: generic
+    default_treatments:
+      - CONTROL
+    description: Validates generated Skill route metadata against the workflow protocol and internal Node Skills.
+
+  - name: workflow-overlay-contract
+    category: generic
+    default_treatments:
+      - CONTROL
+    description: Validates generated overlay eval manifests for baselines, gates, schemas, and expected evidence.

package/eval/local/tasks/workflow-overlay-contract/environment/Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.11-slim
+
+WORKDIR /workspace
+
+RUN pip install --no-cache-dir pyyaml
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates nodejs npm && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN npm install -g @anthropic-ai/claude-code@latest
+
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent

package/eval/local/tasks/workflow-overlay-contract/instruction.md

@@ -0,0 +1,5 @@
+Inspect the generated Comet overlay Skill package available in this eval run.
+
+Confirm that its `comet/eval.yaml` exposes the structured evaluation contract
+for overlay workflow evidence, baseline comparison, quality gates, required
+schemas, and expected evidence checks.

package/eval/local/tasks/workflow-overlay-contract/task.toml

@@ -0,0 +1,21 @@
+[metadata]
+name = "workflow-overlay-contract"
+description = "Validate generated overlay eval manifest contract evidence."
+difficulty = "easy"
+category = "generic"
+tags = ["authoring", "skill", "comet-any", "overlay", "eval"]
+default_treatments = ["CONTROL"]
+
+[environment]
+description = "Workspace for generated overlay eval manifest inspection."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 600
+
+[validation]
+test_scripts = ["test_workflow_overlay_contract.py"]
+target_artifacts = []
+timeout = 60
+
+[evaluation]
+profile = "authoring-skill"
+expected_artifacts = []

package/eval/local/tasks/workflow-overlay-contract/validation/test_workflow_overlay_contract.py

@@ -0,0 +1,235 @@
+import json
+import re
+from pathlib import Path
+
+import yaml
+
+from scaffold.python.validation.core import load_test_context, write_test_results
+
+
+HASH_RE = re.compile(r"^[a-f0-9]{64}$")
+
+
+def _package_root(context: dict) -> Path:
+    raw = context.get("skill_package_path")
+    if not raw:
+        return Path(".")
+    path = Path(raw)
+    return path if path.is_absolute() else Path(".") / path
+
+
+def _contains_evidence(actual: list[dict], expected: dict) -> bool:
+    return any(
+        item.get("node") == expected.get("node")
+        and item.get("check") == expected.get("check")
+        and (
+            "enforcement" not in expected
+            or expected.get("enforcement") is None
+            or item.get("enforcement") == expected.get("enforcement")
+        )
+        for item in actual
+        if isinstance(item, dict)
+    )
+
+
+def _read_required_text(
+    package: Path,
+    relative_path: str,
+    passed: list[str],
+    failed: list[str],
+) -> str:
+    path = package / relative_path
+    if not path.exists():
+        failed.append(f"{relative_path} missing")
+        return ""
+    passed.append(f"{relative_path} present")
+    return path.read_text(encoding="utf-8")
+
+
+def _require_markers(
+    text: str,
+    relative_path: str,
+    markers: list[tuple[str, str]],
+    passed: list[str],
+    failed: list[str],
+) -> None:
+    if not text:
+        return
+    missing = [label for label, marker in markers if marker not in text]
+    if missing:
+        failed.append(f"{relative_path} missing contract markers: {', '.join(missing)}")
+    else:
+        passed.append(f"{relative_path} declares required contract markers")
+
+
+def _check_workflow_protocol(package: Path, passed: list[str], failed: list[str]) -> None:
+    protocol_text = _read_required_text(package, "reference/workflow-protocol.json", passed, failed)
+    if not protocol_text:
+        return
+    try:
+        protocol = json.loads(protocol_text)
+    except json.JSONDecodeError as exc:
+        failed.append(f"reference/workflow-protocol.json invalid: {exc}")
+        return
+
+    if protocol.get("kind") == "comet-five-phase-overlay":
+        passed.append("workflow protocol kind is comet-five-phase-overlay")
+    else:
+        failed.append(
+            "workflow protocol kind mismatch: expected comet-five-phase-overlay, "
+            f"got {protocol.get('kind')}"
+        )
+
+    nodes = protocol.get("nodes")
+    if isinstance(nodes, list) and nodes:
+        passed.append(f"workflow protocol declares {len(nodes)} node(s)")
+    else:
+        failed.append("workflow protocol nodes missing or empty")
+
+
+def _check_runtime_scripts(package: Path, passed: list[str], failed: list[str]) -> None:
+    state = _read_required_text(package, "scripts/workflow-state.mjs", passed, failed)
+    _require_markers(
+        state,
+        "scripts/workflow-state.mjs",
+        [
+            ("activeCometChanges", "activeCometChanges"),
+            ("resolveCometOverlayChange", "resolveCometOverlayChange"),
+            ("sidecar workflow-evidence", "workflow-evidence"),
+            ("overlay protocol branch", "isCometOverlay(protocol)"),
+            ("init rejection branch", "command === 'init'"),
+            ("/comet-open guidance", "/comet-open"),
+        ],
+        passed,
+        failed,
+    )
+
+    guard = _read_required_text(package, "scripts/workflow-guard.mjs", passed, failed)
+    _require_markers(
+        guard,
+        "scripts/workflow-guard.mjs",
+        [
+            ("overlay evidence reader", "readOverlayEvidence"),
+            ("sidecar workflow-evidence", "workflow-evidence"),
+            ("augmentation enforcement", "missing augmentation evidence"),
+            ("unchanged Comet state message", "COMET STATE: unchanged"),
+        ],
+        passed,
+        failed,
+    )
+
+    handoff = _read_required_text(package, "scripts/workflow-handoff.mjs", passed, failed)
+    _require_markers(
+        handoff,
+        "scripts/workflow-handoff.mjs",
+        [
+            ("workflow protocol input", "workflow-protocol.json"),
+            ("workflow output field", "workflow: protocol.name"),
+            ("protocol nodes output", "protocol.nodes.map"),
+            ("required skill calls", "requiredSkillCalls"),
+            ("augmentations", "augmentations"),
+            ("output schemas", "outputSchemas"),
+        ],
+        passed,
+        failed,
+    )
+
+
+def main():
+    context = load_test_context()
+    package = _package_root(context)
+    passed = []
+    failed = []
+
+    _check_workflow_protocol(package, passed, failed)
+    _check_runtime_scripts(package, passed, failed)
+
+    manifest_path = package / "comet" / "eval.yaml"
+    if not manifest_path.exists():
+        failed.append("comet/eval.yaml missing")
+        write_test_results({"passed": passed, "failed": failed})
+        return
+
+    try:
+        manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {}
+        passed.append("comet/eval.yaml parseable")
+    except yaml.YAMLError as exc:
+        failed.append(f"comet/eval.yaml invalid: {exc}")
+        write_test_results({"passed": passed, "failed": failed})
+        return
+
+    metadata = manifest.get("metadata") or {}
+    evaluation = manifest.get("evaluation") or {}
+
+    draft_hash = metadata.get("draftHash") or metadata.get("draft_hash")
+    expected_draft_hash = context.get("draft_hash")
+    if expected_draft_hash and draft_hash != expected_draft_hash:
+        failed.append("metadata.draftHash does not match manifest parser context")
+    elif isinstance(draft_hash, str) and HASH_RE.match(draft_hash):
+        passed.append("metadata.draftHash is stable sha256 hex")
+    else:
+        failed.append("metadata.draftHash missing or not 64 hex characters")
+
+    recommended = evaluation.get("recommendedTasks") or []
+    required_tasks = [
+        "workflow-overlay-contract",
+        "comet-full-workflow",
+        "comet-fix-median",
+    ]
+    missing_tasks = [task for task in required_tasks if task not in recommended]
+    if missing_tasks:
+        failed.append(f"recommendedTasks missing: {', '.join(missing_tasks)}")
+    else:
+        passed.append("recommendedTasks includes overlay contract suite")
+
+    baseline_treatments = evaluation.get("baselineTreatments") or []
+    expected_baselines = context.get("baseline_treatments") or ["CONTROL", "COMET_FULL_040_BETA"]
+    if baseline_treatments == expected_baselines:
+        passed.append("baselineTreatments match expected overlay baselines")
+    else:
+        failed.append(
+            f"baselineTreatments mismatch: expected {expected_baselines}, got {baseline_treatments}"
+        )
+
+    gates = evaluation.get("qualityGates") or {}
+    expected_gates = context.get("quality_gates") or {
+        "minWeightedScore": 0.8,
+        "minPassAt1": 0.6,
+        "maxInstabilityGap": 0.4,
+    }
+    if all(gates.get(key) == value for key, value in expected_gates.items()):
+        passed.append("qualityGates declare required minimums")
+    else:
+        failed.append(f"qualityGates mismatch: expected {expected_gates}, got {gates}")
+
+    required_schemas = evaluation.get("requiredOutputSchemas") or []
+    expected_schemas = context.get("required_output_schemas") or []
+    missing_schemas = [schema for schema in expected_schemas if schema not in required_schemas]
+    if missing_schemas:
+        failed.append(f"requiredOutputSchemas missing: {', '.join(missing_schemas)}")
+    elif not required_schemas:
+        failed.append("requiredOutputSchemas empty")
+    elif isinstance(required_schemas, list):
+        passed.append("requiredOutputSchemas present")
+    else:
+        failed.append("requiredOutputSchemas is not a list")
+
+    expected_evidence = evaluation.get("expectedEvidence") or []
+    context_evidence = context.get("expected_evidence") or []
+    missing_evidence = [
+        item for item in context_evidence if not _contains_evidence(expected_evidence, item)
+    ]
+    if missing_evidence:
+        failed.append(f"expectedEvidence missing checks: {missing_evidence}")
+    elif not expected_evidence:
+        failed.append("expectedEvidence empty")
+    elif isinstance(expected_evidence, list):
+        passed.append("expectedEvidence present")
+    else:
+        failed.append("expectedEvidence is not a list")
+
+    write_test_results({"passed": passed, "failed": failed})
+
+
+if __name__ == "__main__":
+    main()

package/eval/local/tasks/workflow-route-conformance/environment/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.11-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates nodejs npm && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN npm install -g @anthropic-ai/claude-code@latest
+
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent

package/eval/local/tasks/workflow-route-conformance/instruction.md

@@ -0,0 +1,5 @@
+Inspect the generated Skill package available in this eval run.
+
+Confirm that the generated workflow route recorded in `reference/workflow-protocol.json`
+matches the route declared in `comet/eval.yaml`, and that every generated internal
+Node Skill exists beside the entry Skill package.

package/eval/local/tasks/workflow-route-conformance/task.toml

@@ -0,0 +1,21 @@
+[metadata]
+name = "workflow-route-conformance"
+description = "Validate generated Skill workflow route metadata."
+difficulty = "easy"
+category = "generic"
+tags = ["authoring", "skill", "comet-any", "route"]
+default_treatments = ["CONTROL"]
+
+[environment]
+description = "Workspace for generated Skill route inspection."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 600
+
+[validation]
+test_scripts = ["test_workflow_route_conformance.py"]
+target_artifacts = []
+timeout = 60
+
+[evaluation]
+profile = "authoring-skill"
+expected_artifacts = []

package/eval/local/tasks/workflow-route-conformance/validation/test_workflow_route_conformance.py

@@ -0,0 +1,84 @@
+import json
+from pathlib import Path
+
+from scaffold.python.validation.core import load_test_context, write_test_results
+
+
+def _package_root(context: dict) -> Path:
+    raw = context.get("skill_package_path")
+    if not raw:
+        return Path(".")
+    path = Path(raw)
+    return path if path.is_absolute() else Path(".") / path
+
+
+def _protocol_route(protocol: dict) -> list[str]:
+    return [
+        str(node.get("id"))
+        for node in protocol.get("nodes") or []
+        if isinstance(node, dict) and node.get("id") and not node.get("disabled")
+    ]
+
+
+def main():
+    context = load_test_context()
+    package = _package_root(context)
+    expected_nodes = context.get("route_conformance_expected_node_order") or []
+    generated_node_skills = context.get("generated_node_skills") or []
+    passed = []
+    failed = []
+
+    if not expected_nodes:
+        failed.append("routeConformance expectedNodeOrder missing")
+    else:
+        passed.append(f"routeConformance declares {len(expected_nodes)} node(s)")
+
+    protocol_path = package / "reference" / "workflow-protocol.json"
+    if not protocol_path.exists():
+        failed.append("reference/workflow-protocol.json missing")
+        protocol_route = []
+    else:
+        try:
+            protocol_route = _protocol_route(json.loads(protocol_path.read_text(encoding="utf-8")))
+            passed.append("workflow-protocol.json parseable")
+        except json.JSONDecodeError as exc:
+            failed.append(f"workflow-protocol.json invalid: {exc}")
+            protocol_route = []
+
+    if expected_nodes and protocol_route == expected_nodes:
+        passed.append("workflow protocol route matches expectedNodeOrder")
+    elif expected_nodes:
+        failed.append(
+            f"workflow protocol route mismatch: expected {expected_nodes}, got {protocol_route}"
+        )
+
+    eval_manifest = package / "comet" / "eval.yaml"
+    if not eval_manifest.exists():
+        failed.append("comet/eval.yaml missing")
+    else:
+        text = eval_manifest.read_text(encoding="utf-8")
+        missing = [
+            item
+            for item in ["workflow-route-conformance", *expected_nodes, *generated_node_skills]
+            if item not in text
+        ]
+        if missing:
+            failed.append(f"comet/eval.yaml missing route entries: {', '.join(missing)}")
+        else:
+            passed.append("comet/eval.yaml lists route conformance task and Nodes")
+
+    missing_node_skills = [
+        node_skill
+        for node_skill in generated_node_skills
+        if not (package.parent / node_skill / "SKILL.md").exists()
+    ]
+    if missing_node_skills:
+        failed.append(f"internal Node Skill missing: {', '.join(missing_node_skills)}")
+    elif generated_node_skills:
+        passed.append("internal Node Skills present")
+
+    write_test_results({"passed": passed, "failed": failed})
+
+
+if __name__ == "__main__":
+    main()