delimit-cli 4.3.3 → 4.4.0

package/gateway/ai/content_grounding/consume.py

@@ -0,0 +1,280 @@
+"""
+Week 2 consumer API for the grounding layer (LED-1084).
+
+Exposes the gate generators (social drafter, blog pipeline, storyline) must
+call before emitting text. Amendments A6+A9+A10 from the adversarial rebuttal
+(/home/delimit/delimit-private/strategy/CONTENT_GROUNDING_REBUTTAL_2026_04.md):
+
+  A6. Hard bans on unresolved claim categories (comparative, adoption,
+      customer, roadmap) apply here at the gate — not just at schema time.
+  A9. Deterministic extraction gate: extract candidate claims from output →
+      classify → map to allowed_claim_ids OR approved inference rule →
+      reject on any unmatched/uncertain claim.
+  A10. One-strike kill-switch semantics: callers that detect a slippage
+       MUST revert ALL generators to manual-only mode.
+
+Week 2 scope:
+  - fetch_grounding_bundle(venture, days)  → GroundingBundle for a window
+  - build_allowed_claim_set(bundle)        → frozenset of safe-to-use texts
+  - load_feature_whitelist()               → shipped-feature list
+  - unreleased_feature_detector(text)      → True if text claims a feature
+                                              that is NOT in the whitelist
+  - score_draft_grounding(text, bundle)    → 0.0-1.0 grounding score
+                                              (simple coverage heuristic for
+                                               v1; Week 3 upgrades to
+                                               classifier-based)
+
+Not in scope (Week 3+):
+  - Paraphrase classifier (we reject non-verbatim for now)
+  - Implication detector (hard-ban suffices for now)
+  - Comparative claim classifier (ban + whitelist-only for now)
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from .schemas import Claim, ClaimType, GroundedEvent, Visibility
+from .build import build_grounding_index
+
+logger = logging.getLogger("delimit.ai.content_grounding.consume")
+
+FEATURES_FILE = Path(os.environ.get(
+    "DELIMIT_GROUNDING_FEATURES",
+    str(Path.home() / ".delimit" / "content" / "grounding" / "features.json"),
+))
+
+
+# ---------------------------------------------------------------------------
+# GroundingBundle — what generators receive
+# ---------------------------------------------------------------------------
+
+@dataclass
+class GroundingBundle:
+    """A time-windowed snapshot generators use to ground their output.
+
+    Generators MUST NOT emit claims that don't appear (verbatim) in
+    `allowed_claim_texts` or that mention features not in `features`.
+    """
+    venture: str
+    built_at: str
+    window_days: int
+    events: List[GroundedEvent] = field(default_factory=list)
+    allowed_claim_texts: frozenset = field(default_factory=frozenset)
+    features: frozenset = field(default_factory=frozenset)
+
+    def public_events(self) -> List[GroundedEvent]:
+        return [e for e in self.events if e.visibility == Visibility.PUBLIC]
+
+
+# ---------------------------------------------------------------------------
+# Bundle construction
+# ---------------------------------------------------------------------------
+
+def load_feature_whitelist() -> frozenset:
+    """Load the shipped-feature whitelist from features.json.
+
+    If the file is missing, return an empty frozenset. Generators MUST
+    fail-closed on empty whitelist — i.e., refuse to name any feature —
+    rather than fall through to prompt-level trust.
+    """
+    if not FEATURES_FILE.exists():
+        logger.info(
+            "feature whitelist not found at %s — generators will fail-closed on feature claims",
+            FEATURES_FILE,
+        )
+        return frozenset()
+    try:
+        data = json.loads(FEATURES_FILE.read_text())
+        feats = data.get("features") if isinstance(data, dict) else data
+        if isinstance(feats, list):
+            return frozenset(str(f).strip() for f in feats if f)
+    except Exception as e:
+        logger.warning("feature whitelist load failed: %s", e)
+    return frozenset()
+
+
+def build_allowed_claim_set(events: List[GroundedEvent]) -> frozenset:
+    """Extract the set of exact claim texts a generator may use verbatim.
+
+    Returns a frozenset so downstream code can do O(1) membership checks.
+    """
+    texts: List[str] = []
+    for ev in events:
+        for claim in ev.claims:
+            if claim.visibility == Visibility.PUBLIC and not claim.validate():
+                texts.append(claim.text.strip())
+    return frozenset(t for t in texts if t)
+
+
+def fetch_grounding_bundle(
+    venture: str = "delimit",
+    days: int = 7,
+    include_internal: bool = False,
+) -> GroundingBundle:
+    """Primary entrypoint generators call.
+
+    Week 2 default: 7-day window, public-only events. Tighter than the
+    30-day grounding index so generators see recent-and-relevant, not
+    the full history.
+    """
+    idx = build_grounding_index(venture=venture, days=days)
+    events = idx.events if include_internal else idx.public_events()
+    return GroundingBundle(
+        venture=venture,
+        built_at=idx.built_at,
+        window_days=days,
+        events=list(events),
+        allowed_claim_texts=build_allowed_claim_set(events),
+        features=load_feature_whitelist(),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Gate functions (generators call these before emitting)
+# ---------------------------------------------------------------------------
+
+# Very common English verbs that often anchor feature claims in social copy.
+# Used as a lightweight trigger for the unreleased-feature scan. Not a full
+# NLP pipeline — just enough to flag "delimit does X" / "the CLI does Y"
+# constructions for review.
+_FEATURE_CLAIM_TRIGGERS = [
+    r"\bdelimit\s+\w+s\b",          # "delimit detects", "delimit signs"
+    r"\b(?:our|the)\s+(?:cli|action|mcp|server)\s+\w+s\b",
+    r"\b(?:we|delimit)\s+(?:built|ship|shipped|support|supports|have|has)\b",
+    r"\b(?:new|latest)\s+feature\b",
+    r"\b(?:supports|offers|provides)\s+[a-z]",
+]
+
+_TRIGGER_RE = re.compile("|".join(_FEATURE_CLAIM_TRIGGERS), re.IGNORECASE)
+
+
+def unreleased_feature_detector(
+    text: str,
+    features: Optional[frozenset] = None,
+) -> Dict[str, Any]:
+    """Scan generated text for feature claims NOT in the shipped whitelist.
+
+    Returns a dict with:
+      - status: "clean" | "flagged"
+      - triggers: list of regex matches indicating claim-like language
+      - unknown_features_mentioned: list of feature-like substrings that
+        look specific enough to be a claim but don't match any entry in
+        `features`.
+
+    Generators MUST fail-closed on status == "flagged" when the whitelist
+    is loaded (empty whitelist → fail-closed by default).
+    """
+    feats = features if features is not None else load_feature_whitelist()
+    triggers = _TRIGGER_RE.findall(text or "")
+
+    # Word-level scan for "delimit-sounding" specific feature names.
+    # A "specific feature claim" looks like an identifier or a compound
+    # noun that doesn't appear in the features list and is adjacent to a
+    # trigger phrase.
+    unknown_specifics: List[str] = []
+    if triggers:
+        # Pull noun-like tokens near triggers. Low recall, intentionally
+        # conservative. Week 3 upgrade: proper NER.
+        sentences = re.split(r"[.!?]\s+", text or "")
+        for sentence in sentences:
+            if not _TRIGGER_RE.search(sentence):
+                continue
+            for token in re.findall(r"\b([a-z][a-z0-9-]{3,})\b", sentence.lower()):
+                # Only flag tokens that look domain-specific (mixed case markers,
+                # hyphens, or longer-than-common-English). For v1 we skip this
+                # lookup entirely and rely on the whitelist match at caller.
+                if token in feats:
+                    continue
+                # Don't spam: only flag once per token per text.
+                if token not in unknown_specifics and len(token) > 8 and "-" in token:
+                    unknown_specifics.append(token)
+
+    # If we have no whitelist and triggers fired, flag regardless — the
+    # generator has no basis to claim any feature.
+    if triggers and not feats:
+        return {
+            "status": "flagged",
+            "reason": "feature-claim triggers present but feature whitelist is empty",
+            "triggers": triggers[:5],
+            "unknown_features_mentioned": [],
+        }
+
+    if triggers and unknown_specifics:
+        return {
+            "status": "flagged",
+            "reason": f"{len(unknown_specifics)} unknown-feature-looking tokens near claim triggers",
+            "triggers": triggers[:5],
+            "unknown_features_mentioned": unknown_specifics[:10],
+        }
+
+    return {
+        "status": "clean",
+        "triggers": triggers[:5],
+        "unknown_features_mentioned": [],
+    }
+
+
+def score_draft_grounding(
+    text: str,
+    bundle: GroundingBundle,
+    threshold: float = 0.85,
+) -> Dict[str, Any]:
+    """Score how much of a draft is covered by the allowed-claim set.
+
+    v1 algorithm (simple):
+      - Score = proportion of sentences in `text` that either (a) contain
+        at least one verbatim allowed-claim text, or (b) contain no
+        specific feature claim at all (safe conversational filler).
+      - If sentence contains a trigger AND no allowed-claim match,
+        the sentence counts as UNGROUNDED.
+
+    Returns:
+      {
+        "score": 0.0-1.0,
+        "threshold": threshold,
+        "passed": bool,
+        "sentence_count": int,
+        "ungrounded_sentences": List[str]  (first 3 for debug)
+      }
+
+    Week 3 upgrade: proper claim classifier + paraphrase detection.
+    """
+    if not text or not text.strip():
+        return {"score": 1.0, "threshold": threshold, "passed": True, "sentence_count": 0, "ungrounded_sentences": []}
+
+    sentences = [s.strip() for s in re.split(r"(?<=[.!?])\s+", text) if s.strip()]
+    if not sentences:
+        return {"score": 1.0, "threshold": threshold, "passed": True, "sentence_count": 0, "ungrounded_sentences": []}
+
+    ungrounded: List[str] = []
+    grounded_count = 0
+    allowed = bundle.allowed_claim_texts
+
+    for sent in sentences:
+        has_trigger = bool(_TRIGGER_RE.search(sent))
+        if not has_trigger:
+            # No specific feature claim being made → safe.
+            grounded_count += 1
+            continue
+        # Claim-like sentence: require a verbatim match in allowed set.
+        matched = any(allowed_text in sent for allowed_text in allowed)
+        if matched:
+            grounded_count += 1
+        else:
+            ungrounded.append(sent[:120])
+
+    score = grounded_count / len(sentences)
+    return {
+        "score": round(score, 3),
+        "threshold": threshold,
+        "passed": score >= threshold,
+        "sentence_count": len(sentences),
+        "grounded_count": grounded_count,
+        "ungrounded_sentences": ungrounded[:3],
+    }

package/gateway/ai/content_grounding/features.py

@@ -0,0 +1,218 @@
+"""
+Feature whitelist builder for the content grounding layer (LED-1084 Week 2).
+
+Builds `~/.delimit/content/grounding/features.json` from two sources of truth:
+  1. MCP tool registry  — every `@mcp.tool()` decorator in ai/server.py is
+                           a shipped capability the drafter may reference.
+  2. CLI subcommands    — parsed from `delimit --help` output (bin/delimit-cli.js
+                           `program.command(...)` entries) when the CLI binary
+                           is on PATH. Best-effort; we fall back to just the
+                           MCP tool set if the CLI parse fails.
+
+The grounding gate in ai/content_grounding/consume.py calls
+`load_feature_whitelist()` which reads this file. An empty whitelist
+causes the gate to flag ANY feature-claim language, so populating this
+file is a direct prerequisite for meaningful gate scores.
+
+Running this builder is idempotent and safe — it overwrites `features.json`
+with the fresh set, keeps the schema stable, and records `built_at` +
+`source_counts` for audit.
+
+Usage:
+    python -m ai.content_grounding.features build
+    # or programmatically:
+    from ai.content_grounding.features import build_and_persist_features
+    build_and_persist_features()
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set
+
+from .consume import FEATURES_FILE
+
+logger = logging.getLogger("delimit.ai.content_grounding.features")
+
+GATEWAY_ROOT = Path(os.environ.get("DELIMIT_GATEWAY_REPO", "/home/delimit/delimit-gateway"))
+NPM_BIN = Path(os.environ.get("DELIMIT_CLI_BIN", "/home/delimit/npm-delimit/bin/delimit-cli.js"))
+
+
+# ---------------------------------------------------------------------------
+# Source 1 — MCP tool registry (ai/server.py)
+# ---------------------------------------------------------------------------
+
+_MCP_TOOL_DEF_RE = re.compile(
+    r"@mcp\.tool\(\)\s*\n\s*def\s+(delimit_[a-z0-9_]+)\s*\(",
+    re.MULTILINE,
+)
+
+
+def extract_mcp_tools(server_py: Path) -> List[str]:
+    """Extract every `delimit_<name>` tool registered in server.py.
+
+    Returns sorted unique tool names. These are the shipped MCP tools the
+    drafter is allowed to name. A draft that mentions `delimit_wrap` or
+    `delimit_trust_page` passes the feature gate; one that mentions
+    `delimit_coinbase_integration` fails.
+    """
+    if not server_py.is_file():
+        logger.warning("MCP server source not found: %s", server_py)
+        return []
+    try:
+        text = server_py.read_text(errors="replace")
+    except Exception as e:
+        logger.warning("could not read MCP server source: %s", e)
+        return []
+    tools = sorted(set(_MCP_TOOL_DEF_RE.findall(text)))
+    return tools
+
+
+# ---------------------------------------------------------------------------
+# Source 2 — CLI subcommands (bin/delimit-cli.js)
+# ---------------------------------------------------------------------------
+
+# Commander.js patterns we grep for:
+#   program.command("foo")
+#   program.command('foo [bar]')
+#   .command("foo <arg>")
+# Captures just the subcommand name (first token up to space).
+_CLI_COMMAND_RE = re.compile(
+    r"""(?:^|\.)command\(\s*['"]([a-z][a-z0-9_-]*)""",
+    re.MULTILINE | re.IGNORECASE,
+)
+
+
+def extract_cli_commands(cli_path: Path) -> List[str]:
+    """Extract CLI subcommand names from bin/delimit-cli.js.
+
+    Works statically (no subprocess), so it's safe on CI runners without
+    node installed. Returns sorted unique command names.
+    """
+    if not cli_path.is_file():
+        logger.warning("delimit CLI source not found: %s", cli_path)
+        return []
+    try:
+        text = cli_path.read_text(errors="replace")
+    except Exception as e:
+        logger.warning("could not read CLI source: %s", e)
+        return []
+    commands = set()
+    for m in _CLI_COMMAND_RE.finditer(text):
+        name = m.group(1).strip().lower()
+        # Filter out obvious false matches (method names that happen to start
+        # with a word followed by '('). A real commander subcommand won't
+        # contain certain tokens.
+        if name in {"log", "error", "warn", "info", "on", "off", "then", "catch", "parse"}:
+            continue
+        commands.add(name)
+    return sorted(commands)
+
+
+# ---------------------------------------------------------------------------
+# Build + persist
+# ---------------------------------------------------------------------------
+
+def build_feature_set(
+    gateway_root: Optional[Path] = None,
+    cli_path: Optional[Path] = None,
+) -> Dict[str, Any]:
+    """Aggregate features from both sources into a dict ready to persist.
+
+    Shape:
+      {
+        "version": 1,
+        "built_at": "2026-04-24T...",
+        "features": ["delimit_lint", "delimit_wrap", "init", "scan", ...],
+        "source_counts": {"mcp_tools": N, "cli_commands": M},
+        "sources": {
+          "mcp_tools": [...],
+          "cli_commands": [...],
+        }
+      }
+    """
+    gw = gateway_root or GATEWAY_ROOT
+    cp = cli_path or NPM_BIN
+    mcp_tools = extract_mcp_tools(gw / "ai" / "server.py")
+    cli_commands = extract_cli_commands(cp)
+    features = sorted(set(mcp_tools) | set(cli_commands))
+    return {
+        "version": 1,
+        "built_at": datetime.now(timezone.utc).isoformat(),
+        "features": features,
+        "source_counts": {
+            "mcp_tools": len(mcp_tools),
+            "cli_commands": len(cli_commands),
+            "total_unique": len(features),
+        },
+        "sources": {
+            "mcp_tools": mcp_tools,
+            "cli_commands": cli_commands,
+        },
+    }
+
+
+def build_and_persist_features(
+    out_path: Optional[Path] = None,
+    gateway_root: Optional[Path] = None,
+    cli_path: Optional[Path] = None,
+) -> Path:
+    """Build the feature set and write it to features.json.
+
+    Returns the path written. Overwrites any previous file.
+    """
+    target = Path(out_path) if out_path else FEATURES_FILE
+    target.parent.mkdir(parents=True, exist_ok=True)
+    data = build_feature_set(gateway_root=gateway_root, cli_path=cli_path)
+    target.write_text(json.dumps(data, indent=2))
+    logger.info(
+        "wrote features.json: %d mcp_tools + %d cli_commands → %d unique → %s",
+        data["source_counts"]["mcp_tools"],
+        data["source_counts"]["cli_commands"],
+        data["source_counts"]["total_unique"],
+        target,
+    )
+    return target
+
+
+# ---------------------------------------------------------------------------
+# CLI entry
+# ---------------------------------------------------------------------------
+
+def _main() -> int:
+    if len(sys.argv) < 2 or sys.argv[1] not in ("build", "show"):
+        print(
+            "usage: python -m ai.content_grounding.features build | show\n"
+            "  build  rebuild features.json from MCP + CLI sources\n"
+            "  show   print current features.json stats",
+            file=sys.stderr,
+        )
+        return 2
+    if sys.argv[1] == "build":
+        out = build_and_persist_features()
+        data = json.loads(out.read_text())
+        print(f"wrote {out}")
+        print(f"  mcp_tools:      {data['source_counts']['mcp_tools']}")
+        print(f"  cli_commands:   {data['source_counts']['cli_commands']}")
+        print(f"  total unique:   {data['source_counts']['total_unique']}")
+        return 0
+    if sys.argv[1] == "show":
+        if not FEATURES_FILE.exists():
+            print(f"no features.json at {FEATURES_FILE}. run 'build' first.", file=sys.stderr)
+            return 1
+        data = json.loads(FEATURES_FILE.read_text())
+        print(f"features.json  built_at={data.get('built_at')}")
+        print(f"  total:  {data['source_counts']['total_unique']}")
+        print(f"  sample: {data['features'][:8]} ...")
+        return 0
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(_main())

package/gateway/ai/content_grounding/fixtures/fail/01_missing_evidence.json

@@ -0,0 +1,9 @@
+{
+  "event_id": "evt-ledger-LED-999",
+  "type": "feature_shipped",
+  "date": "2026-04-24T00:00:00Z",
+  "venture": "delimit",
+  "evidence_refs": [],
+  "claims": [],
+  "visibility": "internal"
+}

package/gateway/ai/content_grounding/fixtures/fail/02_unknown_evidence_prefix.json

@@ -0,0 +1,9 @@
+{
+  "event_id": "evt-ledger-LED-888",
+  "type": "feature_shipped",
+  "date": "2026-04-24T00:00:00Z",
+  "venture": "delimit",
+  "evidence_refs": ["jira:TICKET-123"],
+  "claims": [],
+  "visibility": "internal"
+}

package/gateway/ai/content_grounding/fixtures/fail/03_banned_comparative.json

@@ -0,0 +1,17 @@
+{
+  "event_id": "evt-ledger-LED-777",
+  "type": "feature_shipped",
+  "date": "2026-04-24T00:00:00Z",
+  "venture": "delimit",
+  "evidence_refs": ["LED-777"],
+  "claims": [
+    {
+      "claim_id": "CLM-LED-777-compare",
+      "type": "comparative",
+      "text": "Delimit is faster than openapi-diff in every benchmark.",
+      "evidence_refs": ["LED-777"],
+      "visibility": "public"
+    }
+  ],
+  "visibility": "public"
+}

package/gateway/ai/content_grounding/fixtures/fail/04_banned_adoption.json

@@ -0,0 +1,17 @@
+{
+  "event_id": "evt-ledger-LED-666",
+  "type": "feature_shipped",
+  "date": "2026-04-24T00:00:00Z",
+  "venture": "delimit",
+  "evidence_refs": ["LED-666"],
+  "claims": [
+    {
+      "claim_id": "CLM-LED-666-adoption",
+      "type": "adoption",
+      "text": "Used in production by 12 open-source projects.",
+      "evidence_refs": ["LED-666"],
+      "visibility": "public"
+    }
+  ],
+  "visibility": "public"
+}

package/gateway/ai/content_grounding/fixtures/fail/05_aggregate_no_numeric.json

@@ -0,0 +1,17 @@
+{
+  "event_id": "evt-ledger-LED-555",
+  "type": "release",
+  "date": "2026-04-24T00:00:00Z",
+  "venture": "delimit",
+  "evidence_refs": ["LED-555"],
+  "claims": [
+    {
+      "claim_id": "CLM-LED-555-tests",
+      "type": "aggregate",
+      "text": "Many tests pass.",
+      "evidence_refs": ["LED-555"],
+      "visibility": "public"
+    }
+  ],
+  "visibility": "public"
+}

package/gateway/ai/content_grounding/fixtures/fail/06_unversioned_inference_rule.json

@@ -0,0 +1,18 @@
+{
+  "event_id": "evt-ledger-LED-444",
+  "type": "feature_shipped",
+  "date": "2026-04-24T00:00:00Z",
+  "venture": "delimit",
+  "evidence_refs": ["LED-444"],
+  "claims": [
+    {
+      "claim_id": "CLM-LED-444-inferred",
+      "type": "feature",
+      "text": "Delimit handles all the edge cases.",
+      "evidence_refs": ["LED-444"],
+      "visibility": "internal",
+      "inference_rule": "generalization"
+    }
+  ],
+  "visibility": "internal"
+}

package/gateway/ai/content_grounding/fixtures/pass/01_feature_shipped.json

@@ -0,0 +1,18 @@
+{
+  "event_id": "evt-ledger-LED-1010",
+  "type": "feature_shipped",
+  "date": "2026-04-24T00:00:00Z",
+  "venture": "delimit",
+  "evidence_refs": ["LED-1010", "git:f6597185b72d"],
+  "claims": [
+    {
+      "claim_id": "CLM-LED-1010-title",
+      "type": "feature",
+      "text": "Tailwind-aware extract_tokens emits framework default spacing, typography, and breakpoint scales.",
+      "evidence_refs": ["LED-1010", "git:f6597185b72d"],
+      "visibility": "public"
+    }
+  ],
+  "visibility": "internal",
+  "source": "ledger:feature"
+}

package/gateway/ai/content_grounding/fixtures/pass/02_aggregate_claim.json

@@ -0,0 +1,23 @@
+{
+  "event_id": "evt-ledger-LED-release-v4.3.4",
+  "type": "release",
+  "date": "2026-04-24T01:39:29Z",
+  "venture": "delimit",
+  "evidence_refs": ["LED-1079", "git:67578dc54e77"],
+  "claims": [
+    {
+      "claim_id": "CLM-release-v4.3.4-test-count",
+      "type": "aggregate",
+      "text": "165 tests pass on delimit-cli v4.3.4.",
+      "evidence_refs": ["git:67578dc54e77"],
+      "visibility": "public",
+      "numeric_evidence": {
+        "value": 165,
+        "unit": "tests",
+        "commit_sha": "67578dc54e77b7dfbed0f6a9ce18baa4ae695c91"
+      }
+    }
+  ],
+  "visibility": "public",
+  "source": "ledger:release"
+}

package/gateway/ai/content_grounding/fixtures/pass/03_attestation.json

@@ -0,0 +1,16 @@
+{
+  "event_id": "evt-att-att_a05050eb8e13277e",
+  "type": "attestation",
+  "date": "2026-04-23T23:52:51.823Z",
+  "venture": "delimit",
+  "evidence_refs": ["attest:att_a05050eb8e13277e", "git:ff0c729fda7b"],
+  "claims": [],
+  "visibility": "public",
+  "source": "attestation",
+  "raw": {
+    "attestation_id": "att_a05050eb8e13277e",
+    "kind": "merge_attestation",
+    "wrapped_exit": 0,
+    "signature_alg": "HMAC-SHA256"
+  }
+}