delimit-cli 4.1.53 → 4.2.0

package/gateway/ai/reddit_proxy.py

@@ -9,24 +9,35 @@ from typing import Any, Dict, List, Optional
 logger = logging.getLogger("delimit.ai.reddit_proxy")
 
 def _get_proxy_config() -> Dict[str, str]:
-    """Load proxy config from private secrets or environment."""
-    config = {"proxy_url": ""}
-    
-    # 1. Check environment variable
+    """Load proxy config from private secrets or environment.
+
+    Returns {proxy_url, token}. The server-side proxy requires a bearer
+    token (LED-988 follow-up) — clients without a token still populate
+    proxy_url but will fail auth at the server unless the server is run
+    without a token (not recommended).
+    """
+    config = {"proxy_url": "", "token": ""}
+
+    # 1. Environment variables
     env_url = os.environ.get("DELIMIT_REDDIT_PROXY")
+    env_token = os.environ.get("DELIMIT_REDDIT_PROXY_TOKEN")
     if env_url:
         config["proxy_url"] = env_url
+    if env_token:
+        config["token"] = env_token
+    if config["proxy_url"]:
         return config
 
-    # 2. Check private secrets file
+    # 2. Secrets file
     secrets_path = Path.home() / ".delimit" / "secrets" / "reddit-proxy.json"
     if secrets_path.exists():
         try:
             secrets = json.loads(secrets_path.read_text())
-            config["proxy_url"] = secrets.get("proxy_url", "")
+            config["proxy_url"] = secrets.get("proxy_url", "") or config["proxy_url"]
+            config["token"] = secrets.get("token", "") or config["token"]
         except Exception as e:
             logger.debug(f"Failed to load reddit-proxy secrets: {e}")
-            
+
     return config
 
 def fetch_subreddit(subreddit: str, sort: str = "new", limit: int = 10) -> List[Dict[str, Any]]:
@@ -42,7 +53,11 @@ def fetch_subreddit(subreddit: str, sort: str = "new", limit: int = 10) -> List[
     if proxy_url:
         try:
             fetch_url = f"{proxy_url}?url={urllib.parse.quote(reddit_url, safe='')}"
-            req = urllib.request.Request(fetch_url, headers={"User-Agent": "Delimit/1.0"})
+            headers = {"User-Agent": "Delimit/1.0"}
+            token = proxy_cfg.get("token", "")
+            if token:
+                headers["Authorization"] = f"Bearer {token}"
+            req = urllib.request.Request(fetch_url, headers=headers)
             with urllib.request.urlopen(req, timeout=10) as resp:
                 body = json.loads(resp.read().decode())
                 children = body.get("data", {}).get("children", [])
@@ -84,7 +99,11 @@ def fetch_thread(thread_id: str) -> Optional[Dict[str, Any]]:
     if proxy_url:
         try:
             fetch_url = f"{proxy_url}?url={urllib.parse.quote(reddit_url, safe='')}"
-            req = urllib.request.Request(fetch_url, headers={"User-Agent": "Delimit/1.0"})
+            headers = {"User-Agent": "Delimit/1.0"}
+            token = proxy_cfg.get("token", "")
+            if token:
+                headers["Authorization"] = f"Bearer {token}"
+            req = urllib.request.Request(fetch_url, headers=headers)
             with urllib.request.urlopen(req, timeout=10) as resp:
                 data = json.loads(resp.read().decode())
                 if isinstance(data, list) and len(data) > 0:

package/gateway/ai/sensing/__init__.py

@@ -0,0 +1,35 @@
+"""Signal sensing layer (LED-877).
+
+Physically separates observational signals from the ledger. Signals are a
+deliberation corpus, not a task queue — they must never be pulled by
+build_loop as work. Import from ai.sensing.signal_store for ingest/query.
+"""
+
+from ai.sensing.schema import Signal, ValidationError, normalize_url, fingerprint_of
+from ai.sensing.signal_store import (
+    ingest,
+    query,
+    dedup_check,
+    age_out_to_warm,
+    freeze_cold,
+    promote_to_ledger,
+    SIGNALS_DIR,
+    HOT_WINDOW_DAYS,
+    WARM_WINDOW_DAYS,
+)
+
+__all__ = [
+    "Signal",
+    "ValidationError",
+    "normalize_url",
+    "fingerprint_of",
+    "ingest",
+    "query",
+    "dedup_check",
+    "age_out_to_warm",
+    "freeze_cold",
+    "promote_to_ledger",
+    "SIGNALS_DIR",
+    "HOT_WINDOW_DAYS",
+    "WARM_WINDOW_DAYS",
+]

package/gateway/ai/sensing/schema.py

@@ -0,0 +1,107 @@
+"""Signal schema + validation (LED-877).
+
+A signal is an observation, not a commitment. Schema enforces enough metadata
+for deliberation to work with, rejects empty-identity rows at ingest (killing
+the LED-876 ghost-engage-task class of bug at its source).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import dataclass, field, asdict
+from typing import Any, Dict, List, Optional
+from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode
+
+
+class ValidationError(ValueError):
+    """Raised when a signal fails schema validation on ingest."""
+
+
+_UTM_RE = re.compile(r"^utm_")
+
+
+def normalize_url(url: str) -> str:
+    """Canonicalize URL: strip utm_* query params, fragment, trailing slash."""
+    if not url:
+        return ""
+    try:
+        p = urlparse(url.strip())
+    except Exception:
+        return url.strip()
+    if not p.scheme:
+        return url.strip()
+    query = [(k, v) for k, v in parse_qsl(p.query) if not _UTM_RE.match(k)]
+    path = p.path.rstrip("/") or "/"
+    cleaned = urlunparse(
+        (p.scheme.lower(), p.netloc.lower(), path, "", urlencode(query), "")
+    )
+    return cleaned
+
+
+def fingerprint_of(platform: str, canonical_url: str, author: str) -> str:
+    """Stable dedup key for a signal."""
+    raw = f"{(platform or '').lower()}|{normalize_url(canonical_url)}|{(author or '').lower()}"
+    return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16]
+
+
+@dataclass
+class Signal:
+    """A sensed observation from an external platform.
+
+    Mandatory: canonical_url AND (author OR content_snippet).
+    Anything weaker than that is rejected at ingest because deliberation
+    cannot draw useful conclusions from a row with no identity.
+    """
+
+    fingerprint: str
+    platform: str
+    canonical_url: str
+    author: str = ""
+    author_handle: str = ""
+    content_snippet: str = ""
+    posted_at: str = ""
+    ingested_at: str = ""
+    classification: str = "signal"
+    relevance_score: float = 0.0
+    themes: List[str] = field(default_factory=list)
+    raw_ref: str = ""
+    id: str = ""
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+
+def validate_and_normalize(raw: Dict[str, Any]) -> Signal:
+    """Convert a raw target dict from social_target.py into a validated Signal.
+
+    Raises ValidationError on missing mandatory fields so bugs surface loudly
+    at ingest rather than producing empty-identity rows that pollute the
+    corpus (the LED-876 failure mode).
+    """
+    platform = (raw.get("platform") or "").strip()
+    canonical_url = normalize_url(raw.get("canonical_url") or raw.get("url") or "")
+    author = (raw.get("author") or "").strip()
+    content_snippet = (raw.get("content_snippet") or raw.get("title") or "").strip()[:500]
+
+    if not canonical_url:
+        raise ValidationError("canonical_url is required")
+    if not author and not content_snippet:
+        raise ValidationError("at least one of author or content_snippet is required")
+    if not platform:
+        raise ValidationError("platform is required")
+
+    return Signal(
+        fingerprint=fingerprint_of(platform, canonical_url, author),
+        platform=platform,
+        canonical_url=canonical_url,
+        author=author,
+        author_handle=(raw.get("author_handle") or "").strip(),
+        content_snippet=content_snippet,
+        posted_at=(raw.get("posted_at") or "").strip(),
+        ingested_at="",  # filled by signal_store.ingest
+        classification=(raw.get("classification") or "signal").strip(),
+        relevance_score=float(raw.get("relevance_score") or 0.0),
+        themes=list(raw.get("themes") or []),
+        raw_ref=(raw.get("raw_ref") or raw.get("source_url") or canonical_url).strip(),
+    )

package/gateway/ai/sensing/signal_store.py

@@ -0,0 +1,348 @@
+"""Signal store (LED-877).
+
+Physically separated from the ledger. Daily shards, append-only. Reuses
+~/.delimit/intel/ as the parent directory so intel_* tooling can already
+query it via intel_dataset_list.
+
+Consumers: delimit sense CLI, delimit_signals_query MCP tool (future).
+NOT a consumer: build_loop, agent_dispatch, ledger_manager.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+import uuid
+from collections import Counter, defaultdict
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional
+
+from ai.sensing.schema import (
+    Signal,
+    ValidationError,
+    fingerprint_of,
+    normalize_url,
+    validate_and_normalize,
+)
+
+
+class StorageError(RuntimeError):
+    """Raised when signal persistence fails."""
+
+
+INTEL_DIR = Path.home() / ".delimit" / "intel"
+SIGNALS_DIR = INTEL_DIR / "signals"
+ARCHIVE_DIR = SIGNALS_DIR / "archive"
+DEDUP_INDEX_PATH = SIGNALS_DIR / "_dedup_index.json"
+
+HOT_WINDOW_DAYS = 7
+WARM_WINDOW_DAYS = 30
+MAX_SIGNALS_PER_AUTHOR_PER_DAY = 3
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _today_shard_path(when: Optional[datetime] = None) -> Path:
+    when = when or _now()
+    return SIGNALS_DIR / f"{when.date().isoformat()}.jsonl"
+
+
+def _ensure_dirs() -> None:
+    SIGNALS_DIR.mkdir(parents=True, exist_ok=True)
+    ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def _load_dedup_index() -> Dict[str, Dict[str, Any]]:
+    if not DEDUP_INDEX_PATH.exists():
+        return {}
+    try:
+        return json.loads(DEDUP_INDEX_PATH.read_text())
+    except (json.JSONDecodeError, OSError):
+        return {}
+
+
+def _save_dedup_index(index: Dict[str, Dict[str, Any]]) -> None:
+    _ensure_dirs()
+    tmp = DEDUP_INDEX_PATH.with_suffix(".tmp")
+    tmp.write_text(json.dumps(index))
+    tmp.replace(DEDUP_INDEX_PATH)
+
+
+def dedup_check(fingerprint: str, window_days: int = WARM_WINDOW_DAYS) -> bool:
+    """Return True if a signal with this fingerprint was ingested within window_days."""
+    if not fingerprint:
+        return False
+    index = _load_dedup_index()
+    entry = index.get(fingerprint)
+    if not entry:
+        return False
+    try:
+        ingested = datetime.fromisoformat(entry.get("ingested_at", ""))
+    except ValueError:
+        return False
+    cutoff = _now() - timedelta(days=window_days)
+    return ingested >= cutoff
+
+
+def _author_count_today(author: str) -> int:
+    if not author:
+        return 0
+    path = _today_shard_path()
+    if not path.exists():
+        return 0
+    count = 0
+    needle = author.lower()
+    try:
+        for line in path.read_text().splitlines():
+            if not line.strip():
+                continue
+            try:
+                row = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if (row.get("author") or "").lower() == needle:
+                count += 1
+    except OSError:
+        return 0
+    return count
+
+
+def ingest(raw: Dict[str, Any]) -> Dict[str, Any]:
+    """Ingest a raw target dict as a validated Signal.
+
+    Raises ValidationError on schema violation (caller decides whether to
+    log-and-skip or propagate). Returns the stored signal with assigned id
+    and ingested_at, or a dedup/rate-limit notice.
+    """
+    signal = validate_and_normalize(raw)
+
+    if dedup_check(signal.fingerprint):
+        index = _load_dedup_index()
+        entry = index.get(signal.fingerprint, {})
+        entry["hit_count"] = int(entry.get("hit_count", 1)) + 1
+        entry["last_seen_at"] = _now().isoformat()
+        index[signal.fingerprint] = entry
+        _save_dedup_index(index)
+        return {
+            "status": "deduped",
+            "fingerprint": signal.fingerprint,
+            "hit_count": entry["hit_count"],
+        }
+
+    if _author_count_today(signal.author) >= MAX_SIGNALS_PER_AUTHOR_PER_DAY:
+        return {
+            "status": "rate_limited",
+            "author": signal.author,
+            "limit": MAX_SIGNALS_PER_AUTHOR_PER_DAY,
+        }
+
+    _ensure_dirs()
+    now = _now()
+    signal.ingested_at = now.isoformat()
+    signal.id = f"SIG-{uuid.uuid4().hex[:10].upper()}"
+
+    shard = _today_shard_path(now)
+    try:
+        with shard.open("a") as f:
+            f.write(json.dumps(signal.to_dict()) + "\n")
+    except OSError as exc:
+        raise StorageError(f"failed to write signal shard {shard}: {exc}") from exc
+
+    index = _load_dedup_index()
+    index[signal.fingerprint] = {
+        "id": signal.id,
+        "ingested_at": signal.ingested_at,
+        "hit_count": 1,
+        "shard": shard.name,
+    }
+    _save_dedup_index(index)
+
+    return {"status": "ingested", "signal": signal.to_dict(), "shard": shard.name}
+
+
+def _iter_shards(since_days: int = HOT_WINDOW_DAYS) -> Iterable[Path]:
+    if not SIGNALS_DIR.exists():
+        return []
+    cutoff = (_now() - timedelta(days=since_days)).date()
+    paths = []
+    for path in SIGNALS_DIR.glob("*.jsonl"):
+        if path.name.startswith("_"):
+            continue
+        try:
+            shard_date = datetime.fromisoformat(path.stem).date()
+        except ValueError:
+            continue
+        if shard_date >= cutoff:
+            paths.append(path)
+    return sorted(paths, reverse=True)
+
+
+def query(
+    since_days: int = 1,
+    platform: str = "",
+    limit: int = 50,
+) -> List[Dict[str, Any]]:
+    """Return signals from the hot window, newest first.
+
+    since_days=1 returns the last 24h of signals (the default `delimit sense`
+    view). platform filters to a specific source; empty = all.
+    """
+    rows: List[Dict[str, Any]] = []
+    want_platform = (platform or "").strip().lower()
+    for shard in _iter_shards(since_days):
+        try:
+            for line in shard.read_text().splitlines():
+                if not line.strip():
+                    continue
+                try:
+                    row = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                if want_platform and (row.get("platform") or "").lower() != want_platform:
+                    continue
+                rows.append(row)
+                if len(rows) >= limit:
+                    return rows
+        except OSError:
+            continue
+    return rows
+
+
+def age_out_to_warm(days: int = HOT_WINDOW_DAYS) -> int:
+    """No-op placeholder: hot/warm separation is a query boundary, not a move.
+
+    We keep all shards in SIGNALS_DIR and rely on query()'s since_days filter
+    to enforce the hot window. Returns the count of shards older than `days`
+    for reporting.
+    """
+    if not SIGNALS_DIR.exists():
+        return 0
+    cutoff = (_now() - timedelta(days=days)).date()
+    old = 0
+    for path in SIGNALS_DIR.glob("*.jsonl"):
+        if path.name.startswith("_"):
+            continue
+        try:
+            shard_date = datetime.fromisoformat(path.stem).date()
+        except ValueError:
+            continue
+        if shard_date < cutoff:
+            old += 1
+    return old
+
+
+def freeze_cold(month: str) -> str:
+    """Move all shards whose date starts with `month` (YYYY-MM) into ARCHIVE_DIR/{month}.jsonl.
+
+    Returns the archive path. Safe to run repeatedly; reruns append.
+    """
+    if not month or len(month) != 7 or month[4] != "-":
+        raise ValidationError(f"month must be YYYY-MM, got {month!r}")
+    _ensure_dirs()
+    archive_path = ARCHIVE_DIR / f"{month}.jsonl"
+    moved = 0
+    with archive_path.open("a") as out:
+        for path in sorted(SIGNALS_DIR.glob(f"{month}-*.jsonl")):
+            try:
+                out.write(path.read_text())
+            except OSError:
+                continue
+            try:
+                path.unlink()
+                moved += 1
+            except OSError:
+                pass
+    return str(archive_path)
+
+
+def promote_to_ledger(
+    signal_id: str,
+    ledger: str = "ops",
+    priority: str = "P2",
+    extra_tags: Optional[List[str]] = None,
+) -> Dict[str, Any]:
+    """Explicit founder-initiated promotion of a signal to a ledger item.
+
+    This is the ONLY path from intel → ledger. Writes via ledger_manager.add_item
+    with promoted_by set so the guard accepts it.
+    """
+    signal = _find_signal(signal_id)
+    if not signal:
+        raise ValidationError(f"signal {signal_id} not found in hot shards")
+
+    from ai.ledger_manager import add_item
+
+    title = f"[{(signal.get('platform') or 'signal').upper()}] Promoted: {signal.get('author') or signal.get('canonical_url')}"
+    description = (
+        f"Promoted from intel store (signal {signal_id}).\n"
+        f"URL: {signal.get('canonical_url', '')}\n"
+        f"Author: {signal.get('author', '')}\n"
+        f"Snippet: {(signal.get('content_snippet') or '')[:400]}\n"
+        f"Posted: {signal.get('posted_at', '')}\n"
+        f"Fingerprint: {signal.get('fingerprint', '')}"
+    )
+    tags = ["promoted-signal", signal.get("platform", "")]
+    if extra_tags:
+        tags.extend(extra_tags)
+
+    # Guard checks source=='promoted_signal' + promoted_by set, so bypass the
+    # social_scan rejection.
+    os.environ.setdefault("_DELIMIT_SIGNAL_PROMOTED_BY", "founder")
+    try:
+        result = add_item(
+            title=title,
+            ledger=ledger,
+            type="task",
+            priority=priority,
+            description=description,
+            source=f"promoted_signal:{signal_id}",
+            tags=tags,
+            context=f"Promoted from signal {signal_id} for strategic action.",
+        )
+    finally:
+        os.environ.pop("_DELIMIT_SIGNAL_PROMOTED_BY", None)
+    return result
+
+
+def _find_signal(signal_id: str) -> Optional[Dict[str, Any]]:
+    if not signal_id:
+        return None
+    for shard in _iter_shards(since_days=WARM_WINDOW_DAYS):
+        try:
+            for line in shard.read_text().splitlines():
+                if not line.strip():
+                    continue
+                try:
+                    row = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                if row.get("id") == signal_id:
+                    return row
+        except OSError:
+            continue
+    return None
+
+
+def digest(since_days: int = HOT_WINDOW_DAYS, top_n: int = 20) -> Dict[str, Any]:
+    """Cluster recent signals by platform + top authors + theme counters."""
+    rows = query(since_days=since_days, limit=1000)
+    by_platform: Counter[str] = Counter()
+    by_author: Counter[str] = Counter()
+    by_theme: Counter[str] = Counter()
+    for row in rows:
+        by_platform[row.get("platform", "?")] += 1
+        by_author[row.get("author", "?")] += 1
+        for theme in row.get("themes") or []:
+            by_theme[theme] += 1
+    return {
+        "window_days": since_days,
+        "total_signals": len(rows),
+        "top_platforms": by_platform.most_common(10),
+        "top_authors": by_author.most_common(top_n),
+        "top_themes": by_theme.most_common(top_n),
+        "sample": rows[:5],
+    }