@heretek-ai/epistemic-swarm 0.7.25 → 1.0.2

package/skills/darkharvest/scripts/harvest.py

@@ -1,289 +0,0 @@
-#!/usr/bin/env python3
-"""
-Darkharvest fetch helper: clone-to-temp + allowlisted scan + SHA-256 cache.
-
-Read-only. Never installs dependencies. Never writes outside .research/
-(except the temp clone, which is dropped). Enforces per-repo caps so
-`--max-repos 10 --depth 3` style defaults cannot OOM the host.
-
-Usage:
-  python3 skills/darkharvest/scripts/harvest.py --repo <url> [--show-context]
-  python3 skills/darkharvest/scripts/harvest.py --repo <url> --export <path>
-
-Closed / non-cloneable targets: prints a metadata-only stub with a
-[NEGATIVE_KNOWLEDGE] gap note instead of failing.
-"""
-
-import argparse
-import json
-import os
-import re
-import shutil
-import subprocess
-import sys
-import tempfile
-from datetime import datetime, timezone
-from pathlib import Path
-
-PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent.parent
-if str(PROJECT_ROOT) not in sys.path:
-    sys.path.insert(0, str(PROJECT_ROOT))
-
-from skills.research_cache.hasher import SourceHasher  # noqa: E402
-
-SKIP_DIRS = {
-    ".git",
-    "node_modules",
-    "__pycache__",
-    ".venv",
-    "venv",
-    "dist",
-    "build",
-    "target",
-    ".next",
-    ".research",
-    "coverage",
-}
-MANIFEST_FILES = [
-    "package.json",
-    "pyproject.toml",
-    "Cargo.toml",
-    "go.mod",
-    "requirements.txt",
-    "LICENSE",
-    "LICENSE.md",
-    "README.md",
-]
-TEXT_EXTS = {".md", ".json", ".toml", ".yaml", ".yml", ".txt", ".py", ".js", ".ts"}
-
-DEFAULT_MAX_FILES = 120
-DEFAULT_MAX_BYTES = 400_000
-DEFAULT_TIMEOUT = 120
-DEFAULT_BASE_DIR = ".research"
-MAX_SCAN_DEPTH = 3
-MAX_SINGLE_FILE_BYTES = 100_000
-SNIPPET_CHARS = 4000
-
-SPDX_APACHE2 = "Apache-2.0"
-PERMISSIVE_SPDX = frozenset({"MIT", SPDX_APACHE2, "BSD-3-Clause", "ISC"})
-COPYLEFT_SPDX = frozenset({"GPL", "AGPL"})
-
-SPDX_MAP = {
-    "mit": "MIT",
-    "apache-2.0": SPDX_APACHE2,
-    "apache 2.0": SPDX_APACHE2,
-    "bsd-3-clause": "BSD-3-Clause",
-    "bsd 3-clause": "BSD-3-Clause",
-    "isc": "ISC",
-    "gpl": "GPL",
-    "agpl": "AGPL",
-    "mpl": "MPL",
-    "lgpl": "LGPL",
-}
-
-
-def sh(cmd, cwd=None, timeout=DEFAULT_TIMEOUT):
-    try:
-        r = subprocess.run(
-            cmd,
-            capture_output=True,
-            text=True,
-            cwd=str(cwd or PROJECT_ROOT),
-            timeout=timeout,
-        )
-        return r.returncode, (r.stdout or "").strip()
-    except Exception as exc:  # noqa: BLE001
-        return 1, f"error: {exc}"
-
-
-def detect_license(text):
-    low = (text or "").lower()
-    for key, spdx in SPDX_MAP.items():
-        if key in low:
-            return spdx
-    return "UNKNOWN"
-
-
-def scan_tree(root, max_files=DEFAULT_MAX_FILES, max_bytes=DEFAULT_MAX_BYTES):
-    root = Path(root)
-    state = _ScanState()
-    for dirpath, dirnames, filenames in os.walk(root):
-        dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
-        rel = Path(dirpath).relative_to(root)
-        if (len(rel.parts) if str(rel) != "." else 0) > MAX_SCAN_DEPTH:
-            dirnames[:] = []
-            continue
-        prefix = "" if str(rel) == "." else str(rel) + os.sep
-        for f in sorted(filenames):
-            if state.capped(max_files, max_bytes):
-                return state.finish(truncated=True)
-            _scan_file(dirpath, prefix + f, state)
-    return state.finish(truncated=False)
-
-
-class _ScanState:
-    """Mutable accumulator for one scan_tree pass (keeps helpers small)."""
-
-    def __init__(self):
-        self.entries = []
-        self.total_bytes = 0
-        self.manifests = {}
-        self.license_text = ""
-        self.readme_head = ""
-
-    def capped(self, max_files, max_bytes):
-        return len(self.entries) >= max_files or self.total_bytes >= max_bytes
-
-    def finish(self, truncated):
-        return (
-            self.entries,
-            self.manifests,
-            self.license_text,
-            self.readme_head,
-            truncated,
-        )
-
-
-def _scan_file(dirpath, rel_entry, state):
-    filename = os.path.basename(rel_entry)
-    try:
-        size = (Path(dirpath) / filename).stat().st_size
-    except OSError:
-        return
-    if size > MAX_SINGLE_FILE_BYTES:
-        return
-    state.entries.append(rel_entry)
-    if filename in MANIFEST_FILES or Path(filename).suffix in TEXT_EXTS:
-        _capture_text(dirpath, filename, state)
-
-
-def _capture_text(dirpath, filename, state):
-    try:
-        content = (Path(dirpath) / filename).read_text(
-            encoding="utf-8", errors="replace"
-        )
-    except OSError:
-        return
-    state.total_bytes += len(content.encode("utf-8", errors="replace"))
-    snippet = content[:SNIPPET_CHARS]
-    if filename in MANIFEST_FILES and filename not in state.manifests:
-        state.manifests[filename] = snippet
-    lowered = filename.lower()
-    if lowered.startswith("license") and not state.license_text:
-        state.license_text = snippet
-    if lowered == "readme.md" and not state.readme_head:
-        state.readme_head = snippet
-
-
-def harvest_repo(
-    url,
-    max_files=DEFAULT_MAX_FILES,
-    max_bytes=DEFAULT_MAX_BYTES,
-    timeout=DEFAULT_TIMEOUT,
-    base_dir=DEFAULT_BASE_DIR,
-):
-    started = datetime.now(timezone.utc).isoformat()
-    tmp = tempfile.mkdtemp(prefix="darkharvest-")
-    try:
-        code, out = sh(["git", "clone", "--depth", "1", url, tmp], timeout=timeout)
-        if code != 0:
-            return {
-                "url": url,
-                "status": "metadata-only",
-                "generated_at": started,
-                "negative_knowledge": (
-                    f"[NEGATIVE_KNOWLEDGE: clone failed for {url}; "
-                    f"used metadata-only row. Detail: {out[:200]}]"
-                ),
-            }
-        entries, manifests, license_text, readme_head, truncated = scan_tree(
-            Path(tmp), max_files=max_files, max_bytes=max_bytes
-        )
-        spdx = detect_license(license_text + manifests.get("package.json", ""))
-        harvestable = (
-            "depend-or-vendor" if spdx in PERMISSIVE_SPDX else "clean-room-rebuild-only"
-        )
-        hasher = SourceHasher(Path(base_dir))
-        cache_blob = (
-            f"# {url}\n\nSPDX: {spdx}\n\n"
-            f"## README head\n{readme_head[:2000]}\n\n"
-            f"## Tree sample ({len(entries)} entries)\n" + "\n".join(entries[:60])
-        )
-        source_hash = hasher.store_source(url, cache_blob, f"Darkharvest {url}")
-        warnings = []
-        if spdx in COPYLEFT_SPDX:
-            warnings.append(
-                "⚠️ LICENSE: strong copyleft — spec rebuild only, never vendor"
-            )
-        elif spdx == "UNKNOWN":
-            warnings.append("⚠️ LICENSE: unknown — treat as clean-room-rebuild-only")
-        if truncated:
-            warnings.append(
-                "⚠️ SCAN: truncated by file/byte caps; treat as partial evidence"
-            )
-        return {
-            "url": url,
-            "status": "scanned",
-            "generated_at": started,
-            "spdx": spdx,
-            "harvest_policy": harvestable,
-            "tree_entries": len(entries),
-            "truncated": truncated,
-            "source_hash": source_hash,
-            "readme_head": readme_head[:1500],
-            "manifests_seen": sorted(manifests.keys()),
-            "warnings": warnings,
-        }
-    finally:
-        shutil.rmtree(tmp, ignore_errors=True)
-
-
-def main():
-    ap = argparse.ArgumentParser(description="Darkharvest competitor fetch helper")
-    ap.add_argument("--repo", required=True, help="Competitor repo URL to scan")
-    ap.add_argument("--show-context", action="store_true")
-    ap.add_argument("--export", default="")
-    ap.add_argument("--max-files", type=int, default=DEFAULT_MAX_FILES)
-    ap.add_argument("--max-bytes", type=int, default=DEFAULT_MAX_BYTES)
-    ap.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT)
-    ap.add_argument("--dir", default=DEFAULT_BASE_DIR)
-    args = ap.parse_args()
-
-    if not re.match(r"^https?://", args.repo):
-        print(f"WARN: {args.repo} is not an http(s) URL; metadata-only stub emitted.")
-        result = {
-            "url": args.repo,
-            "status": "metadata-only",
-            "generated_at": datetime.now(timezone.utc).isoformat(),
-            "negative_knowledge": (
-                f"[NEGATIVE_KNOWLEDGE: non-cloneable target {args.repo}]"
-            ),
-        }
-    else:
-        result = harvest_repo(
-            args.repo,
-            max_files=args.max_files,
-            max_bytes=args.max_bytes,
-            timeout=args.timeout,
-            base_dir=args.dir,
-        )
-
-    if args.show_context or not args.export:
-        print(f"\n🌑 [Darkharvest] {result['url']} → {result['status']}")
-        for key in ("spdx", "harvest_policy", "tree_entries", "source_hash"):
-            if key in result:
-                print(f"  {key}: {result[key]}")
-        for w in result.get("warnings", []):
-            print(f"  {w}")
-        if result.get("negative_knowledge"):
-            print(f"  {result['negative_knowledge'][:200]}")
-
-    if args.export:
-        out = Path(os.path.realpath(args.export))
-        out.parent.mkdir(parents=True, exist_ok=True)
-        out.write_text(json.dumps(result, indent=2), encoding="utf-8")
-        print(f"✅ Harvest scan exported to {out}")
-
-
-if __name__ == "__main__":
-    main()

package/skills/epistemic_search/SKILL.md

@@ -1,67 +0,0 @@
----
-name: epistemic_search
-description: High-integrity web search and document fetch skill with automatic SHA-256 content caching. Use when searching the web, retrieving primary sources, or fetching documentation while enforcing epistemic integrity.
----
-
-# Epistemic Search & Verifiable Source Fetching
-
-This skill provides zero-API-key web search (DuckDuckGo Lite) and content-addressed source fetching.
-Every fetched web page is automatically saved with its SHA-256 fingerprint into `.research/sources/<sha256>.md` for mathematical auditability.
-
-## 1. Web Search (No API Key Required)
-
-Run search with a query:
-```bash
-python3 skills/epistemic_search/scripts/search.py "YOUR SEARCH QUERY"
-```
-Or with JSON piping:
-```bash
-echo '{"query": "zk-SNARK hardware benchmarks", "allowed_domains": ["arxiv.org", "eprint.iacr.org"]}' | python3 skills/epistemic_search/scripts/search.py
-```
-
-Outputs `<search_results>` XML containing title, URL, and snippets.
-
-### Anti-bot "blocked" detection (distinct from no results)
-
-DuckDuckGo Lite sometimes answers with an HTTP-2xx image-selection CAPTCHA (the
-"anomaly" interstitial) instead of results. It is **not** an empty result set,
-so the script detects it and reports it distinctly:
-
-```bash
-# Typed status + actionable hint (status: ok | empty | blocked | error)
-python3 skills/epistemic_search/scripts/search.py --status "YOUR QUERY"
-```
-
-* `blocked` — DDG served its anti-bot page. Surface the hint, back off, and
-  retry later, or switch engines (`.research/config.json` → `search_engine`:
-  `firecrawl` / `searxng`), or export a BYO provider key
-  (`EXA_API_KEY` / `FIRECRAWL_API_KEY` / `PARALLEL_API_KEY` / `TAVILY_API_KEY` /
-  `TINYFISH_API_KEY`).
-* `empty` — a genuine no-hits query.
-* `error` — network/HTTP failure.
-
-The default list contract is unchanged: `search_duckduckgo(...)` still returns a
-list and returns `[]` when blocked (writing the hint to stderr). Programmatic
-callers that must act on a block can use `search_duckduckgo_detailed(...)`
-(status dict) or `require_search_duckduckgo(...)`, which raises `SearchBlocked`.
-Every query through our path is recorded in `.research/retrieval.jsonl` with
-its `status` and `provider`, so a block is no longer indistinguishable from a
-zero-hit query — including the metered rungs (exa/firecrawl/parallel/tavily/
-tinyfish/searxng/console) executed by the OpenCode `websearch` provider, so a
-metered-only run cannot read as `0 searches (~$0.00 est)`.
-
-The preflight reachability probe is deliberately **not** logged (it is not an
-agent search), so a zero-search run reports `our-path searches: 0`.
-
-## 2. Verifiable Web Fetch & Automatic Caching
-
-Fetch any web page or documentation:
-```bash
-python3 skills/epistemic_search/scripts/fetch.py "https://example.com/paper.html"
-```
-
-The script outputs clean markdown and automatically persists the raw source into `.research/sources/<sha256>.md`.
-
-Use the resulting SHA-256 hash when making claims:
-`[VERIFIED: <first_16_chars_of_hash>]`
-Ensure any cited quotes are exact verbatim substrings from the cached document.

package/skills/epistemic_search/__init__.py

@@ -1 +0,0 @@
-# Epistemic Search module

package/skills/epistemic_search/scripts/fetch.py

@@ -1,156 +0,0 @@
-#!/usr/bin/env python3
-"""
-Zero-API-Key Web Content Fetcher with Automatic Content-Addressed SHA-256 Caching.
-Fetches web content, cleans HTML to readable Markdown, and persists directly into
-.research/sources/<sha256>.md for epistemic auditability.
-"""
-
-import sys
-import os
-import re
-import json
-import urllib.request
-import urllib.parse
-from html.parser import HTMLParser
-
-# Add project root to sys.path to import SourceHasher
-PKG_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
-if PKG_ROOT not in sys.path:
-    sys.path.insert(0, PKG_ROOT)
-
-try:
-    from skills.research_cache.hasher import SourceHasher
-except ImportError:
-    SourceHasher = None
-
-USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
-
-class HTMLToMarkdownExtractor(HTMLParser):
-    def __init__(self):
-        super().__init__()
-        self.in_script_or_style = False
-        self.title = ""
-        self.in_title = False
-        self.text_chunks = []
-        self.headings = []
-
-    def handle_starttag(self, tag, attrs):
-        if tag in ("script", "style", "noscript", "svg", "header", "footer", "nav"):
-            self.in_script_or_style = True
-        elif tag == "title":
-            self.in_title = True
-        elif tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
-            self.text_chunks.append(f"\n\n{'#' * int(tag[1])} ")
-        elif tag in ("p", "div", "article", "section"):
-            self.text_chunks.append("\n\n")
-        elif tag == "li":
-            self.text_chunks.append("\n- ")
-        elif tag == "br":
-            self.text_chunks.append("\n")
-
-    def handle_endtag(self, tag):
-        if tag in ("script", "style", "noscript", "svg", "header", "footer", "nav"):
-            self.in_script_or_style = False
-        elif tag == "title":
-            self.in_title = False
-        elif tag in ("p", "div", "article", "section"):
-            self.text_chunks.append("\n")
-
-    def handle_data(self, data):
-        if self.in_script_or_style:
-            return
-        if self.in_title:
-            self.title += data
-        else:
-            self.text_chunks.append(data)
-
-    def get_markdown(self) -> str:
-        raw_text = "".join(self.text_chunks)
-        # Clean multiple blank lines
-        clean_text = re.sub(r"\n{3,}", "\n\n", raw_text).strip()
-        return clean_text
-
-def fetch_and_cache(url: str, research_dir: str = ".research") -> str:
-    headers = {
-        "User-Agent": USER_AGENT,
-        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-        "Accept-Language": "en-US,en;q=0.5",
-    }
-    req = urllib.request.Request(url, headers=headers)
-    try:
-        with urllib.request.urlopen(req, timeout=20) as resp:
-            raw_bytes = resp.read()
-            html = raw_bytes.decode("utf-8", errors="replace")
-    except Exception as e:
-        sys.stderr.write(f"[epistemic-fetch] Error fetching URL {url}: {e}\n")
-        return f"Error fetching {url}: {e}"
-
-    parser = HTMLToMarkdownExtractor()
-    parser.feed(html)
-    page_title = parser.title.strip() or url
-    body_markdown = parser.get_markdown()
-
-    # Cache into .research/sources/<sha256>.md
-    sha256_hash = "uncached"
-    cache_file = "not saved"
-    if SourceHasher:
-        from pathlib import Path
-        hasher = SourceHasher(base_dir=Path(research_dir))
-        sha256_hash = hasher.store_source(
-            url=url,
-            title=page_title,
-            content=body_markdown
-        )
-        cache_file = str(hasher.sources_dir / f"{sha256_hash}.md")
-    else:
-        import hashlib
-        sha256_hash = hashlib.sha256(body_markdown.encode("utf-8")).hexdigest()
-
-    output = []
-    output.append(f"<!-- EPISTEMIC_SOURCE_HASH: {sha256_hash} -->")
-    output.append(f"<!-- CACHED_AT: {cache_file} -->")
-    output.append(f"# {page_title}\n")
-    output.append(f"**Source URL:** {url}")
-    output.append(f"**Content SHA-256:** `{sha256_hash}`")
-    output.append(f"**Verification Tag:** `[VERIFIED: {sha256_hash[:16]}]`\n")
-    output.append(body_markdown)
-
-    return "\n".join(output)
-
-def main():
-    url = ""
-    research_dir = ".research"
-
-    if not sys.stdin.isatty():
-        try:
-            stdin_data = sys.stdin.read().strip()
-            if stdin_data:
-                try:
-                    payload = json.loads(stdin_data)
-                    url = payload.get("url", "")
-                    research_dir = payload.get("research_dir", research_dir)
-                except json.JSONDecodeError:
-                    url = stdin_data
-        except Exception:
-            pass
-
-    args = sys.argv[1:]
-    idx = 0
-    while idx < len(args):
-        arg = args[idx]
-        if arg == "--dir" and idx + 1 < len(args):
-            research_dir = args[idx + 1]
-            idx += 1
-        elif not url and not arg.startswith("--"):
-            url = arg
-        idx += 1
-
-    if not url:
-        sys.stderr.write("Usage: fetch.py [options] <url>\nOr pipe JSON: echo '{\"url\":\"https://...\"}' | fetch.py\n")
-        sys.exit(1)
-
-    result = fetch_and_cache(url=url, research_dir=research_dir)
-    print(result)
-
-if __name__ == "__main__":
-    main()