@lowwattlabs/clawsec 2.0.0

package/lib/intel-sync/manifest.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""ClawSec Intel Manifest Manager
+
+Manages {INTEL_DIR}/manifest.json — tracks source sync status.
+"""
+
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
+from config import INTEL_DIR
+
+MANIFEST_PATH = os.path.join(INTEL_DIR, "manifest.json")
+
+def load_manifest():
+    """Load manifest, create empty if missing."""
+    if os.path.exists(MANIFEST_PATH):
+        with open(MANIFEST_PATH, 'r') as f:
+            try:
+                return json.load(f)
+            except json.JSONDecodeError:
+                pass
+    return {
+        "version": "2.0",
+        "sources": [],
+        "updated_at": None
+    }
+
+def save_manifest(manifest):
+    """Atomic write manifest."""
+    manifest["updated_at"] = datetime.now(timezone.utc).isoformat()
+    tmp = MANIFEST_PATH + ".new"
+    with open(tmp, 'w') as f:
+        json.dump(manifest, f, indent=2, sort_keys=False)
+    os.rename(tmp, MANIFEST_PATH)
+
+def update_source(name, url="", record_count=0, status="success", error_msg=""):
+    """Update or add a source entry in the manifest."""
+    manifest = load_manifest()
+    found = False
+    for src in manifest["sources"]:
+        if src["name"] == name:
+            src["url"] = url
+            src["last_sync"] = datetime.now(timezone.utc).isoformat()
+            src["record_count"] = record_count
+            src["status"] = status
+            if error_msg:
+                src["error"] = error_msg
+            elif "error" in src:
+                del src["error"]
+            found = True
+            break
+    if not found:
+        entry = {
+            "name": name,
+            "url": url,
+            "last_sync": datetime.now(timezone.utc).isoformat(),
+            "record_count": record_count,
+            "status": status
+        }
+        if error_msg:
+            entry["error"] = error_msg
+        manifest["sources"].append(entry)
+    save_manifest(manifest)
+
+def get_source(name):
+    """Get a source entry by name."""
+    manifest = load_manifest()
+    for src in manifest["sources"]:
+        if src["name"] == name:
+            return src
+    return None
+
+def get_status():
+    """Return manifest summary for status display."""
+    return load_manifest()
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: manifest.py <update|status> [args...]")
+        sys.exit(1)
+
+    cmd = sys.argv[1]
+    if cmd == "update":
+        if len(sys.argv) < 4:
+            print("Usage: manifest.py update <name> <record_count> [status] [error_msg]")
+            sys.exit(1)
+        name = sys.argv[2]
+        count = int(sys.argv[3])
+        status = sys.argv[4] if len(sys.argv) > 4 else "success"
+        error = sys.argv[5] if len(sys.argv) > 5 else ""
+        update_source(name, record_count=count, status=status, error_msg=error)
+    elif cmd == "status":
+        manifest = get_status()
+        print(json.dumps(manifest, indent=2))
+    else:
+        print(f"Unknown command: {cmd}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

package/lib/intel-sync/sources/cisa-kev.sh

@@ -0,0 +1,24 @@
+# ⚡ Low Watt Labs
+# CISA KEV sync
+set -euo pipefail
+
+source "$(dirname "$0")/../../common/config.sh"
+source "$(dirname "$0")/../../common/colors.sh"
+source "$(dirname "$0")/../../common/log.sh"
+source "$(dirname "$0")/../../common/utils.sh"
+
+INTEL_DIR="${CLAWSEC_INTEL_DIR}"
+TARGET="${INTEL_DIR}/cisa-kev/known_exploited_vulnerabilities.json"
+URL="https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
+MANIFEST_PY="$(dirname "$0")/../manifest.py"
+
+log_info "Syncing CISA KEV..."
+
+if safe_download "$URL" "$TARGET" "jq empty"; then
+    count=$(jq '.vulnerabilities | length' "$TARGET" 2>/dev/null || echo 0)
+    python3 "$MANIFEST_PY" update cisa-kev "$count" success
+    echo -e "${CHECKMARK} CISA KEV: ${count} vulnerabilities"
+else
+    python3 "$MANIFEST_PY" update cisa-kev 0 failed "download failed"
+    echo -e "${CROSSMARK} CISA KEV: download failed (using stale cache)"
+fi

package/lib/intel-sync/sources/epss.sh

@@ -0,0 +1,34 @@
+# ⚡ Low Watt Labs
+# EPSS sync
+set -euo pipefail
+
+source "$(dirname "$0")/../../common/config.sh"
+source "$(dirname "$0")/../../common/colors.sh"
+source "$(dirname "$0")/../../common/log.sh"
+source "$(dirname "$0")/../../common/utils.sh"
+
+INTEL_DIR="${CLAWSEC_INTEL_DIR}"
+TARGET="${INTEL_DIR}/epss/epss_scores-current.csv"
+URL="https://epss.cyentia.com/epss_scores-current.csv.gz"
+MANIFEST_PY="$(dirname "$0")/../manifest.py"
+
+log_info "Syncing EPSS..."
+
+gz_tmp=$(mktemp "/tmp/epss.XXXXXX.csv.gz")
+if curl -fsSL --max-time 120 --retry 3 --retry-delay 5 "$URL" -o "$gz_tmp"; then
+    csv_tmp=$(mktemp "${TARGET}.XXXXXX.new")
+    if gunzip -c "$gz_tmp" > "$csv_tmp"; then
+        count=$(($(wc -l < "$csv_tmp") - 1))  # minus header
+        mv -f "$csv_tmp" "$TARGET"
+        python3 "$MANIFEST_PY" update epss "$count" success
+        echo -e "${CHECKMARK} EPSS: ${count} CVE scores"
+    else
+        rm -f "$csv_tmp"
+        python3 "$MANIFEST_PY" update epss 0 failed "gunzip failed"
+        echo -e "${CROSSMARK} EPSS: decompression failed"
+    fi
+else
+    python3 "$MANIFEST_PY" update epss 0 failed "download failed"
+    echo -e "${CROSSMARK} EPSS: download failed"
+fi
+rm -f "$gz_tmp"

package/lib/intel-sync/sources/feodo.sh

@@ -0,0 +1,27 @@
+# ⚡ Low Watt Labs
+# Feodo Tracker sync (abuse.ch)
+set -euo pipefail
+
+source "$(dirname "$0")/../../common/config.sh"
+source "$(dirname "$0")/../../common/colors.sh"
+source "$(dirname "$0")/../../common/log.sh"
+source "$(dirname "$0")/../../common/utils.sh"
+
+INTEL_DIR="${CLAWSEC_INTEL_DIR}"
+TARGET="${INTEL_DIR}/feodo/c2_ips.csv"
+URL="https://feodotracker.abuse.ch/downloads/ipblocklist.csv"
+MANIFEST_PY="$(dirname "$0")/../manifest.py"
+
+log_info "Syncing Feodo Tracker..."
+
+tmp=$(mktemp "${TARGET}.XXXXXX.new")
+if curl -fsSL --max-time 60 --retry 3 --retry-delay 5 "$URL" -o "$tmp"; then
+    count=$(grep -cv '^#' "$tmp" 2>/dev/null || echo 0)
+    mv -f "$tmp" "$TARGET"
+    python3 "$MANIFEST_PY" update feodo "$count" success
+    echo -e "${CHECKMARK} Feodo Tracker: ${count} C2 IPs"
+else
+    rm -f "$tmp"
+    python3 "$MANIFEST_PY" update feodo 0 failed "download failed"
+    echo -e "${CROSSMARK} Feodo Tracker: download failed"
+fi

package/lib/intel-sync/sources/malwarebazaar.sh

@@ -0,0 +1,22 @@
+# ⚡ Low Watt Labs
+set -euo pipefail
+source "$(dirname "$0")/../../common/config.sh"
+source "$(dirname "$0")/../../common/colors.sh"
+source "$(dirname "$0")/../../common/log.sh"
+source "$(dirname "$0")/../../common/utils.sh"
+
+INTEL_DIR="${CLAWSEC_INTEL_DIR}"
+TARGET_CSV="${INTEL_DIR}/malwarebazaar/recent_hashes.csv"
+MANIFEST_PY="$(dirname "$0")/../manifest.py"
+CSV_URL="https://bazaar.abuse.ch/export/csv/recent/"
+
+log_info "Syncing MalwareBazaar..."
+if safe_download "$CSV_URL" "$TARGET_CSV"; then
+    count=$(($(wc -l < "$TARGET_CSV") - 8))
+    [[ $count -lt 0 ]] && count=0
+    python3 "$MANIFEST_PY" update malwarebazaar "$count" success
+    echo -e "${CHECKMARK} MalwareBazaar: ${count} samples"
+else
+    python3 "$MANIFEST_PY" update malwarebazaar 0 failed "download failed"
+    echo -e "${CROSSMARK} MalwareBazaar: download failed"
+fi

package/lib/intel-sync/sources/osv.sh

@@ -0,0 +1,101 @@
+# ⚡ Low Watt Labs
+# OSV sync - npm and PyPI ecosystems
+# Downloads OSV advisories and builds a consolidated index for fast lookups
+set -euo pipefail
+
+# Run at low priority so this never starves other processes
+renice -n 15 $$ >/dev/null 2>&1 || true
+ionice -c 3 -p $$ 2>/dev/null || true
+
+source "$(dirname "$0")/../../common/config.sh"
+source "$(dirname "$0")/../../common/colors.sh"
+source "$(dirname "$0")/../../common/log.sh"
+source "$(dirname "$0")/../../common/utils.sh"
+
+INTEL_DIR="${CLAWSEC_INTEL_DIR}"
+OSV_DIR="${INTEL_DIR}/osv"
+MANIFEST_PY="$(dirname "$0")/../manifest.py"
+URL_BASE="https://osv-vulnerabilities.storage.googleapis.com"
+
+ECOSYSTEMS=("npm" "PyPI")
+total=0
+any_fail=0
+
+for eco in "${ECOSYSTEMS[@]}"; do
+    log_info "Syncing OSV $eco..."
+    eco_dir="${OSV_DIR}/${eco}"
+    mkdir -p "$eco_dir"
+
+    zip_url="${URL_BASE}/${eco}/all.zip"
+    zip_tmp=$(mktemp "/tmp/osv-${eco}.XXXXXX.zip")
+
+    if curl -fsSL --max-time 300 --retry 3 --retry-delay 5 "$zip_url" -o "$zip_tmp"; then
+        extract_tmp=$(mktemp -d "/tmp/osv-${eco}-extract.XXXXXX")
+        if unzip -q -o "$zip_tmp" -d "$extract_tmp" 2>/dev/null; then
+            # Clean old JSON files and broken symlinks before replacing
+            find "$eco_dir" -maxdepth 1 -name '*.json' -delete 2>/dev/null || true
+            find "$eco_dir" -maxdepth 1 -type l ! -exec test -e {} \; -delete 2>/dev/null || true
+            find "$extract_tmp" -maxdepth 1 -name '*.json' -exec mv -t "$eco_dir" {} +
+            count=$(find "$eco_dir" -maxdepth 1 -name '*.json' | wc -l)
+            total=$((total + count))
+            log_info "OSV $eco: $count advisories"
+        else
+            any_fail=1
+            log_warn "OSV $eco: extraction failed"
+        fi
+        rm -rf "$extract_tmp"
+    else
+        any_fail=1
+        log_warn "OSV $eco: download failed"
+    fi
+    rm -f "$zip_tmp"
+done
+
+# Build consolidated index: package_name -> list of advisory filenames
+log_info "Building OSV package index..."
+for eco in "${ECOSYSTEMS[@]}"; do
+    eco_dir="${OSV_DIR}/${eco}"
+    index_file="${eco_dir}/index.json"
+
+    if [[ -d "$eco_dir" ]]; then
+        # Build index using Python for speed
+        python3 -c "
+import json, os, sys
+eco_dir = sys.argv[1]
+index = {}
+for fname in os.listdir(eco_dir):
+    if not fname.endswith('.json') or fname == 'index.json':
+        continue
+    fpath = os.path.join(eco_dir, fname)
+    try:
+        with open(fpath) as f:
+            adv = json.load(f)
+        for affected in adv.get('affected', []):
+            pkg = affected.get('package', {})
+            name = pkg.get('name', '')
+            if name:
+                key = name.lower()
+                if key not in index:
+                    index[key] = []
+                index[key].append(fname)
+    except (json.JSONDecodeError, KeyError, OSError):
+        continue
+# Write index atomically
+tmp = index_file + '.new'
+with open(tmp, 'w') as f:
+    json.dump(index, f)
+os.rename(tmp, index_file)
+" "$eco_dir"
+        index_count=$(python3 -c "import json; print(len(json.load(open('$index_file'))))" 2>/dev/null || echo "?")
+        log_info "OSV $eco index: $index_count packages"
+    fi
+done
+
+status="success"
+[[ $any_fail -eq 1 ]] && status="partial"
+python3 "$MANIFEST_PY" update osv "$total" "$status"
+if [[ $any_fail -eq 0 ]]; then
+    echo -e "${CHECKMARK} OSV: ${total} advisories (npm + PyPI, indexed)"
+else
+    echo -e "${WARNMARK} OSV: partial sync — ${total} advisories"
+fi

package/lib/intel-sync/sources/semgrep-rules.sh

@@ -0,0 +1,28 @@
+# ⚡ Low Watt Labs
+set -euo pipefail
+source "$(dirname "$0")/../../common/config.sh"
+source "$(dirname "$0")/../../common/colors.sh"
+source "$(dirname "$0")/../../common/log.sh"
+source "$(dirname "$0")/../../common/utils.sh"
+
+INTEL_DIR="${CLAWSEC_INTEL_DIR}"
+REPO_DIR="${INTEL_DIR}/semgrep-rules/repo"
+MANIFEST_PY="$(dirname "$0")/../manifest.py"
+
+log_info "Syncing Semgrep rules (returntocorp/semgrep-rules)..."
+
+if [[ -d "$REPO_DIR/.git" ]]; then
+    git -C "$REPO_DIR" pull --quiet 2>/dev/null && status="success" || status="failed"
+else
+    rm -rf "$REPO_DIR"
+    git clone --depth 1 https://github.com/returntocorp/semgrep-rules.git "$REPO_DIR" 2>/dev/null && status="success" || status="failed"
+fi
+
+if [[ "$status" == "success" ]]; then
+    count=$(find "$REPO_DIR" -type f \( -name "*.yml" -o -name "*.yaml" \) 2>/dev/null | wc -l)
+    python3 "$MANIFEST_PY" update semgrep-rules "$count" success
+    echo -e "${CHECKMARK} Semgrep rules: ${count} rule files"
+else
+    python3 "$MANIFEST_PY" update semgrep-rules 0 failed "git pull/clone failed"
+    echo -e "${CROSSMARK} Semgrep rules: git sync failed"
+fi

package/lib/intel-sync/sources/threatfox.sh

@@ -0,0 +1,28 @@
+# ⚡ Low Watt Labs
+# ThreatFox sync (abuse.ch)
+set -euo pipefail
+
+source "$(dirname "$0")/../../common/config.sh"
+source "$(dirname "$0")/../../common/colors.sh"
+source "$(dirname "$0")/../../common/log.sh"
+source "$(dirname "$0")/../../common/utils.sh"
+
+INTEL_DIR="${CLAWSEC_INTEL_DIR}"
+TARGET="${INTEL_DIR}/threatfox/iocs.csv"
+URL="https://threatfox.abuse.ch/export/csv/recent/"
+MANIFEST_PY="$(dirname "$0")/../manifest.py"
+
+log_info "Syncing ThreatFox..."
+
+tmp=$(mktemp "${TARGET}.XXXXXX.new")
+if curl -fsSL --max-time 120 --retry 3 --retry-delay 5 "$URL" -o "$tmp"; then
+    # Count actual data lines (skip comment lines starting with #)
+    count=$(grep -cv '^#' "$tmp" 2>/dev/null || echo 0)
+    mv -f "$tmp" "$TARGET"
+    python3 "$MANIFEST_PY" update threatfox "$count" success
+    echo -e "${CHECKMARK} ThreatFox: ${count} IOCs"
+else
+    rm -f "$tmp"
+    python3 "$MANIFEST_PY" update threatfox 0 failed "download failed"
+    echo -e "${CROSSMARK} ThreatFox: download failed"
+fi

package/lib/intel-sync/sources/urlhaus.sh

@@ -0,0 +1,42 @@
+# ⚡ Low Watt Labs
+set -euo pipefail
+source "$(dirname "$0")/../../common/config.sh"
+source "$(dirname "$0")/../../common/colors.sh"
+source "$(dirname "$0")/../../common/log.sh"
+source "$(dirname "$0")/../../common/utils.sh"
+
+INTEL_DIR="${CLAWSEC_INTEL_DIR}"
+TARGET="${INTEL_DIR}/urlhaus/urls.csv"
+URL="https://urlhaus.abuse.ch/downloads/csv/"
+MANIFEST_PY="$(dirname "$0")/../manifest.py"
+# The URLhaus ZIP contains a file named "csv.txt" — we extract it
+# to a stable path so ioc-match.py always knows where to find it.
+URLHAUS_INNER_FILE="csv.txt"
+
+log_info "Syncing URLhaus..."
+mkdir -p "${INTEL_DIR}/urlhaus"
+zip_tmp=$(mktemp "/tmp/urlhaus.XXXXXX.zip")
+if curl -fsSL --max-time 120 --retry 3 --retry-delay 5 "$URL" -o "$zip_tmp"; then
+    csv_tmp=$(mktemp "${TARGET}.XXXXXX.new")
+    if unzip -p "$zip_tmp" "$URLHAUS_INNER_FILE" > "$csv_tmp" 2>/dev/null; then
+        count=$(($(wc -l < "$csv_tmp") - 1))  # minus header
+        # Validate extracted CSV is text, not raw ZIP
+        if file "$csv_tmp" | grep -qi "zip\|archive"; then
+            rm -f "$csv_tmp"
+            python3 "$MANIFEST_PY" update urlhaus 0 failed "extracted file is still a ZIP"
+            echo -e "${CROSSMARK} URLhaus: extracted file is still a ZIP (sync may have failed)"
+        else
+            mv -f "$csv_tmp" "$TARGET"
+        fi
+        python3 "$MANIFEST_PY" update urlhaus "$count" success
+        echo -e "${CHECKMARK} URLhaus: ${count} malicious URLs"
+    else
+        rm -f "$csv_tmp"
+        python3 "$MANIFEST_PY" update urlhaus 0 failed "unzip failed"
+        echo -e "${CROSSMARK} URLhaus: decompression failed"
+    fi
+else
+    python3 "$MANIFEST_PY" update urlhaus 0 failed "download failed"
+    echo -e "${CROSSMARK} URLhaus: download failed"
+fi
+rm -f "$zip_tmp"

package/lib/intel-sync/sources/yara-rules.sh

@@ -0,0 +1,38 @@
+# ⚡ Low Watt Labs
+# YARA rules sync (Neo23x0/signature-base)
+set -euo pipefail
+
+source "$(dirname "$0")/../../common/config.sh"
+source "$(dirname "$0")/../../common/colors.sh"
+source "$(dirname "$0")/../../common/log.sh"
+source "$(dirname "$0")/../../common/utils.sh"
+
+INTEL_DIR="${CLAWSEC_INTEL_DIR}"
+REPO_DIR="${INTEL_DIR}/yara-rules/repo"
+MANIFEST_PY="$(dirname "$0")/../manifest.py"
+
+log_info "Syncing YARA rules (Neo23x0/signature-base)..."
+
+if [[ -d "$REPO_DIR/.git" ]]; then
+    if git -C "$REPO_DIR" pull --quiet 2>/dev/null; then
+        status="success"
+    else
+        status="failed"
+    fi
+else
+    rm -rf "$REPO_DIR"
+    if git clone --depth 1 https://github.com/Neo23x0/signature-base.git "$REPO_DIR" 2>/dev/null; then
+        status="success"
+    else
+        status="failed"
+    fi
+fi
+
+if [[ "$status" == "success" ]]; then
+    count=$(find "$REPO_DIR/yara" -name '*.yar' -o -name '*.yara' 2>/dev/null | wc -l)
+    python3 "$MANIFEST_PY" update yara-rules "$count" success
+    echo -e "${CHECKMARK} YARA rules: ${count} rule files"
+else
+    python3 "$MANIFEST_PY" update yara-rules 0 failed "git pull/clone failed"
+    echo -e "${CROSSMARK} YARA rules: git sync failed"
+fi

package/lib/intel-sync/sync.sh

@@ -0,0 +1,96 @@
+# ⚡ Low Watt Labs — ClawSec Intel Sync
+# ClawSec v2 - Intel Sync Orchestrator
+# Runs all intel source sync jobs, gracefully handling failures
+set -euo pipefail
+
+VERSION="2.0.0"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+SOURCES_DIR="${SCRIPT_DIR}/sources"
+
+source "${SCRIPT_DIR}/../common/config.sh"
+source "${SCRIPT_DIR}/../common/colors.sh"
+source "${SCRIPT_DIR}/../common/log.sh"
+
+usage() {
+    echo "ClawSec v${VERSION} — Intel Sync"
+    echo ""
+    echo "Usage: sync.sh [OPTIONS] [SOURCE...]"
+    echo ""
+    echo "Options:"
+    echo "  --all       Sync all sources (default)"
+    echo "  --list      List available sources"
+    echo "  --status    Show current cache status"
+    echo "  --json      Output status as JSON"
+    echo "  --help      Show this help"
+    echo ""
+    echo "Sources: cisa-kev, osv, epss, malwarebazaar, urlhaus,"
+    echo "         threatfox, feodo, yara-rules, semgrep-rules"
+    exit 0
+}
+
+ALL_SOURCES=(cisa-kev osv epss malwarebazaar urlhaus threatfox feodo yara-rules semgrep-rules)
+requested_sources=()
+json_output=0
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --all)   shift ;;
+        --list)  
+            echo "Available intel sources:"
+            for s in "${ALL_SOURCES[@]}"; do echo "  - $s"; done
+            exit 0 ;;
+        --status|--status-only)
+            python3 "${SCRIPT_DIR}/manifest.py" status
+            exit 0 ;;
+        --json)
+            json_output=1
+            shift ;;
+        --help|-h)
+            usage ;;
+        -*)
+            echo "Unknown option: $1" >&2
+            exit 1 ;;
+        *)
+            requested_sources+=("$1")
+            shift ;;
+    esac
+done
+
+# Default: all sources
+[[ ${#requested_sources[@]} -eq 0 ]] && requested_sources=("${ALL_SOURCES[@]}")
+
+echo -e "${BOLD}╔══════════════════════════════════════╗${RESET}"
+echo -e "${BOLD}║  ClawSec v${VERSION} — Intel Sync        ║${RESET}"
+echo -e "${BOLD}╚══════════════════════════════════════╝${RESET}"
+echo ""
+echo -e "  Syncing ${BOLD}${#requested_sources[@]}${RESET} source(s): ${CYAN}${requested_sources[*]}${RESET}"
+echo ""
+
+start_time=$(date +%s)
+fail_count=0
+
+for src in "${requested_sources[@]}"; do
+    script="${SOURCES_DIR}/${src}.sh"
+    if [[ -x "$script" ]]; then
+        if ! "$script"; then
+            fail_count=$((fail_count + 1))
+        fi
+    else
+        echo -e "${WARNMARK} Unknown source: $src (no script at $script)"
+        fail_count=$((fail_count + 1))
+    fi
+done
+
+end_time=$(date +%s)
+elapsed=$((end_time - start_time))
+
+echo ""
+if [[ $fail_count -eq 0 ]]; then
+    echo -e "${CHECKMARK} Sync complete in ${elapsed}s — all sources fresh"
+else
+    echo -e "${WARNMARK} Sync complete in ${elapsed}s — ${fail_count} source(s) had issues (stale data preserved)"
+fi
+
+if [[ $json_output -eq 1 ]]; then
+    python3 "${SCRIPT_DIR}/manifest.py" status
+fi