crewly 1.1.2 → 1.2.0

package/config/skills/agent/browse-stealth/stealth-browse.py

@@ -0,0 +1,330 @@
+#!/usr/bin/env python3
+"""
+Stealth browser automation using Patchright + Chrome CDP.
+
+Connects to a REAL running Chrome browser via CDP (Chrome DevTools Protocol)
+instead of launching a new Chromium instance. This avoids anti-detection
+triggers like navigator.webdriver, headless fingerprints, and missing
+browser history/extensions.
+
+CRITICAL CDP MODE RULES (violations cause silent failures):
+  - DO NOT call new_context()     → breaks DNS/TLS, causes ERR_CONNECTION_CLOSED
+  - DO NOT call add_init_script() → conflicts with Chrome's initialization flow
+  - DO NOT override User-Agent    → creates HTTP/JS inconsistency
+  - DO inject cookies via add_cookies() on the existing context
+"""
+
+import argparse
+import http.client
+import json
+import os
+import random
+import subprocess
+import sys
+import time
+
+# Bypass macOS proxy tools (ClashX, Surge) that hijack urllib
+os.environ["no_proxy"] = "localhost,127.0.0.1"
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+CDP_PORT = 9222
+SCREENSHOT_DIR = os.path.expanduser("~/.crewly/screenshots")
+
+
+# ── CDP connection helpers ──────────────────────────────────────
+
+def get_ws_url(port: int = CDP_PORT) -> str:
+    """
+    Manually fetch the WebSocket URL from Chrome's CDP endpoint.
+
+    Uses http.client.HTTPConnection (NOT urllib) to avoid macOS proxy
+    tool interference. Note: NO trailing slash on /json/version
+    (Patchright 1.58 + Chrome 144 trailing slash bug).
+    """
+    conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5)
+    try:
+        conn.request("GET", "/json/version")
+        resp = conn.getresponse()
+        if resp.status != 200:
+            raise ConnectionError(f"CDP returned HTTP {resp.status}")
+        data = json.loads(resp.read().decode())
+        return data["webSocketDebuggerUrl"]
+    finally:
+        conn.close()
+
+
+def ensure_chrome_running() -> str:
+    """Launch Chrome via the launcher script if CDP is not responding."""
+    try:
+        return get_ws_url()
+    except Exception:
+        pass
+
+    launcher = os.path.join(SCRIPT_DIR, "launch-chrome-cdp.sh")
+    result = subprocess.run(
+        ["bash", launcher, str(CDP_PORT)],
+        capture_output=True, text=True, timeout=30
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"Failed to launch Chrome: {result.stderr}")
+
+    ws_url = result.stdout.strip().split("\n")[-1]
+    if ws_url.startswith("ws://"):
+        return ws_url
+
+    return get_ws_url()
+
+
+# ── Human-like behavior helpers ─────────────────────────────────
+
+def human_delay(min_s: float = 0.5, max_s: float = 2.0):
+    """Random delay to simulate human hesitation."""
+    time.sleep(random.uniform(min_s, max_s))
+
+
+def human_scroll(page, direction: str = "down", amount: int = 0):
+    """Scroll with natural, variable distances."""
+    if amount == 0:
+        amount = random.randint(200, 600)
+    delta = amount if direction == "down" else -amount
+    page.mouse.wheel(0, delta)
+    human_delay(0.3, 1.0)
+
+
+def human_type(page, selector: str, text: str):
+    """Type text with variable inter-key delays."""
+    page.click(selector)
+    human_delay(0.2, 0.5)
+    for char in text:
+        page.keyboard.type(char)
+        time.sleep(random.uniform(0.05, 0.15))
+
+
+# ── Risk control detection ──────────────────────────────────────
+
+def detect_risk_control(page) -> dict:
+    """Check if the page is showing CAPTCHA or rate-limit signals."""
+    signals = []
+
+    url = page.url.lower()
+    if "captcha" in url or "verify" in url or "challenge" in url:
+        signals.append("captcha_url")
+
+    try:
+        content = page.content()
+        risk_keywords = [
+            "验证码", "滑块验证", "人机验证", "操作频繁",
+            "captcha", "verify you are human", "rate limit",
+            "too many requests", "access denied", "you have been blocked",
+            "your account has been locked", "suspicious activity",
+        ]
+        content_lower = content.lower()
+        for kw in risk_keywords:
+            if kw.lower() in content_lower:
+                signals.append(f"keyword:{kw}")
+    except Exception:
+        pass
+
+    return {
+        "detected": len(signals) > 0,
+        "signals": signals,
+    }
+
+
+# ── Core actions ────────────────────────────────────────────────
+
+def action_read(page, url: str, selectors: list, wait_for: str = None, wait_timeout: int = 15000) -> dict:
+    """Navigate to URL, extract text content from selectors."""
+    page.goto(url, wait_until="domcontentloaded", timeout=60000)
+
+    # For SPA sites (X.com, React apps), wait for JS to finish rendering
+    try:
+        page.wait_for_load_state("networkidle", timeout=15000)
+    except Exception:
+        pass  # Best effort — some SPAs never fully idle
+
+    # Wait for a specific selector if requested
+    if wait_for:
+        try:
+            page.wait_for_selector(wait_for, timeout=wait_timeout)
+        except Exception:
+            pass  # Continue even if wait_for times out — best effort
+
+    human_delay(1.0, 3.0)
+
+    risk = detect_risk_control(page)
+    if risk["detected"]:
+        return {
+            "success": False,
+            "error": "risk_control_detected",
+            "signals": risk["signals"],
+            "advice": "Stop immediately. Risk control triggered.",
+        }
+
+    results = {}
+    if not selectors:
+        # Extract main content heuristically
+        for sel in ["article", "main", "[role='main']", ".content", "#content", "body"]:
+            try:
+                el = page.query_selector(sel)
+                if el:
+                    text = el.inner_text()
+                    if len(text) > 50:
+                        results["content"] = text[:10000]
+                        break
+            except Exception:
+                continue
+        if not results:
+            results["content"] = page.inner_text("body")[:10000]
+    else:
+        for sel in selectors:
+            try:
+                el = page.query_selector(sel)
+                results[sel] = el.inner_text() if el else None
+            except Exception as e:
+                results[sel] = f"error: {e}"
+
+    return {
+        "success": True,
+        "url": page.url,
+        "title": page.title(),
+        "results": results,
+    }
+
+
+def action_screenshot(page, url: str) -> dict:
+    """Navigate and take a screenshot."""
+    os.makedirs(SCREENSHOT_DIR, exist_ok=True)
+
+    page.goto(url, wait_until="domcontentloaded", timeout=60000)
+    try:
+        page.wait_for_load_state("networkidle", timeout=15000)
+    except Exception:
+        pass
+    human_delay(1.5, 3.0)
+
+    risk = detect_risk_control(page)
+    if risk["detected"]:
+        return {
+            "success": False,
+            "error": "risk_control_detected",
+            "signals": risk["signals"],
+        }
+
+    timestamp = int(time.time())
+    filename = f"stealth_{timestamp}.png"
+    filepath = os.path.join(SCREENSHOT_DIR, filename)
+
+    page.screenshot(path=filepath, full_page=False)
+
+    return {
+        "success": True,
+        "url": page.url,
+        "title": page.title(),
+        "screenshot": filepath,
+    }
+
+
+def action_interact(page, url: str, selectors: list) -> dict:
+    """Navigate and interact with elements (click, scroll)."""
+    page.goto(url, wait_until="domcontentloaded", timeout=60000)
+    human_delay(1.0, 2.5)
+
+    risk = detect_risk_control(page)
+    if risk["detected"]:
+        return {
+            "success": False,
+            "error": "risk_control_detected",
+            "signals": risk["signals"],
+        }
+
+    results = []
+    for sel in selectors:
+        try:
+            el = page.query_selector(sel)
+            if el:
+                el.scroll_into_view_if_needed()
+                human_delay(0.3, 0.8)
+                el.click()
+                human_delay(0.5, 1.5)
+                results.append({"selector": sel, "action": "clicked", "success": True})
+            else:
+                results.append({"selector": sel, "action": "not_found", "success": False})
+        except Exception as e:
+            results.append({"selector": sel, "action": "error", "error": str(e), "success": False})
+
+    return {
+        "success": True,
+        "url": page.url,
+        "title": page.title(),
+        "interactions": results,
+    }
+
+
+# ── Main ────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(description="Stealth browse via Patchright + CDP")
+    parser.add_argument("--url", required=True, help="Target URL")
+    parser.add_argument("--action", default="read", choices=["read", "screenshot", "interact"],
+                        help="Action to perform")
+    parser.add_argument("--selectors", nargs="*", default=[], help="CSS selectors")
+    parser.add_argument("--wait-for", default=None, help="CSS selector to wait for before extracting")
+    parser.add_argument("--wait-timeout", type=int, default=15000, help="Timeout in ms for --wait-for")
+    parser.add_argument("--cdp-port", type=int, default=CDP_PORT, help="CDP port")
+    args = parser.parse_args()
+
+    # Ensure patchright is installed
+    try:
+        from patchright.sync_api import sync_playwright
+    except ImportError:
+        print(json.dumps({
+            "success": False,
+            "error": "patchright_not_installed",
+            "fix": "pip3 install patchright && python3 -m patchright install chromium",
+        }))
+        sys.exit(1)
+
+    # Ensure Chrome is running with CDP
+    try:
+        ws_url = ensure_chrome_running()
+    except Exception as e:
+        print(json.dumps({"success": False, "error": f"chrome_launch_failed: {e}"}))
+        sys.exit(1)
+
+    # Connect via Patchright
+    with sync_playwright() as pw:
+        try:
+            browser = pw.chromium.connect_over_cdp(ws_url)
+        except Exception as e:
+            print(json.dumps({"success": False, "error": f"cdp_connect_failed: {e}"}))
+            sys.exit(1)
+
+        # CRITICAL: Use Chrome's existing default context, NOT new_context()
+        contexts = browser.contexts
+        if not contexts:
+            print(json.dumps({"success": False, "error": "no_browser_context_found"}))
+            sys.exit(1)
+
+        context = contexts[0]
+        page = context.new_page()
+
+        try:
+            if args.action == "read":
+                result = action_read(page, args.url, args.selectors, args.wait_for, args.wait_timeout)
+            elif args.action == "screenshot":
+                result = action_screenshot(page, args.url)
+            elif args.action == "interact":
+                result = action_interact(page, args.url, args.selectors)
+            else:
+                result = {"success": False, "error": f"unknown_action: {args.action}"}
+        except Exception as e:
+            result = {"success": False, "error": str(e)}
+        finally:
+            page.close()
+
+        print(json.dumps(result, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()

package/config/skills/agent/competitor-content-tracker/execute.sh

@@ -0,0 +1,232 @@
+#!/bin/bash
+# Competitor Content Tracker — store, query, and compare competitor content data
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${SCRIPT_DIR}/../_common/lib.sh"
+
+INPUT="${1:-}"
+[ -z "$INPUT" ] && error_exit "Usage: execute.sh '{\"action\":\"save|list|compare|latest\",\"competitor\":\"crewai\",...}'"
+
+ACTION=$(echo "$INPUT" | jq -r '.action // empty')
+PROJECT_PATH=$(echo "$INPUT" | jq -r '.projectPath // empty')
+
+require_param "action" "$ACTION"
+
+# Resolve storage directory
+if [ -n "$PROJECT_PATH" ]; then
+  TRACKER_DIR="${PROJECT_PATH}/.crewly/content/competitors"
+else
+  TRACKER_DIR="${HOME}/.crewly/content/competitors"
+fi
+mkdir -p "$TRACKER_DIR"
+
+# Valid competitors
+validate_competitor() {
+  case "$1" in
+    crewai|n8n|relevance-ai|autogen|langchain|langraph|openai|other) return 0 ;;
+    *) error_exit "Unknown competitor: $1. Valid: crewai, n8n, relevance-ai, autogen, langchain, langraph, openai, other" ;;
+  esac
+}
+
+case "$ACTION" in
+
+  # ─────────────────────────────────────────────
+  # SAVE: Store content items from a competitor scan
+  # ─────────────────────────────────────────────
+  save)
+    COMPETITOR=$(echo "$INPUT" | jq -r '.competitor // empty')
+    SOURCE_TYPE=$(echo "$INPUT" | jq -r '.sourceType // empty')
+    ITEMS=$(echo "$INPUT" | jq -r '.items // empty')
+
+    require_param "competitor" "$COMPETITOR"
+    require_param "sourceType" "$SOURCE_TYPE"
+    require_param "items" "$ITEMS"
+
+    validate_competitor "$COMPETITOR"
+
+    # Validate source type
+    case "$SOURCE_TYPE" in
+      blog|twitter|linkedin|github-release|changelog|youtube|community|press|other) ;;
+      *) error_exit "Invalid sourceType: $SOURCE_TYPE. Valid: blog, twitter, linkedin, github-release, changelog, youtube, community, press, other" ;;
+    esac
+
+    # Validate items is a JSON array
+    if ! echo "$ITEMS" | jq 'type == "array"' 2>/dev/null | grep -q true; then
+      error_exit "items must be a JSON array"
+    fi
+
+    NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+    TODAY=$(date -u +%Y-%m-%d)
+    SCAN_ID="comp-$(date +%s)-$((RANDOM % 1000))"
+
+    # Enrich items with metadata
+    ENRICHED=$(echo "$ITEMS" | jq \
+      --arg competitor "$COMPETITOR" \
+      --arg sourceType "$SOURCE_TYPE" \
+      --arg scanId "$SCAN_ID" \
+      --arg ts "$NOW" \
+      '[.[] | . + {"competitor": $competitor, "sourceType": $sourceType, "scanId": $scanId, "scannedAt": $ts}]')
+
+    COUNT=$(echo "$ENRICHED" | jq 'length')
+
+    # Save to competitor-specific file
+    COMP_DIR="${TRACKER_DIR}/${COMPETITOR}"
+    mkdir -p "$COMP_DIR"
+    SCAN_FILE="${COMP_DIR}/${TODAY}-${SOURCE_TYPE}.json"
+
+    if [ -f "$SCAN_FILE" ]; then
+      EXISTING=$(cat "$SCAN_FILE")
+      MERGED=$(jq -n --argjson existing "$EXISTING" --argjson new "$ENRICHED" \
+        '{"scans": ($existing.scans + [{"scanId": $new[0].scanId, "scannedAt": $new[0].scannedAt, "count": ($new | length), "items": $new}])}')
+      echo "$MERGED" > "$SCAN_FILE"
+    else
+      jq -n --argjson items "$ENRICHED" --arg competitor "$COMPETITOR" --arg sourceType "$SOURCE_TYPE" --arg date "$TODAY" \
+        '{"competitor": $competitor, "sourceType": $sourceType, "date": $date, "scans": [{"scanId": $items[0].scanId, "scannedAt": $items[0].scannedAt, "count": ($items | length), "items": $items}]}' > "$SCAN_FILE"
+    fi
+
+    jq -n \
+      --arg scanId "$SCAN_ID" \
+      --arg competitor "$COMPETITOR" \
+      --arg sourceType "$SOURCE_TYPE" \
+      --argjson count "$COUNT" \
+      --arg file "$SCAN_FILE" \
+      '{"success":true,"action":"save","scanId":$scanId,"competitor":$competitor,"sourceType":$sourceType,"count":$count,"file":$file}'
+    ;;
+
+  # ─────────────────────────────────────────────
+  # LIST: List tracked content by competitor
+  # ─────────────────────────────────────────────
+  list)
+    FILTER_COMP=$(echo "$INPUT" | jq -r '.competitor // empty')
+    LIMIT=$(echo "$INPUT" | jq -r '.limit // "20"')
+
+    RESULTS="[]"
+
+    SEARCH_DIRS=""
+    if [ -n "$FILTER_COMP" ]; then
+      validate_competitor "$FILTER_COMP"
+      SEARCH_DIRS="${TRACKER_DIR}/${FILTER_COMP}"
+    else
+      SEARCH_DIRS="$TRACKER_DIR"
+    fi
+
+    for f in $(find "$SEARCH_DIRS" -name "*.json" -type f 2>/dev/null | sort -r | head -"$LIMIT"); do
+      [ -f "$f" ] || continue
+      COMP=$(jq -r '.competitor // "unknown"' "$f" 2>/dev/null || echo "unknown")
+      STYPE=$(jq -r '.sourceType // "unknown"' "$f" 2>/dev/null || echo "unknown")
+      FDATE=$(jq -r '.date // "unknown"' "$f" 2>/dev/null || echo "unknown")
+      TOTAL=$(jq '[.scans[].count] | add // 0' "$f" 2>/dev/null || echo "0")
+      LAST=$(jq -r '.scans[-1].scannedAt // "unknown"' "$f" 2>/dev/null || echo "unknown")
+
+      RESULTS=$(echo "$RESULTS" | jq \
+        --arg file "$f" \
+        --arg competitor "$COMP" \
+        --arg sourceType "$STYPE" \
+        --arg date "$FDATE" \
+        --argjson totalItems "$TOTAL" \
+        --arg lastScan "$LAST" \
+        '. + [{"file":$file,"competitor":$competitor,"sourceType":$sourceType,"date":$date,"totalItems":$totalItems,"lastScan":$lastScan}]')
+    done
+
+    COUNT=$(echo "$RESULTS" | jq 'length')
+    jq -n --argjson results "$RESULTS" --argjson count "$COUNT" \
+      '{"success":true,"action":"list","count":$count,"results":$results}'
+    ;;
+
+  # ─────────────────────────────────────────────
+  # LATEST: Get latest content from a competitor
+  # ─────────────────────────────────────────────
+  latest)
+    FILTER_COMP=$(echo "$INPUT" | jq -r '.competitor // empty')
+    FILTER_TYPE=$(echo "$INPUT" | jq -r '.sourceType // empty')
+    LIMIT=$(echo "$INPUT" | jq -r '.limit // "15"')
+
+    require_param "competitor" "$FILTER_COMP"
+    validate_competitor "$FILTER_COMP"
+
+    COMP_DIR="${TRACKER_DIR}/${FILTER_COMP}"
+    ALL_ITEMS="[]"
+
+    if [ -d "$COMP_DIR" ]; then
+      for f in $(ls -t "$COMP_DIR"/*.json 2>/dev/null | head -5); do
+        [ -f "$f" ] || continue
+        if [ -n "$FILTER_TYPE" ]; then
+          STYPE=$(jq -r '.sourceType // ""' "$f")
+          [ "$STYPE" != "$FILTER_TYPE" ] && continue
+        fi
+        ITEMS=$(jq '.scans[-1].items // []' "$f" 2>/dev/null || echo "[]")
+        ALL_ITEMS=$(jq -n --argjson a "$ALL_ITEMS" --argjson b "$ITEMS" '$a + $b')
+      done
+    fi
+
+    ALL_ITEMS=$(echo "$ALL_ITEMS" | jq --argjson limit "$LIMIT" '.[:$limit]')
+    COUNT=$(echo "$ALL_ITEMS" | jq 'length')
+
+    jq -n --arg competitor "$FILTER_COMP" --argjson items "$ALL_ITEMS" --argjson count "$COUNT" \
+      '{"success":true,"action":"latest","competitor":$competitor,"count":$count,"items":$items}'
+    ;;
+
+  # ─────────────────────────────────────────────
+  # COMPARE: Compare content activity across competitors
+  # ─────────────────────────────────────────────
+  compare)
+    COMPETITORS_INPUT=$(echo "$INPUT" | jq -r '.competitors // "crewai,n8n,relevance-ai"')
+    DAYS=$(echo "$INPUT" | jq -r '.days // "7"')
+
+    IFS=',' read -ra COMP_ARRAY <<< "$COMPETITORS_INPUT"
+
+    # Calculate date threshold
+    THRESHOLD=$(date -u -v-"${DAYS}"d +%Y-%m-%d 2>/dev/null || date -u -d "-${DAYS} days" +%Y-%m-%d 2>/dev/null || date -u +%Y-%m-%d)
+
+    COMPARISON="[]"
+
+    for comp in "${COMP_ARRAY[@]}"; do
+      comp=$(echo "$comp" | xargs)
+      COMP_DIR="${TRACKER_DIR}/${comp}"
+
+      TOTAL_ITEMS=0
+      BY_TYPE="{}"
+      LATEST_TITLES="[]"
+
+      if [ -d "$COMP_DIR" ]; then
+        for f in "$COMP_DIR"/*.json; do
+          [ -f "$f" ] || continue
+          FDATE=$(jq -r '.date // ""' "$f" 2>/dev/null)
+          [ -z "$FDATE" ] && continue
+
+          if [ "$FDATE" \> "$THRESHOLD" ] || [ "$FDATE" = "$THRESHOLD" ]; then
+            STYPE=$(jq -r '.sourceType // "unknown"' "$f" 2>/dev/null)
+            FCOUNT=$(jq '[.scans[].count] | add // 0' "$f" 2>/dev/null || echo "0")
+            TOTAL_ITEMS=$((TOTAL_ITEMS + FCOUNT))
+
+            BY_TYPE=$(echo "$BY_TYPE" | jq --arg t "$STYPE" --argjson c "$FCOUNT" \
+              '. + {($t): ((.[$t] // 0) + $c)}')
+
+            # Get latest titles
+            TITLES=$(jq '[.scans[-1].items[:3][] | .title // "untitled"]' "$f" 2>/dev/null || echo "[]")
+            LATEST_TITLES=$(jq -n --argjson a "$LATEST_TITLES" --argjson b "$TITLES" '$a + $b')
+          fi
+        done
+      fi
+
+      LATEST_TITLES=$(echo "$LATEST_TITLES" | jq '.[:5]')
+
+      COMPARISON=$(echo "$COMPARISON" | jq \
+        --arg competitor "$comp" \
+        --argjson totalItems "$TOTAL_ITEMS" \
+        --argjson byType "$BY_TYPE" \
+        --argjson latestTitles "$LATEST_TITLES" \
+        '. + [{"competitor":$competitor,"totalItems":$totalItems,"bySourceType":$byType,"recentTitles":$latestTitles}]')
+    done
+
+    jq -n \
+      --argjson comparison "$COMPARISON" \
+      --arg period "${DAYS} days" \
+      --arg since "$THRESHOLD" \
+      '{"success":true,"action":"compare","period":$period,"since":$since,"competitors":$comparison}'
+    ;;
+
+  *)
+    error_exit "Unknown action: $ACTION. Valid: save, list, latest, compare"
+    ;;
+esac