forgedock 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -2
- package/bin/forgedock.mjs +286 -10
- package/commands/analytics.md +129 -52
- package/commands/audit-agents.md +29 -5
- package/commands/audit.md +55 -17
- package/commands/autopilot.md +11 -6
- package/commands/cleanup.md +44 -24
- package/commands/deploy-info.md +7 -3
- package/commands/forge-stats.md +13 -4
- package/commands/forgedock-init.md +534 -0
- package/commands/geo-audit.md +114 -26
- package/commands/incident-response.md +4 -1
- package/commands/issue.md +26 -25
- package/commands/milestone.md +80 -59
- package/commands/orchestrate.md +38 -21
- package/commands/pipeline-health.md +26 -15
- package/commands/qa-sweep.md +68 -5
- package/commands/quality-gate.md +18 -0
- package/commands/review-pr-agents.md +58 -90
- package/commands/review-pr.md +58 -16
- package/commands/rollback.md +8 -5
- package/commands/security-audit.md +68 -12
- package/commands/sync-ecosystem.md +196 -156
- package/commands/validate.md +1 -1
- package/commands/work-on/build/architect.md +2 -2
- package/commands/work-on/build/context.md +2 -2
- package/commands/work-on/build/implement.md +3 -3
- package/commands/work-on/build/validate.md +3 -3
- package/commands/work-on/build.md +6 -6
- package/commands/work-on/close.md +59 -30
- package/commands/work-on/decompose.md +6 -6
- package/commands/work-on/investigate.md +16 -20
- package/commands/work-on/review.md +4 -4
- package/commands/work-on-monolithic.md +15 -16
- package/commands/work-on.md +45 -33
- package/docs/CODEX.md +1 -1
- package/docs/CONFIG.md +398 -0
- package/docs/WORKFLOW.md +115 -72
- package/package.json +2 -2
- package/scripts/verify-host-headers.sh +14 -9
- package/scripts/verify-route-registration.sh +10 -10
- package/commands/failure-recon.md +0 -375
|
@@ -35,9 +35,14 @@ BLOCKING=0
|
|
|
35
35
|
WARNINGS=0
|
|
36
36
|
|
|
37
37
|
# Internal service patterns — requests to these need explicit Host headers
|
|
38
|
-
|
|
38
|
+
# Generic defaults cover localhost, RFC 1918 addresses, common service name prefixes, and IP env vars.
|
|
39
|
+
# Project-specific prefixes (e.g. "myapp-") can be added via FORGE_INTERNAL_PATTERNS (pipe-separated).
|
|
40
|
+
INTERNAL_PATTERNS="localhost|127\.0\.0\.1|api-|worker-|172\.[0-9]+\.[0-9]+\.[0-9]+|\\\$\{?[a-z_]*ip|\\\$\{?[A-Z_]*IP"
|
|
41
|
+
if [ -n "${FORGE_INTERNAL_PATTERNS:-}" ]; then
|
|
42
|
+
INTERNAL_PATTERNS="${INTERNAL_PATTERNS}|${FORGE_INTERNAL_PATTERNS}"
|
|
43
|
+
fi
|
|
39
44
|
|
|
40
|
-
|
|
45
|
+
while read -r f; do
|
|
41
46
|
FILEPATH="$REPO_ROOT/$f"
|
|
42
47
|
if [ ! -f "$FILEPATH" ]; then
|
|
43
48
|
continue
|
|
@@ -52,7 +57,7 @@ echo "$SHELL_FILES" | while read -r f; do
|
|
|
52
57
|
fi
|
|
53
58
|
|
|
54
59
|
echo "=== Checking: $f ==="
|
|
55
|
-
|
|
60
|
+
while IFS= read -r line; do
|
|
56
61
|
LINENO=$(echo "$line" | cut -d: -f1)
|
|
57
62
|
CONTENT=$(echo "$line" | cut -d: -f2-)
|
|
58
63
|
|
|
@@ -67,8 +72,8 @@ echo "$SHELL_FILES" | while read -r f; do
|
|
|
67
72
|
echo " Fix: Add -H 'Host: localhost' (or appropriate allowed host from services/api/app/main.py allowed_hosts)"
|
|
68
73
|
BLOCKING=$((BLOCKING + 1))
|
|
69
74
|
fi
|
|
70
|
-
done
|
|
71
|
-
done || true
|
|
75
|
+
done < <(echo "$CURL_LINES")
|
|
76
|
+
done < <(echo "$SHELL_FILES") || true
|
|
72
77
|
|
|
73
78
|
# --- Client-side proxy bypass check ---
|
|
74
79
|
# Check frontend files for direct /api/v1/ calls (bypasses Next.js auth proxy)
|
|
@@ -77,7 +82,7 @@ FRONTEND_FILES=$(echo "$FILES" | grep -E '\.(tsx?|jsx?)$' | grep -v 'route\.ts$'
|
|
|
77
82
|
if [ -n "$FRONTEND_FILES" ]; then
|
|
78
83
|
echo ""
|
|
79
84
|
echo "=== Client-Side Proxy Bypass Check ==="
|
|
80
|
-
|
|
85
|
+
while read -r f; do
|
|
81
86
|
FILEPATH="$REPO_ROOT/$f"
|
|
82
87
|
if [ ! -f "$FILEPATH" ]; then
|
|
83
88
|
continue
|
|
@@ -92,15 +97,15 @@ if [ -n "$FRONTEND_FILES" ]; then
|
|
|
92
97
|
|
|
93
98
|
if [ -n "$DIRECT_CALLS" ]; then
|
|
94
99
|
echo "BLOCKING: $f — direct /api/v1/ call bypasses Next.js proxy (auth will fail)"
|
|
95
|
-
|
|
100
|
+
while IFS= read -r match; do
|
|
96
101
|
echo " $match"
|
|
97
|
-
done
|
|
102
|
+
done < <(echo "$DIRECT_CALLS")
|
|
98
103
|
echo " Fix: Use /api/* proxy routes instead of /api/v1/* direct FastAPI calls"
|
|
99
104
|
BLOCKING=$((BLOCKING + 1))
|
|
100
105
|
else
|
|
101
106
|
echo "OK: $f — no direct backend calls"
|
|
102
107
|
fi
|
|
103
|
-
done
|
|
108
|
+
done < <(echo "$FRONTEND_FILES")
|
|
104
109
|
fi
|
|
105
110
|
|
|
106
111
|
echo ""
|
|
@@ -27,7 +27,7 @@ BLOCKING=0
|
|
|
27
27
|
WARNINGS=0
|
|
28
28
|
|
|
29
29
|
# --- Next.js Route Handler registration check ---
|
|
30
|
-
|
|
30
|
+
while read -r f; do
|
|
31
31
|
URL_PATH=$(echo "$f" | sed 's|^web/src/app||; s|/route\.ts$||; s|\[.*\]|*|g')
|
|
32
32
|
ROUTE_SEGMENT=$(echo "$URL_PATH" | sed 's|/api/v1/||; s|/.*||')
|
|
33
33
|
|
|
@@ -48,10 +48,10 @@ echo "$FILES" | grep -E "^web/src/app/api/.*route\.ts$" | while read -r f; do
|
|
|
48
48
|
WARNINGS=$((WARNINGS + 1))
|
|
49
49
|
fi
|
|
50
50
|
fi
|
|
51
|
-
done || true
|
|
51
|
+
done < <(echo "$FILES" | grep -E "^web/src/app/api/.*route\.ts$") || true
|
|
52
52
|
|
|
53
53
|
# --- Python API Router registration check ---
|
|
54
|
-
|
|
54
|
+
while read -r f; do
|
|
55
55
|
ROUTER_NAME=$(basename "$f" .py)
|
|
56
56
|
MAIN_PY="$REPO_ROOT/services/api/app/main.py"
|
|
57
57
|
|
|
@@ -66,10 +66,10 @@ echo "$FILES" | grep -E "^services/api/app/routers/.*\.py$" | while read -r f; d
|
|
|
66
66
|
echo "WARNING: Cannot verify router registration — main.py not found at $MAIN_PY"
|
|
67
67
|
WARNINGS=$((WARNINGS + 1))
|
|
68
68
|
fi
|
|
69
|
-
done || true
|
|
69
|
+
done < <(echo "$FILES" | grep -E "^services/api/app/routers/.*\.py$") || true
|
|
70
70
|
|
|
71
71
|
# --- Python Middleware registration check ---
|
|
72
|
-
|
|
72
|
+
while read -r f; do
|
|
73
73
|
MIDDLEWARE_NAME=$(basename "$f" .py)
|
|
74
74
|
MAIN_PY="$REPO_ROOT/services/api/app/main.py"
|
|
75
75
|
|
|
@@ -81,10 +81,10 @@ echo "$FILES" | grep -E "^services/api/app/middleware/.*\.py$" | while read -r f
|
|
|
81
81
|
WARNINGS=$((WARNINGS + 1))
|
|
82
82
|
fi
|
|
83
83
|
fi
|
|
84
|
-
done || true
|
|
84
|
+
done < <(echo "$FILES" | grep -E "^services/api/app/middleware/.*\.py$") || true
|
|
85
85
|
|
|
86
86
|
# --- Shared module import check ---
|
|
87
|
-
|
|
87
|
+
while read -r f; do
|
|
88
88
|
MODULE_NAME=$(basename "$f" .py)
|
|
89
89
|
|
|
90
90
|
IMPORTERS=$(grep -rl "$MODULE_NAME" "$REPO_ROOT/services/api/" "$REPO_ROOT/services/worker/" 2>/dev/null | head -5)
|
|
@@ -94,10 +94,10 @@ echo "$FILES" | grep -E "^shared/.*\.py$" | while read -r f; do
|
|
|
94
94
|
echo "WARNING: Shared module '$MODULE_NAME' ($f) — no imports found in services/api/ or services/worker/"
|
|
95
95
|
WARNINGS=$((WARNINGS + 1))
|
|
96
96
|
fi
|
|
97
|
-
done || true
|
|
97
|
+
done < <(echo "$FILES" | grep -E "^shared/.*\.py$") || true
|
|
98
98
|
|
|
99
99
|
# --- Component import check ---
|
|
100
|
-
|
|
100
|
+
while read -r f; do
|
|
101
101
|
COMPONENT_NAME=$(basename "$f" .tsx)
|
|
102
102
|
|
|
103
103
|
IMPORTERS=$(grep -rl "$COMPONENT_NAME" "$REPO_ROOT/web/src/app/" "$REPO_ROOT/web/src/components/" 2>/dev/null | grep -v "$f" | head -3)
|
|
@@ -107,7 +107,7 @@ echo "$FILES" | grep -E "^web/src/components/.*\.tsx$" | while read -r f; do
|
|
|
107
107
|
echo "WARNING: Component '$COMPONENT_NAME' ($f) — no imports found. May be unused or new."
|
|
108
108
|
WARNINGS=$((WARNINGS + 1))
|
|
109
109
|
fi
|
|
110
|
-
done || true
|
|
110
|
+
done < <(echo "$FILES" | grep -E "^web/src/components/.*\.tsx$") || true
|
|
111
111
|
|
|
112
112
|
# --- Exit code ---
|
|
113
113
|
if [ "$BLOCKING" -gt 0 ]; then
|
|
@@ -1,375 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
description: Weekly failure recon — pull prod failures, test at each tier, create tier-hardening issues
|
|
3
|
-
argument-hint: [full | pull | test | report | issues-only]
|
|
4
|
-
---
|
|
5
|
-
|
|
6
|
-
# /failure-recon — Weekly Scraper Tier Hardening Pipeline
|
|
7
|
-
|
|
8
|
-
**Input**: $ARGUMENTS (default: `full`)
|
|
9
|
-
|
|
10
|
-
You are AlterLab's scraper intelligence system. Your mission: **improve the platform's scraping capabilities holistically** — tiers, detection, content extraction, and the intelligence feedback loop that ties it all together.
|
|
11
|
-
|
|
12
|
-
**The scraping ecosystem is a closed loop:**
|
|
13
|
-
```
|
|
14
|
-
BlackBox Recorder → TrafficAnalyzer → Learnings → LearningApplier → playbook:active:{domain}
|
|
15
|
-
↓
|
|
16
|
-
Tier Feedback (cortex:domain:stats) → get_optimal_starting_tier() → Smarter tier selection
|
|
17
|
-
↓
|
|
18
|
-
Challenge Detection → notify_scrape_complete() → Cortex Queue → Agents (CodeFix, AntiBot, etc.)
|
|
19
|
-
↓
|
|
20
|
-
scrape_diagnostics / domain_analytics → This command (failure-recon) → Issues → /work-on → Improvement
|
|
21
|
-
```
|
|
22
|
-
Every fix to ANY part of this loop compounds — better detection feeds better tier intelligence, which reduces wasted escalations, which captures better traffic data in BlackBox, which generates better learnings.
|
|
23
|
-
|
|
24
|
-
**Agent model policy**: Default `model: "sonnet"`. If Sonnet is rate-limited, fall back to `model: "opus"`. User can override with `--model <name>`. Pass the resolved model in every `Agent`/`Task` call. Each agent prompt specifies the exact tier to test, the Docker endpoint to hit, and the expected output format.
|
|
25
|
-
|
|
26
|
-
**Core principles:**
|
|
27
|
-
1. **Solve at the lowest tier.** Every URL should work at the cheapest, fastest tier possible.
|
|
28
|
-
2. **Platform improvements, not domain fixes.** Cortex + BlackBox Recorder handle per-domain learning automatically. This command improves the PLATFORM — the tiers, detection providers, content extraction, feedback loop, and intelligence pipeline.
|
|
29
|
-
3. **No domain-specific fixes.** NEVER create issues like "fix scraping for zillow.com" or "add playbook for X." Instead: "Tier 3 stealth headers don't spoof sec-fetch-* correctly" (evidenced by failures across 6 domains). Production URLs are EVIDENCE, not the target.
|
|
30
|
-
4. **Leverage the full ecosystem.** Issues should consider:
|
|
31
|
-
- Could BlackBox Recorder capture more useful data to feed this improvement?
|
|
32
|
-
- Could TrafficAnalyzer learn this pattern automatically with better signals?
|
|
33
|
-
- Could the tier feedback loop (`cortex:domain:stats`) make smarter decisions here?
|
|
34
|
-
- Could the LearningApplier auto-apply this fix with the right confidence threshold?
|
|
35
|
-
- Could the challenge detection providers (`shared/detection/providers/`) detect this pattern?
|
|
36
|
-
- Could `detected_antibot` in `scrape_diagnostics` inform future tier selection?
|
|
37
|
-
5. **No regressions.** Every fix must preserve what already works. Check previous fixes.
|
|
38
|
-
6. **GitHub is the memory.** Every issue links to prior work. Any agent picking this up later has full context.
|
|
39
|
-
7. **No defeatism.** "Unsolvable" means "needs a new technique," not "can't be done."
|
|
40
|
-
|
|
41
|
-
**Good issues from this command:**
|
|
42
|
-
- "Tier 2 TLS fingerprint provides 0% incremental bypass — rethink what capability T2 adds" (tier capability)
|
|
43
|
-
- "Browser tier crashes instead of extracting partial content — graceful degradation needed" (resilience)
|
|
44
|
-
- "CSR shell detection doesn't trigger hydration-aware waiting at T4" (content extraction)
|
|
45
|
-
- "detected_antibot never populated — breaking the diagnostics→intelligence feedback loop" (ecosystem plumbing)
|
|
46
|
-
- "Tier feedback loop doesn't account for challenge type — sites with Cloudflare should skip T1-T3" (intelligence)
|
|
47
|
-
- "BlackBox Recorder only captures T4 traffic — T2/T3 failures have no data for TrafficAnalyzer" (data collection gap)
|
|
48
|
-
|
|
49
|
-
**NOT acceptable:**
|
|
50
|
-
- "Fix scraping for zillow.com" (domain-specific — Cortex handles this)
|
|
51
|
-
- "Add playbook for easydrop.one" (domain-specific — CodeFixAgent handles this)
|
|
52
|
-
- "alternativeto.net needs X" (domain-specific — tier feedback loop handles this)
|
|
53
|
-
|
|
54
|
-
---
|
|
55
|
-
|
|
56
|
-
## Phase 1: Pull & Normalize Production Failures
|
|
57
|
-
|
|
58
|
-
First, verify local containers are up:
|
|
59
|
-
```bash
|
|
60
|
-
cd /home/mrdubey/projects/ScraperAPI/alterlab
|
|
61
|
-
docker compose ps --format "{{.Name}} {{.Status}}" | grep -E "api|worker|redis|postgres"
|
|
62
|
-
```
|
|
63
|
-
|
|
64
|
-
If containers aren't running, tell the user and offer to start them.
|
|
65
|
-
|
|
66
|
-
Then run the recon script:
|
|
67
|
-
```bash
|
|
68
|
-
cd /home/mrdubey/projects/ScraperAPI
|
|
69
|
-
|
|
70
|
-
# Full pipeline: pull → normalize → tier-by-tier test → report
|
|
71
|
-
python3 tools/failure-recon/recon.py full --days 7 --sample-size 10 --max-total 200 --concurrency 3
|
|
72
|
-
|
|
73
|
-
# Or specific phases:
|
|
74
|
-
# python3 tools/failure-recon/recon.py pull --days 7 # Pull only (no testing)
|
|
75
|
-
# python3 tools/failure-recon/recon.py test # Re-test existing sample
|
|
76
|
-
# python3 tools/failure-recon/recon.py report # Re-read latest report
|
|
77
|
-
```
|
|
78
|
-
|
|
79
|
-
If `$ARGUMENTS` is `issues-only`, skip directly to Phase 3 (read existing report, create issues).
|
|
80
|
-
|
|
81
|
-
**This takes 30-90 minutes** depending on sample size. The script tests each URL at Tier 1, then 2, then 3, then 4 — stopping at the first tier that returns useful content. This is how we find where each tier's boundary is.
|
|
82
|
-
|
|
83
|
-
---
|
|
84
|
-
|
|
85
|
-
## Phase 2: Analyze the Report
|
|
86
|
-
|
|
87
|
-
Read the latest report:
|
|
88
|
-
```bash
|
|
89
|
-
ls -t /home/mrdubey/projects/ScraperAPI/tools/failure-recon/reports/recon_*.json | head -1
|
|
90
|
-
```
|
|
91
|
-
|
|
92
|
-
Read the JSON. Key sections:
|
|
93
|
-
|
|
94
|
-
### 2A: Tier Pass Rates (`tier_pass_rates`)
|
|
95
|
-
|
|
96
|
-
This is the **scoreboard**. For each tier, what percentage of failed-in-prod URLs can it handle?
|
|
97
|
-
|
|
98
|
-
```
|
|
99
|
-
Tier 1 (curl) : ██░░░░░░░░░░░░░░░░░░ 12/200 (6%)
|
|
100
|
-
Tier 2 (http) : █████░░░░░░░░░░░░░░░ 45/200 (22.5%)
|
|
101
|
-
Tier 3 (stealth) : ████████████░░░░░░░░ 110/200 (55%)
|
|
102
|
-
Tier 4 (browser) : ████████████████░░░░ 160/200 (80%)
|
|
103
|
-
```
|
|
104
|
-
|
|
105
|
-
The GAP between tiers is the hardening opportunity. If Tier 3 solves 55% but Tier 4 solves 80%, there are 50 URLs where Tier 3 ALMOST works — those are Tier 3 hardening candidates.
|
|
106
|
-
|
|
107
|
-
### 2B: Tier Hardening Findings (`tier_findings`)
|
|
108
|
-
|
|
109
|
-
Each finding says: "N URLs fail at Tier X but pass at Tier Y — here's what Tier X needs."
|
|
110
|
-
|
|
111
|
-
**Analyze the root cause for each finding:**
|
|
112
|
-
|
|
113
|
-
1. **What's the detection vector?** Read the errors and protection type.
|
|
114
|
-
- `blocked_403` + Cloudflare → TLS fingerprint or header mismatch at this tier
|
|
115
|
-
- `timeout` + DataDome → Solver not triggering or too slow
|
|
116
|
-
- `empty_response` → Content extraction failing, not necessarily blocked
|
|
117
|
-
- `soft_block` → Page loads but returns challenge HTML instead of content
|
|
118
|
-
|
|
119
|
-
2. **What does the passing tier do differently?** Compare tier capabilities:
|
|
120
|
-
- Tier 1→2: Chrome TLS fingerprint. If URLs pass at T2 but not T1, the site checks TLS.
|
|
121
|
-
- Tier 2→3: Proxy + stealth headers. If URLs pass at T3 but not T2, the site checks IP reputation or specific headers.
|
|
122
|
-
- Tier 3→4: Full browser JS. If URLs pass at T4 but not T3, the site requires JS execution — but CAN we do lighter JS at T3?
|
|
123
|
-
|
|
124
|
-
3. **What specific code change would make the lower tier work?**
|
|
125
|
-
- Better headers at Tier 1/2 (`services/worker/worker/consumers/unified_consumer.py`)
|
|
126
|
-
- Better fingerprinting at Tier 2 (`services/worker/worker/anti_detection/`)
|
|
127
|
-
- Better proxy selection at Tier 3 (proxy pool, country selection)
|
|
128
|
-
- Better challenge detection at Tier 4 (`services/worker/worker/browser/challenge_detector.py`, solvers)
|
|
129
|
-
|
|
130
|
-
### 2C: Unsolvable URLs
|
|
131
|
-
|
|
132
|
-
URLs failing at ALL tiers need new techniques. Analyze:
|
|
133
|
-
- Is the protection system one we have a solver for? (Check `shared/detection/providers/`)
|
|
134
|
-
- Is the solver being triggered? (Check `challenge_detector.py` patterns)
|
|
135
|
-
- Is it a new protection variant we haven't seen?
|
|
136
|
-
|
|
137
|
-
---
|
|
138
|
-
|
|
139
|
-
## Phase 3: Research Prior Work (CRITICAL)
|
|
140
|
-
|
|
141
|
-
**Before creating ANY issue**, search GitHub for prior work in the same area. This is the context layer.
|
|
142
|
-
|
|
143
|
-
```bash
|
|
144
|
-
# Search for existing open scraper issues
|
|
145
|
-
gh issue list --state open --label "scraper" --limit 50 --json number,title
|
|
146
|
-
|
|
147
|
-
# Search for closed issues about the same protection type
|
|
148
|
-
gh issue list --state closed --label "scraper" --search "{protection_type}" --limit 20 --json number,title,closedAt
|
|
149
|
-
|
|
150
|
-
# Search for related PRs
|
|
151
|
-
gh pr list --state merged --search "tier hardening {protection}" --limit 10 --json number,title,mergedAt
|
|
152
|
-
|
|
153
|
-
# Check recent scraper-related commits
|
|
154
|
-
cd /home/mrdubey/projects/ScraperAPI/alterlab
|
|
155
|
-
git log --oneline --all --grep="tier" --grep="hardening" --since="3 months ago" | head -20
|
|
156
|
-
git log --oneline --all --grep="{protection_type}" --since="3 months ago" | head -20
|
|
157
|
-
```
|
|
158
|
-
|
|
159
|
-
**For each finding, gather:**
|
|
160
|
-
1. Related closed issues (what was tried before?)
|
|
161
|
-
2. Related merged PRs (what code was changed?)
|
|
162
|
-
3. Any open issues that overlap (don't duplicate!)
|
|
163
|
-
|
|
164
|
-
If an open issue already covers this finding, **comment on it** with the new test data instead of creating a duplicate.
|
|
165
|
-
|
|
166
|
-
---
|
|
167
|
-
|
|
168
|
-
## Phase 4: Create Platform Improvement Issues
|
|
169
|
-
|
|
170
|
-
**CRITICAL: Issues must describe PLATFORM-LEVEL capabilities to improve, not domain-specific fixes.**
|
|
171
|
-
|
|
172
|
-
Use production data as EVIDENCE (e.g., "observed on 6 domains including zillow.com and coles.com.au"), but the fix must be a general improvement (e.g., "add pre-timeout content extraction fallback to browser tier").
|
|
173
|
-
|
|
174
|
-
Issue categories to create:
|
|
175
|
-
|
|
176
|
-
### A. Tier Capability Gaps
|
|
177
|
-
When a tier has low incremental pass rate over the tier below it. The issue should propose specific anti-detection or protocol improvements to that tier.
|
|
178
|
-
|
|
179
|
-
Title pattern: `"Tier {N} ({name}): {capability problem} — {%} incremental pass rate"`
|
|
180
|
-
|
|
181
|
-
### B. Resilience / Graceful Degradation
|
|
182
|
-
When a tier fails catastrophically (crash, timeout with zero output) instead of returning partial results. The issue should propose fallback behaviors.
|
|
183
|
-
|
|
184
|
-
Title pattern: `"Tier {N}: {failure mode} should {graceful behavior} instead"`
|
|
185
|
-
|
|
186
|
-
### C. Content Extraction Pipeline
|
|
187
|
-
When content exists but isn't extracted properly (CSR shells, shadow DOM, iframes, lazy-loaded content). These are cross-tier issues.
|
|
188
|
-
|
|
189
|
-
Title pattern: `"{extraction problem} — affects {N} URLs across {M} domains"`
|
|
190
|
-
|
|
191
|
-
### D. Detection / Intelligence Pipeline
|
|
192
|
-
When the platform lacks data to make good decisions (e.g., `detected_antibot` not populated, tier intelligence over-escalating). These improve the feedback loop.
|
|
193
|
-
|
|
194
|
-
Title pattern: `"{data/intelligence problem} — blocking {downstream capability}"`
|
|
195
|
-
|
|
196
|
-
### Issue Body Structure
|
|
197
|
-
|
|
198
|
-
Every issue MUST include:
|
|
199
|
-
1. **Platform capability being improved** — what general capability is this? (fingerprinting, content extraction, wait strategy, etc.)
|
|
200
|
-
2. **Evidence from production** — URLs/domains as evidence, NOT as the fix target
|
|
201
|
-
3. **Prior work** — linked issues and PRs
|
|
202
|
-
4. **Investigation path** — code paths to examine, not "fix domain X"
|
|
203
|
-
5. **Validation plan** — how to verify the improvement works across the board, not just for one site
|
|
204
|
-
6. **Regression guard** — URLs that currently pass and must continue to
|
|
205
|
-
|
|
206
|
-
**NEVER frame an issue as "fix scraping for X.com" — always as "improve {capability} (evidenced by failures on X.com, Y.com, Z.com)"**
|
|
207
|
-
- **Prod impact**: {total_failures} failures/week from this protection type
|
|
208
|
-
EOF
|
|
209
|
-
)"
|
|
210
|
-
```
|
|
211
|
-
|
|
212
|
-
### Issue Template: Unsolvable URLs (New Technique Needed)
|
|
213
|
-
|
|
214
|
-
```bash
|
|
215
|
-
gh issue create --title "feat(scraper): add {Protection} bypass technique — {count} URLs unsolvable at all tiers" --label "feature,P1,scraper" --body "$(cat <<'EOF'
|
|
216
|
-
## Problem
|
|
217
|
-
|
|
218
|
-
{count} URLs fail at ALL tiers (1-4) against {Protection} protection. The platform has no working bypass for this protection type or variant.
|
|
219
|
-
|
|
220
|
-
## Root Cause (if known)
|
|
221
|
-
|
|
222
|
-
{Analysis}: Our current solver either doesn't exist for this protection type, exists but doesn't trigger for this pattern, or triggers but can't complete the challenge.
|
|
223
|
-
- `shared/detection/providers/{protection}.py`: {does it detect correctly?}
|
|
224
|
-
- `challenge_detector.py`: {what happens when detected?}
|
|
225
|
-
|
|
226
|
-
## Affected Files
|
|
227
|
-
|
|
228
|
-
Files that need changes:
|
|
229
|
-
1. `services/worker/worker/browser/challenge_detector.py` — {what detection changes are needed}
|
|
230
|
-
2. `shared/detection/providers/{protection}.py` — {solver/detection updates}
|
|
231
|
-
3. `{other files}` — {what needs to change}
|
|
232
|
-
|
|
233
|
-
## Acceptance Criteria
|
|
234
|
-
|
|
235
|
-
- [ ] {Protection} protection detected correctly by challenge_detector.py
|
|
236
|
-
- [ ] Solver triggers and completes challenge for the failing URL pattern
|
|
237
|
-
- [ ] Regression guard: {N} currently-passing URLs continue to pass
|
|
238
|
-
- [ ] Verified in next failure-recon run
|
|
239
|
-
|
|
240
|
-
## Context
|
|
241
|
-
|
|
242
|
-
### Failing URLs
|
|
243
|
-
{table: URL | Domain | T4 Error | T4 Status Code}
|
|
244
|
-
|
|
245
|
-
### Error Pattern
|
|
246
|
-
{Common errors at Tier 4 — the highest tier we tried}
|
|
247
|
-
|
|
248
|
-
### Prior Work
|
|
249
|
-
{Related issues and PRs}
|
|
250
|
-
|
|
251
|
-
### Investigation Path
|
|
252
|
-
1. [ ] Run one URL manually in Playwright to observe the challenge
|
|
253
|
-
2. [ ] Check if detection provider identifies the protection correctly
|
|
254
|
-
3. [ ] Check if solver is triggered (add logging if needed)
|
|
255
|
-
4. [ ] Identify what the challenge requires (fingerprint, behavioral, CAPTCHA, etc.)
|
|
256
|
-
5. [ ] Prototype solver or enhance existing one
|
|
257
|
-
6. [ ] Test with URLs from this issue
|
|
258
|
-
EOF
|
|
259
|
-
)"
|
|
260
|
-
```
|
|
261
|
-
|
|
262
|
-
### Issue Template: Tier Intelligence (Over-Escalation)
|
|
263
|
-
|
|
264
|
-
```bash
|
|
265
|
-
gh issue create --title "refactor(scraper): tier intelligence over-escalation — {count} URLs use Tier {high} but solvable at Tier {low}" --label "refactor,P2,scraper" --body "$(cat <<'EOF'
|
|
266
|
-
## Problem
|
|
267
|
-
|
|
268
|
-
{count} URLs used Tier {high} in production but are solvable at Tier {low} locally. The tier escalation logic or Cortex tier prediction is overshooting by 2+ tiers, wasting browser resources unnecessarily.
|
|
269
|
-
|
|
270
|
-
## Root Cause (if known)
|
|
271
|
-
|
|
272
|
-
{Analysis — is Cortex over-recommending? Is the escalation too aggressive? Are there stale tier recommendations in Redis?}
|
|
273
|
-
|
|
274
|
-
## Affected Files
|
|
275
|
-
|
|
276
|
-
Files that need changes:
|
|
277
|
-
1. `{cortex/tier-selection file}` — {what needs to change in escalation logic}
|
|
278
|
-
2. `{Redis tier cache file}` — {stale recommendation handling if applicable}
|
|
279
|
-
|
|
280
|
-
## Acceptance Criteria
|
|
281
|
-
|
|
282
|
-
- [ ] Tier escalation for {protection type} reduced by {N} tiers on average
|
|
283
|
-
- [ ] No regression: URLs that genuinely need higher tiers still escalate correctly
|
|
284
|
-
- [ ] Verified in next failure-recon run
|
|
285
|
-
|
|
286
|
-
## Context
|
|
287
|
-
|
|
288
|
-
### Examples
|
|
289
|
-
| URL | Prod Tier | Actual Min Tier | Tiers Saved |
|
|
290
|
-
|-----|-----------|-----------------|-------------|
|
|
291
|
-
{table rows}
|
|
292
|
-
|
|
293
|
-
### Prior Work
|
|
294
|
-
{Related issues}
|
|
295
|
-
EOF
|
|
296
|
-
)"
|
|
297
|
-
```
|
|
298
|
-
|
|
299
|
-
---
|
|
300
|
-
|
|
301
|
-
## Phase 5: Link Everything Together
|
|
302
|
-
|
|
303
|
-
After creating issues, ensure the context web is connected:
|
|
304
|
-
|
|
305
|
-
1. **Cross-reference new issues** — if Issue A (Tier 2 Cloudflare) and Issue B (Tier 3 Cloudflare) are related, comment on each linking to the other.
|
|
306
|
-
|
|
307
|
-
2. **Reference the recon report** — include the report timestamp in each issue so agents know which data to look at:
|
|
308
|
-
```
|
|
309
|
-
> Data from failure-recon run {timestamp}. Report: `tools/failure-recon/reports/recon_{timestamp}.json`
|
|
310
|
-
```
|
|
311
|
-
|
|
312
|
-
3. **Label consistently**: All issues get `scraper` label. Tier hardening gets `bug`. New techniques get `feature`.
|
|
313
|
-
|
|
314
|
-
---
|
|
315
|
-
|
|
316
|
-
## Phase 6: Summary
|
|
317
|
-
|
|
318
|
-
Print the final summary:
|
|
319
|
-
|
|
320
|
-
```
|
|
321
|
-
## Failure Recon — Tier Hardening Report ({date})
|
|
322
|
-
|
|
323
|
-
### Production Snapshot (Last 7 Days)
|
|
324
|
-
- Total failures: {N} | Unique URLs: {N} | Domains: {N}
|
|
325
|
-
|
|
326
|
-
### Tier Scoreboard
|
|
327
|
-
| Tier | Name | Passing | Rate | Hardening Opportunity |
|
|
328
|
-
|------|------|---------|------|----------------------|
|
|
329
|
-
| 1 | curl | {n}/{total} | {%} | {gap to next tier} URLs |
|
|
330
|
-
| 2 | http | {n}/{total} | {%} | {gap} URLs |
|
|
331
|
-
| 3 | stealth | {n}/{total} | {%} | {gap} URLs |
|
|
332
|
-
| 4 | browser | {n}/{total} | {%} | — |
|
|
333
|
-
| ALL | combined | {n}/{total} | {%} | {unsolvable} unsolvable |
|
|
334
|
-
|
|
335
|
-
### Issues Created
|
|
336
|
-
| # | Title | Priority | Tier | Protection | Hardening Target |
|
|
337
|
-
|---|-------|----------|------|------------|-----------------|
|
|
338
|
-
{table}
|
|
339
|
-
|
|
340
|
-
### Issues Updated (prior work found)
|
|
341
|
-
| # | Title | New Data Added |
|
|
342
|
-
|---|-------|---------------|
|
|
343
|
-
{table of existing issues that got new comments}
|
|
344
|
-
|
|
345
|
-
### Regression Watch
|
|
346
|
-
{N} URLs currently passing — these MUST be preserved. Any /work-on agent picking up
|
|
347
|
-
a tier-hardening issue should verify these still pass before merging.
|
|
348
|
-
```
|
|
349
|
-
|
|
350
|
-
---
|
|
351
|
-
|
|
352
|
-
## Modes Reference
|
|
353
|
-
|
|
354
|
-
| Argument | What It Does | Duration | When |
|
|
355
|
-
|----------|-------------|----------|------|
|
|
356
|
-
| `full` (default) | Pull → Test all tiers → Report → Issues | 30-90 min | Weekly ritual |
|
|
357
|
-
| `pull` | Pull + normalize only | 2 min | Quick status check |
|
|
358
|
-
| `test` | Re-test existing sample | 20-60 min | After deploying fixes |
|
|
359
|
-
| `report` | Re-read latest report | Instant | Review without re-pulling |
|
|
360
|
-
| `issues-only` | Read report → Create issues | 5 min | Create issues from reviewed report |
|
|
361
|
-
|
|
362
|
-
---
|
|
363
|
-
|
|
364
|
-
## Philosophy
|
|
365
|
-
|
|
366
|
-
AlterLab doesn't accept failure. Every bot protection is a puzzle — Cloudflare, DataDome, Akamai, Kasada, all of them. The question isn't "can we bypass this?" — it's "which tier should bypass this?"
|
|
367
|
-
|
|
368
|
-
**Tier 1 should handle everything that doesn't check fingerprints.**
|
|
369
|
-
**Tier 2 should handle everything that checks TLS but not IP.**
|
|
370
|
-
**Tier 3 should handle everything that checks IP but not JS.**
|
|
371
|
-
**Tier 4 should handle everything.**
|
|
372
|
-
|
|
373
|
-
When a URL fails at Tier 4, that's not a limitation — that's a solver bug or a new pattern to learn.
|
|
374
|
-
|
|
375
|
-
This command runs weekly. Each run should produce measurable tier improvements. Compare this week's scoreboard to last week's — the numbers should go UP, never down.
|