forgedock 1.0.0 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/README.md +19 -2
  2. package/bin/forgedock.mjs +286 -10
  3. package/commands/analytics.md +129 -52
  4. package/commands/audit-agents.md +29 -5
  5. package/commands/audit.md +55 -17
  6. package/commands/autopilot.md +11 -6
  7. package/commands/cleanup.md +44 -24
  8. package/commands/deploy-info.md +7 -3
  9. package/commands/forge-stats.md +13 -4
  10. package/commands/forgedock-init.md +534 -0
  11. package/commands/geo-audit.md +114 -26
  12. package/commands/incident-response.md +4 -1
  13. package/commands/issue.md +26 -25
  14. package/commands/milestone.md +80 -59
  15. package/commands/orchestrate.md +38 -21
  16. package/commands/pipeline-health.md +26 -15
  17. package/commands/qa-sweep.md +68 -5
  18. package/commands/quality-gate.md +18 -0
  19. package/commands/review-pr-agents.md +58 -90
  20. package/commands/review-pr.md +58 -16
  21. package/commands/rollback.md +8 -5
  22. package/commands/security-audit.md +68 -12
  23. package/commands/sync-ecosystem.md +196 -156
  24. package/commands/validate.md +1 -1
  25. package/commands/work-on/build/architect.md +2 -2
  26. package/commands/work-on/build/context.md +2 -2
  27. package/commands/work-on/build/implement.md +3 -3
  28. package/commands/work-on/build/validate.md +3 -3
  29. package/commands/work-on/build.md +6 -6
  30. package/commands/work-on/close.md +59 -30
  31. package/commands/work-on/decompose.md +6 -6
  32. package/commands/work-on/investigate.md +16 -20
  33. package/commands/work-on/review.md +4 -4
  34. package/commands/work-on-monolithic.md +15 -16
  35. package/commands/work-on.md +45 -33
  36. package/docs/CODEX.md +1 -1
  37. package/docs/CONFIG.md +398 -0
  38. package/docs/WORKFLOW.md +115 -72
  39. package/package.json +2 -2
  40. package/scripts/verify-host-headers.sh +14 -9
  41. package/scripts/verify-route-registration.sh +10 -10
  42. package/commands/failure-recon.md +0 -375
@@ -35,9 +35,14 @@ BLOCKING=0
35
35
  WARNINGS=0
36
36
 
37
37
  # Internal service patterns — requests to these need explicit Host headers
38
- INTERNAL_PATTERNS="localhost|127\.0\.0\.1|api-|worker-|alterlab-|172\.[0-9]+\.[0-9]+\.[0-9]+|\\\$\{?[a-z_]*ip|\\\$\{?[A-Z_]*IP"
38
+ # Generic defaults cover localhost, RFC 1918 addresses, common service name prefixes, and IP env vars.
39
+ # Project-specific prefixes (e.g. "myapp-") can be added via FORGE_INTERNAL_PATTERNS (pipe-separated).
40
+ INTERNAL_PATTERNS="localhost|127\.0\.0\.1|api-|worker-|172\.[0-9]+\.[0-9]+\.[0-9]+|\\\$\{?[a-z_]*ip|\\\$\{?[A-Z_]*IP"
41
+ if [ -n "${FORGE_INTERNAL_PATTERNS:-}" ]; then
42
+ INTERNAL_PATTERNS="${INTERNAL_PATTERNS}|${FORGE_INTERNAL_PATTERNS}"
43
+ fi
39
44
 
40
- echo "$SHELL_FILES" | while read -r f; do
45
+ while read -r f; do
41
46
  FILEPATH="$REPO_ROOT/$f"
42
47
  if [ ! -f "$FILEPATH" ]; then
43
48
  continue
@@ -52,7 +57,7 @@ echo "$SHELL_FILES" | while read -r f; do
52
57
  fi
53
58
 
54
59
  echo "=== Checking: $f ==="
55
- echo "$CURL_LINES" | while IFS= read -r line; do
60
+ while IFS= read -r line; do
56
61
  LINENO=$(echo "$line" | cut -d: -f1)
57
62
  CONTENT=$(echo "$line" | cut -d: -f2-)
58
63
 
@@ -67,8 +72,8 @@ echo "$SHELL_FILES" | while read -r f; do
67
72
  echo " Fix: Add -H 'Host: localhost' (or appropriate allowed host from services/api/app/main.py allowed_hosts)"
68
73
  BLOCKING=$((BLOCKING + 1))
69
74
  fi
70
- done
71
- done || true
75
+ done < <(echo "$CURL_LINES")
76
+ done < <(echo "$SHELL_FILES") || true
72
77
 
73
78
  # --- Client-side proxy bypass check ---
74
79
  # Check frontend files for direct /api/v1/ calls (bypasses Next.js auth proxy)
@@ -77,7 +82,7 @@ FRONTEND_FILES=$(echo "$FILES" | grep -E '\.(tsx?|jsx?)$' | grep -v 'route\.ts$'
77
82
  if [ -n "$FRONTEND_FILES" ]; then
78
83
  echo ""
79
84
  echo "=== Client-Side Proxy Bypass Check ==="
80
- echo "$FRONTEND_FILES" | while read -r f; do
85
+ while read -r f; do
81
86
  FILEPATH="$REPO_ROOT/$f"
82
87
  if [ ! -f "$FILEPATH" ]; then
83
88
  continue
@@ -92,15 +97,15 @@ if [ -n "$FRONTEND_FILES" ]; then
92
97
 
93
98
  if [ -n "$DIRECT_CALLS" ]; then
94
99
  echo "BLOCKING: $f — direct /api/v1/ call bypasses Next.js proxy (auth will fail)"
95
- echo "$DIRECT_CALLS" | while IFS= read -r match; do
100
+ while IFS= read -r match; do
96
101
  echo " $match"
97
- done
102
+ done < <(echo "$DIRECT_CALLS")
98
103
  echo " Fix: Use /api/* proxy routes instead of /api/v1/* direct FastAPI calls"
99
104
  BLOCKING=$((BLOCKING + 1))
100
105
  else
101
106
  echo "OK: $f — no direct backend calls"
102
107
  fi
103
- done
108
+ done < <(echo "$FRONTEND_FILES")
104
109
  fi
105
110
 
106
111
  echo ""
@@ -27,7 +27,7 @@ BLOCKING=0
27
27
  WARNINGS=0
28
28
 
29
29
  # --- Next.js Route Handler registration check ---
30
- echo "$FILES" | grep -E "^web/src/app/api/.*route\.ts$" | while read -r f; do
30
+ while read -r f; do
31
31
  URL_PATH=$(echo "$f" | sed 's|^web/src/app||; s|/route\.ts$||; s|\[.*\]|*|g')
32
32
  ROUTE_SEGMENT=$(echo "$URL_PATH" | sed 's|/api/v1/||; s|/.*||')
33
33
 
@@ -48,10 +48,10 @@ echo "$FILES" | grep -E "^web/src/app/api/.*route\.ts$" | while read -r f; do
48
48
  WARNINGS=$((WARNINGS + 1))
49
49
  fi
50
50
  fi
51
- done || true
51
+ done < <(echo "$FILES" | grep -E "^web/src/app/api/.*route\.ts$") || true
52
52
 
53
53
  # --- Python API Router registration check ---
54
- echo "$FILES" | grep -E "^services/api/app/routers/.*\.py$" | while read -r f; do
54
+ while read -r f; do
55
55
  ROUTER_NAME=$(basename "$f" .py)
56
56
  MAIN_PY="$REPO_ROOT/services/api/app/main.py"
57
57
 
@@ -66,10 +66,10 @@ echo "$FILES" | grep -E "^services/api/app/routers/.*\.py$" | while read -r f; d
66
66
  echo "WARNING: Cannot verify router registration — main.py not found at $MAIN_PY"
67
67
  WARNINGS=$((WARNINGS + 1))
68
68
  fi
69
- done || true
69
+ done < <(echo "$FILES" | grep -E "^services/api/app/routers/.*\.py$") || true
70
70
 
71
71
  # --- Python Middleware registration check ---
72
- echo "$FILES" | grep -E "^services/api/app/middleware/.*\.py$" | while read -r f; do
72
+ while read -r f; do
73
73
  MIDDLEWARE_NAME=$(basename "$f" .py)
74
74
  MAIN_PY="$REPO_ROOT/services/api/app/main.py"
75
75
 
@@ -81,10 +81,10 @@ echo "$FILES" | grep -E "^services/api/app/middleware/.*\.py$" | while read -r f
81
81
  WARNINGS=$((WARNINGS + 1))
82
82
  fi
83
83
  fi
84
- done || true
84
+ done < <(echo "$FILES" | grep -E "^services/api/app/middleware/.*\.py$") || true
85
85
 
86
86
  # --- Shared module import check ---
87
- echo "$FILES" | grep -E "^shared/.*\.py$" | while read -r f; do
87
+ while read -r f; do
88
88
  MODULE_NAME=$(basename "$f" .py)
89
89
 
90
90
  IMPORTERS=$(grep -rl "$MODULE_NAME" "$REPO_ROOT/services/api/" "$REPO_ROOT/services/worker/" 2>/dev/null | head -5)
@@ -94,10 +94,10 @@ echo "$FILES" | grep -E "^shared/.*\.py$" | while read -r f; do
94
94
  echo "WARNING: Shared module '$MODULE_NAME' ($f) — no imports found in services/api/ or services/worker/"
95
95
  WARNINGS=$((WARNINGS + 1))
96
96
  fi
97
- done || true
97
+ done < <(echo "$FILES" | grep -E "^shared/.*\.py$") || true
98
98
 
99
99
  # --- Component import check ---
100
- echo "$FILES" | grep -E "^web/src/components/.*\.tsx$" | while read -r f; do
100
+ while read -r f; do
101
101
  COMPONENT_NAME=$(basename "$f" .tsx)
102
102
 
103
103
  IMPORTERS=$(grep -rl "$COMPONENT_NAME" "$REPO_ROOT/web/src/app/" "$REPO_ROOT/web/src/components/" 2>/dev/null | grep -v "$f" | head -3)
@@ -107,7 +107,7 @@ echo "$FILES" | grep -E "^web/src/components/.*\.tsx$" | while read -r f; do
107
107
  echo "WARNING: Component '$COMPONENT_NAME' ($f) — no imports found. May be unused or new."
108
108
  WARNINGS=$((WARNINGS + 1))
109
109
  fi
110
- done || true
110
+ done < <(echo "$FILES" | grep -E "^web/src/components/.*\.tsx$") || true
111
111
 
112
112
  # --- Exit code ---
113
113
  if [ "$BLOCKING" -gt 0 ]; then
@@ -1,375 +0,0 @@
1
- ---
2
- description: Weekly failure recon — pull prod failures, test at each tier, create tier-hardening issues
3
- argument-hint: [full | pull | test | report | issues-only]
4
- ---
5
-
6
- # /failure-recon — Weekly Scraper Tier Hardening Pipeline
7
-
8
- **Input**: $ARGUMENTS (default: `full`)
9
-
10
- You are AlterLab's scraper intelligence system. Your mission: **improve the platform's scraping capabilities holistically** — tiers, detection, content extraction, and the intelligence feedback loop that ties it all together.
11
-
12
- **The scraping ecosystem is a closed loop:**
13
- ```
14
- BlackBox Recorder → TrafficAnalyzer → Learnings → LearningApplier → playbook:active:{domain}
15
- ↓
16
- Tier Feedback (cortex:domain:stats) → get_optimal_starting_tier() → Smarter tier selection
17
- ↓
18
- Challenge Detection → notify_scrape_complete() → Cortex Queue → Agents (CodeFix, AntiBot, etc.)
19
- ↓
20
- scrape_diagnostics / domain_analytics → This command (failure-recon) → Issues → /work-on → Improvement
21
- ```
22
- Every fix to ANY part of this loop compounds — better detection feeds better tier intelligence, which reduces wasted escalations, which captures better traffic data in BlackBox, which generates better learnings.
23
-
24
- **Agent model policy**: Default `model: "sonnet"`. If Sonnet is rate-limited, fall back to `model: "opus"`. User can override with `--model <name>`. Pass the resolved model in every `Agent`/`Task` call. Each agent prompt specifies the exact tier to test, the Docker endpoint to hit, and the expected output format.
25
-
26
- **Core principles:**
27
- 1. **Solve at the lowest tier.** Every URL should work at the cheapest, fastest tier possible.
28
- 2. **Platform improvements, not domain fixes.** Cortex + BlackBox Recorder handle per-domain learning automatically. This command improves the PLATFORM — the tiers, detection providers, content extraction, feedback loop, and intelligence pipeline.
29
- 3. **No domain-specific fixes.** NEVER create issues like "fix scraping for zillow.com" or "add playbook for X." Instead: "Tier 3 stealth headers don't spoof sec-fetch-* correctly" (evidenced by failures across 6 domains). Production URLs are EVIDENCE, not the target.
30
- 4. **Leverage the full ecosystem.** Issues should consider:
31
- - Could BlackBox Recorder capture more useful data to feed this improvement?
32
- - Could TrafficAnalyzer learn this pattern automatically with better signals?
33
- - Could the tier feedback loop (`cortex:domain:stats`) make smarter decisions here?
34
- - Could the LearningApplier auto-apply this fix with the right confidence threshold?
35
- - Could the challenge detection providers (`shared/detection/providers/`) detect this pattern?
36
- - Could `detected_antibot` in `scrape_diagnostics` inform future tier selection?
37
- 5. **No regressions.** Every fix must preserve what already works. Check previous fixes.
38
- 6. **GitHub is the memory.** Every issue links to prior work. Any agent picking this up later has full context.
39
- 7. **No defeatism.** "Unsolvable" means "needs a new technique," not "can't be done."
40
-
41
- **Good issues from this command:**
42
- - "Tier 2 TLS fingerprint provides 0% incremental bypass — rethink what capability T2 adds" (tier capability)
43
- - "Browser tier crashes instead of extracting partial content — graceful degradation needed" (resilience)
44
- - "CSR shell detection doesn't trigger hydration-aware waiting at T4" (content extraction)
45
- - "detected_antibot never populated — breaking the diagnostics→intelligence feedback loop" (ecosystem plumbing)
46
- - "Tier feedback loop doesn't account for challenge type — sites with Cloudflare should skip T1-T3" (intelligence)
47
- - "BlackBox Recorder only captures T4 traffic — T2/T3 failures have no data for TrafficAnalyzer" (data collection gap)
48
-
49
- **NOT acceptable:**
50
- - "Fix scraping for zillow.com" (domain-specific — Cortex handles this)
51
- - "Add playbook for easydrop.one" (domain-specific — CodeFixAgent handles this)
52
- - "alternativeto.net needs X" (domain-specific — tier feedback loop handles this)
53
-
54
- ---
55
-
56
- ## Phase 1: Pull & Normalize Production Failures
57
-
58
- First, verify local containers are up:
59
- ```bash
60
- cd /home/mrdubey/projects/ScraperAPI/alterlab
61
- docker compose ps --format "{{.Name}} {{.Status}}" | grep -E "api|worker|redis|postgres"
62
- ```
63
-
64
- If containers aren't running, tell the user and offer to start them.
65
-
66
- Then run the recon script:
67
- ```bash
68
- cd /home/mrdubey/projects/ScraperAPI
69
-
70
- # Full pipeline: pull → normalize → tier-by-tier test → report
71
- python3 tools/failure-recon/recon.py full --days 7 --sample-size 10 --max-total 200 --concurrency 3
72
-
73
- # Or specific phases:
74
- # python3 tools/failure-recon/recon.py pull --days 7 # Pull only (no testing)
75
- # python3 tools/failure-recon/recon.py test # Re-test existing sample
76
- # python3 tools/failure-recon/recon.py report # Re-read latest report
77
- ```
78
-
79
- If `$ARGUMENTS` is `issues-only`, skip directly to Phase 3 (read existing report, create issues).
80
-
81
- **This takes 30-90 minutes** depending on sample size. The script tests each URL at Tier 1, then 2, then 3, then 4 — stopping at the first tier that returns useful content. This is how we find where each tier's boundary is.
82
-
83
- ---
84
-
85
- ## Phase 2: Analyze the Report
86
-
87
- Read the latest report:
88
- ```bash
89
- ls -t /home/mrdubey/projects/ScraperAPI/tools/failure-recon/reports/recon_*.json | head -1
90
- ```
91
-
92
- Read the JSON. Key sections:
93
-
94
- ### 2A: Tier Pass Rates (`tier_pass_rates`)
95
-
96
- This is the **scoreboard**. For each tier, what percentage of failed-in-prod URLs can it handle?
97
-
98
- ```
99
- Tier 1 (curl) : ██░░░░░░░░░░░░░░░░░░ 12/200 (6%)
100
- Tier 2 (http) : █████░░░░░░░░░░░░░░░ 45/200 (22.5%)
101
- Tier 3 (stealth) : ████████████░░░░░░░░ 110/200 (55%)
102
- Tier 4 (browser) : ████████████████░░░░ 160/200 (80%)
103
- ```
104
-
105
- The GAP between tiers is the hardening opportunity. If Tier 3 solves 55% but Tier 4 solves 80%, there are 50 URLs where Tier 3 ALMOST works — those are Tier 3 hardening candidates.
106
-
107
- ### 2B: Tier Hardening Findings (`tier_findings`)
108
-
109
- Each finding says: "N URLs fail at Tier X but pass at Tier Y — here's what Tier X needs."
110
-
111
- **Analyze the root cause for each finding:**
112
-
113
- 1. **What's the detection vector?** Read the errors and protection type.
114
- - `blocked_403` + Cloudflare → TLS fingerprint or header mismatch at this tier
115
- - `timeout` + DataDome → Solver not triggering or too slow
116
- - `empty_response` → Content extraction failing, not necessarily blocked
117
- - `soft_block` → Page loads but returns challenge HTML instead of content
118
-
119
- 2. **What does the passing tier do differently?** Compare tier capabilities:
120
- - Tier 1→2: Chrome TLS fingerprint. If URLs pass at T2 but not T1, the site checks TLS.
121
- - Tier 2→3: Proxy + stealth headers. If URLs pass at T3 but not T2, the site checks IP reputation or specific headers.
122
- - Tier 3→4: Full browser JS. If URLs pass at T4 but not T3, the site requires JS execution — but CAN we do lighter JS at T3?
123
-
124
- 3. **What specific code change would make the lower tier work?**
125
- - Better headers at Tier 1/2 (`services/worker/worker/consumers/unified_consumer.py`)
126
- - Better fingerprinting at Tier 2 (`services/worker/worker/anti_detection/`)
127
- - Better proxy selection at Tier 3 (proxy pool, country selection)
128
- - Better challenge detection at Tier 4 (`services/worker/worker/browser/challenge_detector.py`, solvers)
129
-
130
- ### 2C: Unsolvable URLs
131
-
132
- URLs failing at ALL tiers need new techniques. Analyze:
133
- - Is the protection system one we have a solver for? (Check `shared/detection/providers/`)
134
- - Is the solver being triggered? (Check `challenge_detector.py` patterns)
135
- - Is it a new protection variant we haven't seen?
136
-
137
- ---
138
-
139
- ## Phase 3: Research Prior Work (CRITICAL)
140
-
141
- **Before creating ANY issue**, search GitHub for prior work in the same area. This is the context layer.
142
-
143
- ```bash
144
- # Search for existing open scraper issues
145
- gh issue list --state open --label "scraper" --limit 50 --json number,title
146
-
147
- # Search for closed issues about the same protection type
148
- gh issue list --state closed --label "scraper" --search "{protection_type}" --limit 20 --json number,title,closedAt
149
-
150
- # Search for related PRs
151
- gh pr list --state merged --search "tier hardening {protection}" --limit 10 --json number,title,mergedAt
152
-
153
- # Check recent scraper-related commits
154
- cd /home/mrdubey/projects/ScraperAPI/alterlab
155
- git log --oneline --all --grep="tier" --grep="hardening" --since="3 months ago" | head -20
156
- git log --oneline --all --grep="{protection_type}" --since="3 months ago" | head -20
157
- ```
158
-
159
- **For each finding, gather:**
160
- 1. Related closed issues (what was tried before?)
161
- 2. Related merged PRs (what code was changed?)
162
- 3. Any open issues that overlap (don't duplicate!)
163
-
164
- If an open issue already covers this finding, **comment on it** with the new test data instead of creating a duplicate.
165
-
166
- ---
167
-
168
- ## Phase 4: Create Platform Improvement Issues
169
-
170
- **CRITICAL: Issues must describe PLATFORM-LEVEL capabilities to improve, not domain-specific fixes.**
171
-
172
- Use production data as EVIDENCE (e.g., "observed on 6 domains including zillow.com and coles.com.au"), but the fix must be a general improvement (e.g., "add pre-timeout content extraction fallback to browser tier").
173
-
174
- Issue categories to create:
175
-
176
- ### A. Tier Capability Gaps
177
- When a tier has low incremental pass rate over the tier below it. The issue should propose specific anti-detection or protocol improvements to that tier.
178
-
179
- Title pattern: `"Tier {N} ({name}): {capability problem} — {%} incremental pass rate"`
180
-
181
- ### B. Resilience / Graceful Degradation
182
- When a tier fails catastrophically (crash, timeout with zero output) instead of returning partial results. The issue should propose fallback behaviors.
183
-
184
- Title pattern: `"Tier {N}: {failure mode} should {graceful behavior} instead"`
185
-
186
- ### C. Content Extraction Pipeline
187
- When content exists but isn't extracted properly (CSR shells, shadow DOM, iframes, lazy-loaded content). These are cross-tier issues.
188
-
189
- Title pattern: `"{extraction problem} — affects {N} URLs across {M} domains"`
190
-
191
- ### D. Detection / Intelligence Pipeline
192
- When the platform lacks data to make good decisions (e.g., `detected_antibot` not populated, tier intelligence over-escalating). These improve the feedback loop.
193
-
194
- Title pattern: `"{data/intelligence problem} — blocking {downstream capability}"`
195
-
196
- ### Issue Body Structure
197
-
198
- Every issue MUST include:
199
- 1. **Platform capability being improved** — what general capability is this? (fingerprinting, content extraction, wait strategy, etc.)
200
- 2. **Evidence from production** — URLs/domains as evidence, NOT as the fix target
201
- 3. **Prior work** — linked issues and PRs
202
- 4. **Investigation path** — code paths to examine, not "fix domain X"
203
- 5. **Validation plan** — how to verify the improvement works across the board, not just for one site
204
- 6. **Regression guard** — URLs that currently pass and must continue to
205
-
206
- **NEVER frame an issue as "fix scraping for X.com" — always as "improve {capability} (evidenced by failures on X.com, Y.com, Z.com)"**
207
- - **Prod impact**: {total_failures} failures/week from this protection type
208
- EOF
209
- )"
210
- ```
211
-
212
- ### Issue Template: Unsolvable URLs (New Technique Needed)
213
-
214
- ```bash
215
- gh issue create --title "feat(scraper): add {Protection} bypass technique — {count} URLs unsolvable at all tiers" --label "feature,P1,scraper" --body "$(cat <<'EOF'
216
- ## Problem
217
-
218
- {count} URLs fail at ALL tiers (1-4) against {Protection} protection. The platform has no working bypass for this protection type or variant.
219
-
220
- ## Root Cause (if known)
221
-
222
- {Analysis}: Our current solver either doesn't exist for this protection type, exists but doesn't trigger for this pattern, or triggers but can't complete the challenge.
223
- - `shared/detection/providers/{protection}.py`: {does it detect correctly?}
224
- - `challenge_detector.py`: {what happens when detected?}
225
-
226
- ## Affected Files
227
-
228
- Files that need changes:
229
- 1. `services/worker/worker/browser/challenge_detector.py` — {what detection changes are needed}
230
- 2. `shared/detection/providers/{protection}.py` — {solver/detection updates}
231
- 3. `{other files}` — {what needs to change}
232
-
233
- ## Acceptance Criteria
234
-
235
- - [ ] {Protection} protection detected correctly by challenge_detector.py
236
- - [ ] Solver triggers and completes challenge for the failing URL pattern
237
- - [ ] Regression guard: {N} currently-passing URLs continue to pass
238
- - [ ] Verified in next failure-recon run
239
-
240
- ## Context
241
-
242
- ### Failing URLs
243
- {table: URL | Domain | T4 Error | T4 Status Code}
244
-
245
- ### Error Pattern
246
- {Common errors at Tier 4 — the highest tier we tried}
247
-
248
- ### Prior Work
249
- {Related issues and PRs}
250
-
251
- ### Investigation Path
252
- 1. [ ] Run one URL manually in Playwright to observe the challenge
253
- 2. [ ] Check if detection provider identifies the protection correctly
254
- 3. [ ] Check if solver is triggered (add logging if needed)
255
- 4. [ ] Identify what the challenge requires (fingerprint, behavioral, CAPTCHA, etc.)
256
- 5. [ ] Prototype solver or enhance existing one
257
- 6. [ ] Test with URLs from this issue
258
- EOF
259
- )"
260
- ```
261
-
262
- ### Issue Template: Tier Intelligence (Over-Escalation)
263
-
264
- ```bash
265
- gh issue create --title "refactor(scraper): tier intelligence over-escalation — {count} URLs use Tier {high} but solvable at Tier {low}" --label "refactor,P2,scraper" --body "$(cat <<'EOF'
266
- ## Problem
267
-
268
- {count} URLs used Tier {high} in production but are solvable at Tier {low} locally. The tier escalation logic or Cortex tier prediction is overshooting by 2+ tiers, wasting browser resources unnecessarily.
269
-
270
- ## Root Cause (if known)
271
-
272
- {Analysis — is Cortex over-recommending? Is the escalation too aggressive? Are there stale tier recommendations in Redis?}
273
-
274
- ## Affected Files
275
-
276
- Files that need changes:
277
- 1. `{cortex/tier-selection file}` — {what needs to change in escalation logic}
278
- 2. `{Redis tier cache file}` — {stale recommendation handling if applicable}
279
-
280
- ## Acceptance Criteria
281
-
282
- - [ ] Tier escalation for {protection type} reduced by {N} tiers on average
283
- - [ ] No regression: URLs that genuinely need higher tiers still escalate correctly
284
- - [ ] Verified in next failure-recon run
285
-
286
- ## Context
287
-
288
- ### Examples
289
- | URL | Prod Tier | Actual Min Tier | Tiers Saved |
290
- |-----|-----------|-----------------|-------------|
291
- {table rows}
292
-
293
- ### Prior Work
294
- {Related issues}
295
- EOF
296
- )"
297
- ```
298
-
299
- ---
300
-
301
- ## Phase 5: Link Everything Together
302
-
303
- After creating issues, ensure the context web is connected:
304
-
305
- 1. **Cross-reference new issues** — if Issue A (Tier 2 Cloudflare) and Issue B (Tier 3 Cloudflare) are related, comment on each linking to the other.
306
-
307
- 2. **Reference the recon report** — include the report timestamp in each issue so agents know which data to look at:
308
- ```
309
- > Data from failure-recon run {timestamp}. Report: `tools/failure-recon/reports/recon_{timestamp}.json`
310
- ```
311
-
312
- 3. **Label consistently**: All issues get `scraper` label. Tier hardening gets `bug`. New techniques get `feature`.
313
-
314
- ---
315
-
316
- ## Phase 6: Summary
317
-
318
- Print the final summary:
319
-
320
- ```
321
- ## Failure Recon — Tier Hardening Report ({date})
322
-
323
- ### Production Snapshot (Last 7 Days)
324
- - Total failures: {N} | Unique URLs: {N} | Domains: {N}
325
-
326
- ### Tier Scoreboard
327
- | Tier | Name | Passing | Rate | Hardening Opportunity |
328
- |------|------|---------|------|----------------------|
329
- | 1 | curl | {n}/{total} | {%} | {gap to next tier} URLs |
330
- | 2 | http | {n}/{total} | {%} | {gap} URLs |
331
- | 3 | stealth | {n}/{total} | {%} | {gap} URLs |
332
- | 4 | browser | {n}/{total} | {%} | — |
333
- | ALL | combined | {n}/{total} | {%} | {unsolvable} unsolvable |
334
-
335
- ### Issues Created
336
- | # | Title | Priority | Tier | Protection | Hardening Target |
337
- |---|-------|----------|------|------------|-----------------|
338
- {table}
339
-
340
- ### Issues Updated (prior work found)
341
- | # | Title | New Data Added |
342
- |---|-------|---------------|
343
- {table of existing issues that got new comments}
344
-
345
- ### Regression Watch
346
- {N} URLs currently passing — these MUST be preserved. Any /work-on agent picking up
347
- a tier-hardening issue should verify these still pass before merging.
348
- ```
349
-
350
- ---
351
-
352
- ## Modes Reference
353
-
354
- | Argument | What It Does | Duration | When |
355
- |----------|-------------|----------|------|
356
- | `full` (default) | Pull → Test all tiers → Report → Issues | 30-90 min | Weekly ritual |
357
- | `pull` | Pull + normalize only | 2 min | Quick status check |
358
- | `test` | Re-test existing sample | 20-60 min | After deploying fixes |
359
- | `report` | Re-read latest report | Instant | Review without re-pulling |
360
- | `issues-only` | Read report → Create issues | 5 min | Create issues from reviewed report |
361
-
362
- ---
363
-
364
- ## Philosophy
365
-
366
- AlterLab doesn't accept failure. Every bot protection is a puzzle — Cloudflare, DataDome, Akamai, Kasada, all of them. The question isn't "can we bypass this?" — it's "which tier should bypass this?"
367
-
368
- **Tier 1 should handle everything that doesn't check fingerprints.**
369
- **Tier 2 should handle everything that checks TLS but not IP.**
370
- **Tier 3 should handle everything that checks IP but not JS.**
371
- **Tier 4 should handle everything.**
372
-
373
- When a URL fails at Tier 4, that's not a limitation — that's a solver bug or a new pattern to learn.
374
-
375
- This command runs weekly. Each run should produce measurable tier improvements. Compare this week's scoreboard to last week's — the numbers should go UP, never down.