@pencil-agent/nano-pencil 2.0.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +267 -267
- package/dist/build-meta.json +3 -3
- package/dist/core/export-html/AGENT.md +11 -11
- package/dist/core/export-html/template.css +971 -971
- package/dist/core/export-html/template.html +54 -54
- package/dist/core/mcp/mcp-client.d.ts +3 -1
- package/dist/core/mcp/mcp-client.js +6 -6
- package/dist/core/mcp/mcp-config.d.ts +3 -3
- package/dist/core/mcp/mcp-config.js +1 -1
- package/dist/core/mcp/mcp-manager.d.ts +5 -1
- package/dist/core/mcp/mcp-manager.js +1 -1
- package/dist/core/platform/config/resource-loader.d.ts +2 -0
- package/dist/core/platform/config/resource-loader.js +2 -2
- package/dist/core/runtime/agent-session.d.ts +12 -0
- package/dist/core/runtime/agent-session.js +8 -8
- package/dist/core/runtime/sdk.d.ts +8 -0
- package/dist/core/runtime/sdk.js +1 -1
- package/dist/extensions/builtin/AGENT.md +115 -115
- package/dist/extensions/builtin/browser/AGENT.md +17 -17
- package/dist/extensions/builtin/browser/agent-workspace/agent_helpers.py +12 -12
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/amazon/product-search.md +198 -198
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/archive-org/scraping.md +341 -341
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/arxiv/scraping.md +311 -311
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/arxiv-bulk/scraping.md +333 -333
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/atlas/overview.md +70 -70
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/booking-com/scraping.md +578 -578
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/capterra/scraping.md +440 -440
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/centilebrain/generate-estimates.md +110 -110
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/coingecko/scraping.md +325 -325
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/coinmarketcap/scraping.md +463 -463
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/coursera/scraping.md +360 -360
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/craigslist/scraping.md +390 -390
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/crossref/scraping.md +568 -568
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/dev-to/scraping.md +323 -323
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/duckduckgo/scraping.md +349 -349
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/ebay/scraping.md +435 -435
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/etsy/scraping.md +506 -506
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/eventbrite/scraping.md +363 -363
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/expedia/automation.md +168 -168
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/facebook/groups.md +236 -236
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/facebook/pages.md +295 -295
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/framer/editor.md +108 -108
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/fred/scraping.md +493 -493
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/g2/scraping.md +580 -580
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/genius/scraping.md +511 -511
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/github/repo-actions.md +65 -65
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/github/scraping.md +184 -184
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/glassdoor/scraping.md +543 -543
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/gmail/compose.md +122 -122
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/goodreads/scraping.md +461 -461
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/gutenberg/scraping.md +383 -383
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/hackernews/scraping.md +243 -243
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/howlongtobeat/scraping.md +473 -473
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/imdb/scraping.md +271 -271
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/itch-io/scraping.md +436 -436
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/job-boards/indeed-glassdoor.md +1021 -1021
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/letterboxd/scraping.md +349 -349
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/linkedin/invitation-manager.md +109 -109
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/loom/folder-enumeration.md +170 -170
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/macrotrends/scraping.md +537 -537
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/medium/article-hydration.md +120 -120
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/medium/scraping.md +414 -414
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/metacritic/scraping.md +477 -477
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/musicbrainz/scraping.md +478 -478
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/nasa/scraping.md +339 -339
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/news-aggregation/multi-source.md +205 -205
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/open-library/scraping.md +472 -472
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/openalex/scraping.md +470 -470
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/openstreetmap/scraping.md +490 -490
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/package-registries/npm-pypi.md +478 -478
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/polymarket/scraping.md +234 -234
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/producthunt/scraping.md +307 -307
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/pubmed/scraping.md +421 -421
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/quora/scraping.md +364 -364
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/rawg/scraping.md +352 -352
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/reddit/scraping.md +124 -124
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/rest-countries/scraping.md +233 -233
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/sec-edgar/scraping.md +361 -361
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/shopify-admin/README.md +36 -36
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/shopify-admin/embedded-apps.md +72 -72
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/shopify-admin/knowledge-base.md +109 -109
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/shopify-admin/polaris-inputs.md +137 -137
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/soundcloud/scraping.md +362 -362
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/spotify/scraping.md +339 -339
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/stackoverflow/scraping.md +435 -435
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/steam/scraping.md +575 -575
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/substack/scraping.md +338 -338
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/thetechgeeks/pricing.md +52 -52
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/tiktok/upload.md +107 -107
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/tradingview/scraping.md +309 -309
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/trello/boards-and-lists.md +88 -88
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/trustpilot/scraping.md +375 -375
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/walmart/scraping.md +444 -444
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/wayback-machine/scraping.md +306 -306
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/weather/scraping.md +398 -398
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/wellfound/scraping.md +596 -596
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/world-bank/scraping.md +356 -356
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/xiaohongshu/scraping.md +84 -84
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/youtube/scraping.md +418 -418
- package/dist/extensions/builtin/browser/agent-workspace/domain-skills/zillow/scraping.md +433 -433
- package/dist/extensions/builtin/browser/browser.md +73 -73
- package/dist/extensions/builtin/browser/install.md +142 -142
- package/dist/extensions/builtin/browser/interaction-skills/connection.md +48 -48
- package/dist/extensions/builtin/browser/interaction-skills/cookies.md +3 -3
- package/dist/extensions/builtin/browser/interaction-skills/cross-origin-iframes.md +3 -3
- package/dist/extensions/builtin/browser/interaction-skills/dialogs.md +64 -64
- package/dist/extensions/builtin/browser/interaction-skills/downloads.md +3 -3
- package/dist/extensions/builtin/browser/interaction-skills/drag-and-drop.md +3 -3
- package/dist/extensions/builtin/browser/interaction-skills/dropdowns.md +3 -3
- package/dist/extensions/builtin/browser/interaction-skills/iframes.md +3 -3
- package/dist/extensions/builtin/browser/interaction-skills/network-requests.md +3 -3
- package/dist/extensions/builtin/browser/interaction-skills/print-as-pdf.md +3 -3
- package/dist/extensions/builtin/browser/interaction-skills/profile-sync.md +90 -90
- package/dist/extensions/builtin/browser/interaction-skills/screenshots.md +17 -17
- package/dist/extensions/builtin/browser/interaction-skills/scrolling.md +3 -3
- package/dist/extensions/builtin/browser/interaction-skills/shadow-dom.md +3 -3
- package/dist/extensions/builtin/browser/interaction-skills/tabs.md +69 -69
- package/dist/extensions/builtin/browser/interaction-skills/uploads.md +1 -1
- package/dist/extensions/builtin/browser/interaction-skills/viewport.md +3 -3
- package/dist/extensions/builtin/browser/src/browser_harness/AGENT.md +15 -15
- package/dist/extensions/builtin/browser/src/browser_harness/__init__.py +8 -8
- package/dist/extensions/builtin/browser/src/browser_harness/_ipc.py +90 -90
- package/dist/extensions/builtin/browser/src/browser_harness/admin.py +722 -722
- package/dist/extensions/builtin/browser/src/browser_harness/daemon.py +328 -328
- package/dist/extensions/builtin/browser/src/browser_harness/helpers.py +396 -396
- package/dist/extensions/builtin/browser/src/browser_harness/run.py +103 -103
- package/dist/extensions/builtin/discipline/skills/brainstorming/SKILL.md +33 -33
- package/dist/extensions/builtin/discipline/skills/executing-plans/SKILL.md +25 -25
- package/dist/extensions/builtin/discipline/skills/finishing-development-branch/SKILL.md +25 -25
- package/dist/extensions/builtin/discipline/skills/receiving-code-review/SKILL.md +22 -22
- package/dist/extensions/builtin/discipline/skills/requesting-code-review/SKILL.md +31 -31
- package/dist/extensions/builtin/discipline/skills/systematic-debugging/SKILL.md +28 -28
- package/dist/extensions/builtin/discipline/skills/test-driven-development/SKILL.md +32 -32
- package/dist/extensions/builtin/discipline/skills/using-git-worktrees/SKILL.md +25 -25
- package/dist/extensions/builtin/discipline/skills/verification-before-completion/SKILL.md +27 -27
- package/dist/extensions/builtin/discipline/skills/writing-plans/SKILL.md +26 -26
- package/dist/extensions/builtin/goal/README.md +67 -67
- package/dist/extensions/builtin/grub/README.md +112 -112
- package/dist/extensions/builtin/link-world/agent-workspace/README.md +16 -16
- package/dist/extensions/builtin/link-world/internet-search/internet-search.md +65 -65
- package/dist/extensions/builtin/link-world/link-world-agent.md +82 -82
- package/dist/extensions/builtin/link-world/linkworld.md +313 -313
- package/dist/extensions/builtin/link-world/network-routing/network-routing.md +67 -67
- package/dist/extensions/builtin/loop/README.md +92 -92
- package/dist/extensions/builtin/mcp/figma-design.md +68 -68
- package/dist/extensions/builtin/mcp/mcp-management.md +85 -85
- package/dist/extensions/builtin/recap/AGENT.md +15 -15
- package/dist/extensions/builtin/sal/README.md +72 -72
- package/dist/extensions/builtin/security-audit/README.md +289 -289
- package/dist/extensions/builtin/team/AGENT.md +112 -112
- package/dist/extensions/builtin/team/TESTING.md +299 -299
- package/dist/extensions/builtin/token-save/README.md +56 -56
- package/dist/extensions/optional/AGENT.md +10 -10
- package/dist/modes/interactive/interactive-mode.js +36 -36
- package/dist/modes/interactive/theme/dark.json +85 -85
- package/dist/modes/interactive/theme/light.json +84 -84
- package/dist/modes/interactive/theme/theme-schema.json +335 -335
- package/dist/modes/interactive/theme/warm.json +81 -81
- package/dist/node_modules/@pencil-agent/agent-core/dist/agent-loop.js +3 -2
- package/dist/node_modules/@pencil-agent/agent-core/dist/structured-adaptive-agent-loop.js +2 -1
- package/dist/node_modules/@pencil-agent/ai/dist/cli.js +0 -0
- package/docs/cc-agent-design.md +1297 -0
- package/docs/cc-tui-design.md +1333 -0
- package/docs/codex-goal-command-impl.md +1055 -1055
- package/docs/codex-goal-vs-grub.md +500 -500
- package/docs/custom-provider.md +27 -27
- package/docs/extensions.md +27 -27
- package/docs/keybindings.md +27 -27
- package/docs/loop /351/207/215/346/236/204/345/256/214/346/210/220/346/200/273/347/273/223.md" +250 -250
- package/docs/loop /351/207/215/346/236/204/345/256/214/346/210/220/346/212/245/345/221/212.md" +122 -122
- package/docs/loop /351/207/215/346/236/204/346/226/271/346/241/210.md" +1222 -1222
- package/docs/loop /351/207/215/346/236/204/346/226/271/346/241/210/345/256/236/347/216/260/346/212/245/345/221/212.md" +158 -158
- package/docs/loop /351/207/215/346/236/204/346/226/271/346/241/210/345/257/271/346/257/224/345/210/206/346/236/220.md" +128 -128
- package/docs/loop /351/207/215/346/236/204/350/256/241/345/210/222.md" +320 -320
- package/docs/loop-usage-examples.md +214 -214
- package/docs/models.md +27 -27
- package/docs/nanoPencil-/345/255/246/344/271/240/350/256/241/345/210/222.md +170 -0
- package/docs/packages.md +27 -27
- package/docs/pi-design-philosophy.md +457 -457
- package/docs/planmode.md +1987 -1987
- package/docs/prompt-templates.md +27 -27
- package/docs/providers.md +27 -27
- package/docs/scan-report.md +3820 -0
- package/docs/sdk.md +27 -27
- package/docs/skills.md +27 -27
- package/docs/themes.md +27 -27
- package/docs/tui.md +27 -27
- package/docs//345/257/271/346/240/207Claude-Code.md +1775 -0
- package/docs//351/230/277/351/207/214/345/267/264/345/267/264/350/264/242/346/212/245/345/210/206/346/236/220/344/271/246.md +261 -0
- package/package.json +190 -190
- package/docs/ACP/345/215/217/350/256/256/351/233/206/346/210/220/345/274/200/345/217/221/346/226/207/346/241/243.md +0 -851
- package/docs/SDK-TESTING.md +0 -364
- package/docs/mem-core/346/212/200/346/234/257/346/226/207/346/241/243.md +0 -593
- package/docs/startup-performance-optimization.md +0 -301
- package/docs//350/256/244/347/237/245/345/234/260/345/233/276.md +0 -47
package/dist/extensions/builtin/browser/agent-workspace/domain-skills/producthunt/scraping.md
CHANGED
|
@@ -1,307 +1,307 @@
|
|
|
1
|
-
# Product Hunt Scraping Skills
|
|
2
|
-
|
|
3
|
-
Field-tested against https://www.producthunt.com on 2026-04-18.
|
|
4
|
-
All selectors verified with actual browser runs.
|
|
5
|
-
|
|
6
|
-
---
|
|
7
|
-
|
|
8
|
-
## Page Structure Overview
|
|
9
|
-
|
|
10
|
-
Product Hunt is a React SPA. Key structural facts discovered:
|
|
11
|
-
|
|
12
|
-
- **No login wall** — all product data is accessible without signing in
|
|
13
|
-
- **No cookie banner** — page loads cleanly with no consent dialogs
|
|
14
|
-
- **Product URLs use `/products/` not `/posts/`** — the `a[href^="/posts/"]` selector matches nothing
|
|
15
|
-
- **`data-test` attributes** are the most reliable selectors throughout the site
|
|
16
|
-
- **4 homepage sections**: today, yesterday, last week, last month (5 products each, plus "see all")
|
|
17
|
-
- **Today's votes are hidden** for the first 4 hours of each day (`—` instead of count)
|
|
18
|
-
- **Homepage has 30 fixed post-items** — scrolling does NOT load more
|
|
19
|
-
- **`goto_url()` may return `ERR_ABORTED`** for producthunt.com in some browser sessions — use `new_tab()` instead
|
|
20
|
-
|
|
21
|
-
---
|
|
22
|
-
|
|
23
|
-
## Navigation Pattern
|
|
24
|
-
|
|
25
|
-
```python
|
|
26
|
-
# goto_url() may fail on Product Hunt — use new_tab() reliably
|
|
27
|
-
tid = new_tab("https://www.producthunt.com")
|
|
28
|
-
wait(4) # React SPA needs time; wait_for_load() alone is insufficient
|
|
29
|
-
page = page_info()
|
|
30
|
-
# Verify: url should be 'https://www.producthunt.com/'
|
|
31
|
-
```
|
|
32
|
-
|
|
33
|
-
---
|
|
34
|
-
|
|
35
|
-
## Homepage — Extract Daily Product Feed
|
|
36
|
-
|
|
37
|
-
The homepage shows today's launches plus rolling sections for yesterday, last week, last month.
|
|
38
|
-
|
|
39
|
-
### Working selector: `[data-test^="post-item-"]`
|
|
40
|
-
|
|
41
|
-
```python
|
|
42
|
-
# Full extraction with name, tagline, slug, votes, topics
|
|
43
|
-
products = js("""
|
|
44
|
-
JSON.stringify(
|
|
45
|
-
Array.from(document.querySelectorAll('[data-test^="post-item-"]')).map(el => {
|
|
46
|
-
var id = el.getAttribute('data-test').replace('post-item-', '');
|
|
47
|
-
var nameEl = el.querySelector('[data-test^="post-name-"]');
|
|
48
|
-
var productLink = el.querySelector('a[href^="/products/"]');
|
|
49
|
-
var voteBtn = el.querySelector('[data-test="vote-button"]');
|
|
50
|
-
var voteCount = voteBtn ? voteBtn.textContent.trim() : null;
|
|
51
|
-
var topicLinks = Array.from(el.querySelectorAll('a[href^="/topics/"]')).map(a => a.textContent.trim());
|
|
52
|
-
var name = nameEl ? nameEl.textContent.trim() : '';
|
|
53
|
-
var lines = el.innerText.split('\\n').map(l => l.trim()).filter(l => l);
|
|
54
|
-
var tagline = lines.find(l => l !== name && !topicLinks.includes(l) && l !== '•' && !/^[0-9—]/.test(l) && l.length > 5);
|
|
55
|
-
return {
|
|
56
|
-
id: id,
|
|
57
|
-
name: name,
|
|
58
|
-
slug: productLink ? productLink.getAttribute('href') : null,
|
|
59
|
-
votes: voteCount,
|
|
60
|
-
topics: topicLinks,
|
|
61
|
-
tagline: tagline || null
|
|
62
|
-
};
|
|
63
|
-
})
|
|
64
|
-
)
|
|
65
|
-
""")
|
|
66
|
-
```
|
|
67
|
-
|
|
68
|
-
**Sample output:**
|
|
69
|
-
```json
|
|
70
|
-
[
|
|
71
|
-
{"id":"1126372","name":"Vercel Flags","slug":"/products/vercel","votes":"—","topics":["Software Engineering","Developer Tools"],"tagline":"Feature flags, targeting rules, rollouts. All from Vercel."},
|
|
72
|
-
{"id":"1125388","name":"1. Claude Opus 4.7","slug":"/products/claude-opus-4-7","votes":"466","topics":["API","Artificial Intelligence","Development"],"tagline":"Claude's most capable model for reasoning and agentic coding"}
|
|
73
|
-
]
|
|
74
|
-
```
|
|
75
|
-
|
|
76
|
-
**Votes:**
|
|
77
|
-
- `"—"` = vote count hidden (today's products during first 4 hours)
|
|
78
|
-
- `"466"` = numeric string (yesterday/older products)
|
|
79
|
-
- `"2,152"` = comma-formatted for large counts (parse: `voteCount.replace(',', '')`)
|
|
80
|
-
|
|
81
|
-
**Name prefix**: Ranked products show rank in name: `"1. Claude Opus 4.7"` — strip with `re.sub(r'^\d+\. ', '', name)`.
|
|
82
|
-
|
|
83
|
-
---
|
|
84
|
-
|
|
85
|
-
## Daily Leaderboard — Best URL for Complete Daily Lists
|
|
86
|
-
|
|
87
|
-
The leaderboard shows all products for any given day with actual vote counts.
|
|
88
|
-
|
|
89
|
-
```
|
|
90
|
-
URL: https://www.producthunt.com/leaderboard/daily/YYYY/M/D
|
|
91
|
-
Example: https://www.producthunt.com/leaderboard/daily/2026/4/18
|
|
92
|
-
```
|
|
93
|
-
|
|
94
|
-
- Uses zero-padded-free month/day (April = `4`, not `04`)
|
|
95
|
-
- Uses same `[data-test^="post-item-"]` selector
|
|
96
|
-
- Shows 12–19 products per day
|
|
97
|
-
- Same extraction JS as homepage works identically
|
|
98
|
-
|
|
99
|
-
**Yesterday's results with real vote counts:**
|
|
100
|
-
```json
|
|
101
|
-
{"id":"1125388","name":"1. Claude Opus 4.7","slug":"/products/claude-opus-4-7","votes":"466","tagline":"Claude's most capable model for reasoning and agentic coding"}
|
|
102
|
-
{"id":"1118208","name":"2. Build Check","slug":"/products/build-check-for-outsiders","votes":"396"}
|
|
103
|
-
```
|
|
104
|
-
|
|
105
|
-
---
|
|
106
|
-
|
|
107
|
-
## Weekly Leaderboard
|
|
108
|
-
|
|
109
|
-
```
|
|
110
|
-
URL: https://www.producthunt.com/leaderboard/weekly/YYYY/WW
|
|
111
|
-
Example: https://www.producthunt.com/leaderboard/weekly/2026/16
|
|
112
|
-
```
|
|
113
|
-
|
|
114
|
-
- Week number is ISO week (week 16 = April 13–19, 2026)
|
|
115
|
-
- Current week may return 0 items until the week ends
|
|
116
|
-
- Same `[data-test^="post-item-"]` selector
|
|
117
|
-
|
|
118
|
-
---
|
|
119
|
-
|
|
120
|
-
## Monthly Leaderboard
|
|
121
|
-
|
|
122
|
-
```
|
|
123
|
-
URL: https://www.producthunt.com/leaderboard/monthly/YYYY/M
|
|
124
|
-
Example: https://www.producthunt.com/leaderboard/monthly/2026/4
|
|
125
|
-
```
|
|
126
|
-
|
|
127
|
-
---
|
|
128
|
-
|
|
129
|
-
## Topic Page
|
|
130
|
-
|
|
131
|
-
URL: `https://www.producthunt.com/topics/developer-tools`
|
|
132
|
-
|
|
133
|
-
Selector changes on topic pages — uses `[data-test^="product:"]` not `post-item-`.
|
|
134
|
-
|
|
135
|
-
```python
|
|
136
|
-
# Navigate to topic
|
|
137
|
-
new_tab("https://www.producthunt.com/topics/developer-tools")
|
|
138
|
-
wait(3)
|
|
139
|
-
|
|
140
|
-
products = js("""
|
|
141
|
-
JSON.stringify(
|
|
142
|
-
Array.from(document.querySelectorAll('[data-test^="product:"]')).map(el => {
|
|
143
|
-
var slug = el.getAttribute('data-test').replace('product:', '');
|
|
144
|
-
var link = el.querySelector('a[href^="/products/"]');
|
|
145
|
-
return {
|
|
146
|
-
slug: slug,
|
|
147
|
-
href: link ? link.getAttribute('href') : null,
|
|
148
|
-
text: el.outerText.trim().substring(0, 200)
|
|
149
|
-
};
|
|
150
|
-
})
|
|
151
|
-
)
|
|
152
|
-
""")
|
|
153
|
-
```
|
|
154
|
-
|
|
155
|
-
**Sample output:**
|
|
156
|
-
```json
|
|
157
|
-
{"slug":"figma","href":"/products/figma","text":"FigmaThe collaborative interface design tool4.9 (1.4K reviews)..."}
|
|
158
|
-
```
|
|
159
|
-
|
|
160
|
-
Returns ~15 top-rated products in the topic, not recent launches.
|
|
161
|
-
|
|
162
|
-
---
|
|
163
|
-
|
|
164
|
-
## Category Page
|
|
165
|
-
|
|
166
|
-
URL: `https://www.producthunt.com/categories/ai-agents`
|
|
167
|
-
|
|
168
|
-
Same `[data-test^="product:"]` selector as topics. Returns 15 top-reviewed products in that category.
|
|
169
|
-
|
|
170
|
-
```json
|
|
171
|
-
{"slug":"elevenlabs","href":"/products/elevenlabs","text":"ElevenLabs\nCreate natural AI voices instantly...\n4.9 (165 reviews)"}
|
|
172
|
-
```
|
|
173
|
-
|
|
174
|
-
---
|
|
175
|
-
|
|
176
|
-
## Product Detail Page
|
|
177
|
-
|
|
178
|
-
URL: `https://www.producthunt.com/products/claude-opus-4-7`
|
|
179
|
-
|
|
180
|
-
### Get total vote count (sidebar button)
|
|
181
|
-
```python
|
|
182
|
-
# Use [data-test="vote-button"] — different from [data-test="action-bar-vote-button"]
|
|
183
|
-
vote_text = js("document.querySelector('[data-test=\"vote-button\"]').outerText.trim().replace(/\\s+/g, ' ')")
|
|
184
|
-
# Returns: "Upvote • 466 points"
|
|
185
|
-
# Parse votes: vote_text.split('•')[1].strip().replace(' points', '').replace(',', '')
|
|
186
|
-
```
|
|
187
|
-
|
|
188
|
-
### Get review count and rating
|
|
189
|
-
```python
|
|
190
|
-
review_link = js("JSON.stringify(Array.from(document.querySelectorAll('a')).filter(a => a.href && a.href.includes('/reviews') && a.outerText.includes('review')).map(a => a.outerText.trim()).slice(0, 1))")
|
|
191
|
-
# Returns: ["1 review"] or ["5.0\n(731 reviews)"]
|
|
192
|
-
```
|
|
193
|
-
|
|
194
|
-
### Get day rank (sidebar shows "#1 Day Rank")
|
|
195
|
-
No dedicated `data-test` for rank — parse from sidebar context or use leaderboard position.
|
|
196
|
-
|
|
197
|
-
### Comments (action-bar-vote-button)
|
|
198
|
-
Each comment has its own `[data-test="action-bar-vote-button"]` with text like `"Upvote (13)"`.
|
|
199
|
-
|
|
200
|
-
---
|
|
201
|
-
|
|
202
|
-
## Search Results
|
|
203
|
-
|
|
204
|
-
URL: `https://www.producthunt.com/search?q=AI+agent`
|
|
205
|
-
|
|
206
|
-
Selector: `[data-test^="spotlight-result-product-"]`
|
|
207
|
-
|
|
208
|
-
```python
|
|
209
|
-
new_tab("https://www.producthunt.com/search?q=AI+agent")
|
|
210
|
-
wait(3)
|
|
211
|
-
|
|
212
|
-
results = js("""
|
|
213
|
-
JSON.stringify(
|
|
214
|
-
Array.from(document.querySelectorAll('[data-test^="spotlight-result-product-"]')).map(el => {
|
|
215
|
-
var id = el.getAttribute('data-test').replace('spotlight-result-product-', '');
|
|
216
|
-
var lines = el.outerText.trim().split('\\n').map(l => l.trim()).filter(l => l);
|
|
217
|
-
return {
|
|
218
|
-
id: id,
|
|
219
|
-
name: lines[0] || null,
|
|
220
|
-
tagline: lines[1] || null,
|
|
221
|
-
review_text: lines[2] || null
|
|
222
|
-
};
|
|
223
|
-
})
|
|
224
|
-
)
|
|
225
|
-
""")
|
|
226
|
-
```
|
|
227
|
-
|
|
228
|
-
**Note:** Search result elements are `<button>` elements (not `<a>` links), so there is no `href` in the DOM. Product URL must be constructed: `https://www.producthunt.com/products/<slug>` where slug must be derived by other means. The element's `data-test` ID matches the numeric product ID, not the slug.
|
|
229
|
-
|
|
230
|
-
**Sample output:**
|
|
231
|
-
```json
|
|
232
|
-
{"id":"526014","name":"/ai","tagline":"Access ChatGPT anywhere you type '/ai'","review_text":"2 reviews"}
|
|
233
|
-
{"id":"991302","name":"Naoma AI Demo Agent","tagline":"The first video agent that runs conversational product demos","review_text":"5 reviews"}
|
|
234
|
-
```
|
|
235
|
-
|
|
236
|
-
---
|
|
237
|
-
|
|
238
|
-
## Key Selector Reference
|
|
239
|
-
|
|
240
|
-
| Page | Selector | Count | Notes |
|
|
241
|
-
|------|----------|-------|-------|
|
|
242
|
-
| Homepage | `[data-test^="post-item-"]` | 30 | 4 sections × ~5–7 products |
|
|
243
|
-
| Homepage | `[data-test^="post-name-"]` | 30 | Product name elements |
|
|
244
|
-
| Homepage/Leaderboard | `[data-test="vote-button"]` | varies | `—` for hidden; numeric for visible |
|
|
245
|
-
| Topics/Categories | `[data-test^="product:"]` | ~15 | Top-rated products |
|
|
246
|
-
| Search | `[data-test^="spotlight-result-product-"]` | 10 | Button elements, no href |
|
|
247
|
-
| Product detail | `[data-test="vote-button"]` | 1 | Main vote: "Upvote • N points" |
|
|
248
|
-
| Product detail | `[data-test="action-bar-vote-button"]` | many | Comment upvotes: "Upvote (N)" |
|
|
249
|
-
|
|
250
|
-
---
|
|
251
|
-
|
|
252
|
-
## Common Pitfalls
|
|
253
|
-
|
|
254
|
-
1. **`innerText` returns `None` on complex elements** — use `outerText` or break into simple single-property expressions. Avoid chaining DOM traversal inside `JSON.stringify()` on large objects.
|
|
255
|
-
|
|
256
|
-
2. **`goto_url()` returns `ERR_ABORTED`** for producthunt.com in some browser sessions — always use `new_tab("url")` instead.
|
|
257
|
-
|
|
258
|
-
3. **`a[href^="/posts/"]` matches nothing** — Product Hunt uses `/products/` for product URLs, not `/posts/`.
|
|
259
|
-
|
|
260
|
-
4. **Today's votes are always `—`** during the first 4 hours of the day — use yesterday's leaderboard for confirmed vote counts.
|
|
261
|
-
|
|
262
|
-
5. **Homepage does not lazy-load more products on scroll** — 30 items is the fixed set. Use leaderboard pages for complete daily listings.
|
|
263
|
-
|
|
264
|
-
6. **JSON.stringify of DOM-heavy objects returns `None`** — serialize only primitives (strings, numbers) not live DOM node properties.
|
|
265
|
-
|
|
266
|
-
7. **Ranked product names contain rank prefix** — `"1. Claude Opus 4.7"` — strip with regex `re.sub(r'^\d+\.\s+', '', name)`.
|
|
267
|
-
|
|
268
|
-
8. **`wait(3)` required after `wait_for_load()`** — the React SPA continues rendering after the load event.
|
|
269
|
-
|
|
270
|
-
---
|
|
271
|
-
|
|
272
|
-
## Recommended Workflow for Scraping Today's Launches
|
|
273
|
-
|
|
274
|
-
```python
|
|
275
|
-
# 1. Open Product Hunt in a new tab
|
|
276
|
-
new_tab("https://www.producthunt.com/leaderboard/daily/2026/4/18")
|
|
277
|
-
wait(4)
|
|
278
|
-
|
|
279
|
-
# 2. Extract all products with metadata
|
|
280
|
-
products = js("""
|
|
281
|
-
JSON.stringify(
|
|
282
|
-
Array.from(document.querySelectorAll('[data-test^="post-item-"]')).map(el => {
|
|
283
|
-
var id = el.getAttribute('data-test').replace('post-item-', '');
|
|
284
|
-
var nameEl = el.querySelector('[data-test^="post-name-"]');
|
|
285
|
-
var productLink = el.querySelector('a[href^="/products/"]');
|
|
286
|
-
var voteBtn = el.querySelector('[data-test="vote-button"]');
|
|
287
|
-
var topicLinks = Array.from(el.querySelectorAll('a[href^="/topics/"]')).map(a => a.textContent.trim());
|
|
288
|
-
var name = nameEl ? nameEl.textContent.trim() : '';
|
|
289
|
-
var lines = el.innerText.split('\\n').map(l => l.trim()).filter(l => l);
|
|
290
|
-
var tagline = lines.find(l => l !== name && !topicLinks.includes(l) && l !== '•' && !/^[0-9—]/.test(l) && l.length > 5);
|
|
291
|
-
return {
|
|
292
|
-
id: id,
|
|
293
|
-
name: name,
|
|
294
|
-
slug: productLink ? productLink.getAttribute('href') : null,
|
|
295
|
-
votes: voteBtn ? voteBtn.textContent.trim() : null,
|
|
296
|
-
topics: topicLinks,
|
|
297
|
-
tagline: tagline || null
|
|
298
|
-
};
|
|
299
|
-
})
|
|
300
|
-
)
|
|
301
|
-
""")
|
|
302
|
-
import json
|
|
303
|
-
data = json.loads(products)
|
|
304
|
-
print(f"Found {len(data)} products")
|
|
305
|
-
for p in data:
|
|
306
|
-
print(f" {p['name']} — {p['votes']} votes — {p['tagline']}")
|
|
307
|
-
```
|
|
1
|
+
# Product Hunt Scraping Skills
|
|
2
|
+
|
|
3
|
+
Field-tested against https://www.producthunt.com on 2026-04-18.
|
|
4
|
+
All selectors verified with actual browser runs.
|
|
5
|
+
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Page Structure Overview
|
|
9
|
+
|
|
10
|
+
Product Hunt is a React SPA. Key structural facts discovered:
|
|
11
|
+
|
|
12
|
+
- **No login wall** — all product data is accessible without signing in
|
|
13
|
+
- **No cookie banner** — page loads cleanly with no consent dialogs
|
|
14
|
+
- **Product URLs use `/products/` not `/posts/`** — the `a[href^="/posts/"]` selector matches nothing
|
|
15
|
+
- **`data-test` attributes** are the most reliable selectors throughout the site
|
|
16
|
+
- **4 homepage sections**: today, yesterday, last week, last month (5 products each, plus "see all")
|
|
17
|
+
- **Today's votes are hidden** for the first 4 hours of each day (`—` instead of count)
|
|
18
|
+
- **Homepage has 30 fixed post-items** — scrolling does NOT load more
|
|
19
|
+
- **`goto_url()` may return `ERR_ABORTED`** for producthunt.com in some browser sessions — use `new_tab()` instead
|
|
20
|
+
|
|
21
|
+
---
|
|
22
|
+
|
|
23
|
+
## Navigation Pattern
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
# goto_url() may fail on Product Hunt — use new_tab() reliably
|
|
27
|
+
tid = new_tab("https://www.producthunt.com")
|
|
28
|
+
wait(4) # React SPA needs time; wait_for_load() alone is insufficient
|
|
29
|
+
page = page_info()
|
|
30
|
+
# Verify: url should be 'https://www.producthunt.com/'
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
## Homepage — Extract Daily Product Feed
|
|
36
|
+
|
|
37
|
+
The homepage shows today's launches plus rolling sections for yesterday, last week, last month.
|
|
38
|
+
|
|
39
|
+
### Working selector: `[data-test^="post-item-"]`
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
# Full extraction with name, tagline, slug, votes, topics
|
|
43
|
+
products = js("""
|
|
44
|
+
JSON.stringify(
|
|
45
|
+
Array.from(document.querySelectorAll('[data-test^="post-item-"]')).map(el => {
|
|
46
|
+
var id = el.getAttribute('data-test').replace('post-item-', '');
|
|
47
|
+
var nameEl = el.querySelector('[data-test^="post-name-"]');
|
|
48
|
+
var productLink = el.querySelector('a[href^="/products/"]');
|
|
49
|
+
var voteBtn = el.querySelector('[data-test="vote-button"]');
|
|
50
|
+
var voteCount = voteBtn ? voteBtn.textContent.trim() : null;
|
|
51
|
+
var topicLinks = Array.from(el.querySelectorAll('a[href^="/topics/"]')).map(a => a.textContent.trim());
|
|
52
|
+
var name = nameEl ? nameEl.textContent.trim() : '';
|
|
53
|
+
var lines = el.innerText.split('\\n').map(l => l.trim()).filter(l => l);
|
|
54
|
+
var tagline = lines.find(l => l !== name && !topicLinks.includes(l) && l !== '•' && !/^[0-9—]/.test(l) && l.length > 5);
|
|
55
|
+
return {
|
|
56
|
+
id: id,
|
|
57
|
+
name: name,
|
|
58
|
+
slug: productLink ? productLink.getAttribute('href') : null,
|
|
59
|
+
votes: voteCount,
|
|
60
|
+
topics: topicLinks,
|
|
61
|
+
tagline: tagline || null
|
|
62
|
+
};
|
|
63
|
+
})
|
|
64
|
+
)
|
|
65
|
+
""")
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
**Sample output:**
|
|
69
|
+
```json
|
|
70
|
+
[
|
|
71
|
+
{"id":"1126372","name":"Vercel Flags","slug":"/products/vercel","votes":"—","topics":["Software Engineering","Developer Tools"],"tagline":"Feature flags, targeting rules, rollouts. All from Vercel."},
|
|
72
|
+
{"id":"1125388","name":"1. Claude Opus 4.7","slug":"/products/claude-opus-4-7","votes":"466","topics":["API","Artificial Intelligence","Development"],"tagline":"Claude's most capable model for reasoning and agentic coding"}
|
|
73
|
+
]
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
**Votes:**
|
|
77
|
+
- `"—"` = vote count hidden (today's products during first 4 hours)
|
|
78
|
+
- `"466"` = numeric string (yesterday/older products)
|
|
79
|
+
- `"2,152"` = comma-formatted for large counts (parse: `voteCount.replace(',', '')`)
|
|
80
|
+
|
|
81
|
+
**Name prefix**: Ranked products show rank in name: `"1. Claude Opus 4.7"` — strip with `re.sub(r'^\d+\. ', '', name)`.
|
|
82
|
+
|
|
83
|
+
---
|
|
84
|
+
|
|
85
|
+
## Daily Leaderboard — Best URL for Complete Daily Lists
|
|
86
|
+
|
|
87
|
+
The leaderboard shows all products for any given day with actual vote counts.
|
|
88
|
+
|
|
89
|
+
```
|
|
90
|
+
URL: https://www.producthunt.com/leaderboard/daily/YYYY/M/D
|
|
91
|
+
Example: https://www.producthunt.com/leaderboard/daily/2026/4/18
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
- Uses zero-padded-free month/day (April = `4`, not `04`)
|
|
95
|
+
- Uses same `[data-test^="post-item-"]` selector
|
|
96
|
+
- Shows 12–19 products per day
|
|
97
|
+
- Same extraction JS as homepage works identically
|
|
98
|
+
|
|
99
|
+
**Yesterday's results with real vote counts:**
|
|
100
|
+
```json
|
|
101
|
+
{"id":"1125388","name":"1. Claude Opus 4.7","slug":"/products/claude-opus-4-7","votes":"466","tagline":"Claude's most capable model for reasoning and agentic coding"}
|
|
102
|
+
{"id":"1118208","name":"2. Build Check","slug":"/products/build-check-for-outsiders","votes":"396"}
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
---
|
|
106
|
+
|
|
107
|
+
## Weekly Leaderboard
|
|
108
|
+
|
|
109
|
+
```
|
|
110
|
+
URL: https://www.producthunt.com/leaderboard/weekly/YYYY/WW
|
|
111
|
+
Example: https://www.producthunt.com/leaderboard/weekly/2026/16
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
- Week number is ISO week (week 16 = April 13–19, 2026)
|
|
115
|
+
- Current week may return 0 items until the week ends
|
|
116
|
+
- Same `[data-test^="post-item-"]` selector
|
|
117
|
+
|
|
118
|
+
---
|
|
119
|
+
|
|
120
|
+
## Monthly Leaderboard
|
|
121
|
+
|
|
122
|
+
```
|
|
123
|
+
URL: https://www.producthunt.com/leaderboard/monthly/YYYY/M
|
|
124
|
+
Example: https://www.producthunt.com/leaderboard/monthly/2026/4
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
---
|
|
128
|
+
|
|
129
|
+
## Topic Page
|
|
130
|
+
|
|
131
|
+
URL: `https://www.producthunt.com/topics/developer-tools`
|
|
132
|
+
|
|
133
|
+
Selector changes on topic pages — uses `[data-test^="product:"]` not `post-item-`.
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
# Navigate to topic
|
|
137
|
+
new_tab("https://www.producthunt.com/topics/developer-tools")
|
|
138
|
+
wait(3)
|
|
139
|
+
|
|
140
|
+
products = js("""
|
|
141
|
+
JSON.stringify(
|
|
142
|
+
Array.from(document.querySelectorAll('[data-test^="product:"]')).map(el => {
|
|
143
|
+
var slug = el.getAttribute('data-test').replace('product:', '');
|
|
144
|
+
var link = el.querySelector('a[href^="/products/"]');
|
|
145
|
+
return {
|
|
146
|
+
slug: slug,
|
|
147
|
+
href: link ? link.getAttribute('href') : null,
|
|
148
|
+
text: el.outerText.trim().substring(0, 200)
|
|
149
|
+
};
|
|
150
|
+
})
|
|
151
|
+
)
|
|
152
|
+
""")
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
**Sample output:**
|
|
156
|
+
```json
|
|
157
|
+
{"slug":"figma","href":"/products/figma","text":"FigmaThe collaborative interface design tool4.9 (1.4K reviews)..."}
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Returns ~15 top-rated products in the topic, not recent launches.
|
|
161
|
+
|
|
162
|
+
---
|
|
163
|
+
|
|
164
|
+
## Category Page
|
|
165
|
+
|
|
166
|
+
URL: `https://www.producthunt.com/categories/ai-agents`
|
|
167
|
+
|
|
168
|
+
Same `[data-test^="product:"]` selector as topics. Returns 15 top-reviewed products in that category.
|
|
169
|
+
|
|
170
|
+
```json
|
|
171
|
+
{"slug":"elevenlabs","href":"/products/elevenlabs","text":"ElevenLabs\nCreate natural AI voices instantly...\n4.9 (165 reviews)"}
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
---
|
|
175
|
+
|
|
176
|
+
## Product Detail Page
|
|
177
|
+
|
|
178
|
+
URL: `https://www.producthunt.com/products/claude-opus-4-7`
|
|
179
|
+
|
|
180
|
+
### Get total vote count (sidebar button)
|
|
181
|
+
```python
|
|
182
|
+
# Use [data-test="vote-button"] — different from [data-test="action-bar-vote-button"]
|
|
183
|
+
vote_text = js("document.querySelector('[data-test=\"vote-button\"]').outerText.trim().replace(/\\s+/g, ' ')")
|
|
184
|
+
# Returns: "Upvote • 466 points"
|
|
185
|
+
# Parse votes: vote_text.split('•')[1].strip().replace(' points', '').replace(',', '')
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
### Get review count and rating
|
|
189
|
+
```python
|
|
190
|
+
review_link = js("JSON.stringify(Array.from(document.querySelectorAll('a')).filter(a => a.href && a.href.includes('/reviews') && a.outerText.includes('review')).map(a => a.outerText.trim()).slice(0, 1))")
|
|
191
|
+
# Returns: ["1 review"] or ["5.0\n(731 reviews)"]
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
### Get day rank (sidebar shows "#1 Day Rank")
|
|
195
|
+
No dedicated `data-test` for rank — parse from sidebar context or use leaderboard position.
|
|
196
|
+
|
|
197
|
+
### Comments (action-bar-vote-button)
|
|
198
|
+
Each comment has its own `[data-test="action-bar-vote-button"]` with text like `"Upvote (13)"`.
|
|
199
|
+
|
|
200
|
+
---
|
|
201
|
+
|
|
202
|
+
## Search Results
|
|
203
|
+
|
|
204
|
+
URL: `https://www.producthunt.com/search?q=AI+agent`
|
|
205
|
+
|
|
206
|
+
Selector: `[data-test^="spotlight-result-product-"]`
|
|
207
|
+
|
|
208
|
+
```python
|
|
209
|
+
new_tab("https://www.producthunt.com/search?q=AI+agent")
|
|
210
|
+
wait(3)
|
|
211
|
+
|
|
212
|
+
results = js("""
|
|
213
|
+
JSON.stringify(
|
|
214
|
+
Array.from(document.querySelectorAll('[data-test^="spotlight-result-product-"]')).map(el => {
|
|
215
|
+
var id = el.getAttribute('data-test').replace('spotlight-result-product-', '');
|
|
216
|
+
var lines = el.outerText.trim().split('\\n').map(l => l.trim()).filter(l => l);
|
|
217
|
+
return {
|
|
218
|
+
id: id,
|
|
219
|
+
name: lines[0] || null,
|
|
220
|
+
tagline: lines[1] || null,
|
|
221
|
+
review_text: lines[2] || null
|
|
222
|
+
};
|
|
223
|
+
})
|
|
224
|
+
)
|
|
225
|
+
""")
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
**Note:** Search result elements are `<button>` elements (not `<a>` links), so there is no `href` in the DOM. Product URL must be constructed: `https://www.producthunt.com/products/<slug>` where slug must be derived by other means. The element's `data-test` ID matches the numeric product ID, not the slug.
|
|
229
|
+
|
|
230
|
+
**Sample output:**
|
|
231
|
+
```json
|
|
232
|
+
{"id":"526014","name":"/ai","tagline":"Access ChatGPT anywhere you type '/ai'","review_text":"2 reviews"}
|
|
233
|
+
{"id":"991302","name":"Naoma AI Demo Agent","tagline":"The first video agent that runs conversational product demos","review_text":"5 reviews"}
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
---
|
|
237
|
+
|
|
238
|
+
## Key Selector Reference
|
|
239
|
+
|
|
240
|
+
| Page | Selector | Count | Notes |
|
|
241
|
+
|------|----------|-------|-------|
|
|
242
|
+
| Homepage | `[data-test^="post-item-"]` | 30 | 4 sections × ~5–7 products |
|
|
243
|
+
| Homepage | `[data-test^="post-name-"]` | 30 | Product name elements |
|
|
244
|
+
| Homepage/Leaderboard | `[data-test="vote-button"]` | varies | `—` for hidden; numeric for visible |
|
|
245
|
+
| Topics/Categories | `[data-test^="product:"]` | ~15 | Top-rated products |
|
|
246
|
+
| Search | `[data-test^="spotlight-result-product-"]` | 10 | Button elements, no href |
|
|
247
|
+
| Product detail | `[data-test="vote-button"]` | 1 | Main vote: "Upvote • N points" |
|
|
248
|
+
| Product detail | `[data-test="action-bar-vote-button"]` | many | Comment upvotes: "Upvote (N)" |
|
|
249
|
+
|
|
250
|
+
---
|
|
251
|
+
|
|
252
|
+
## Common Pitfalls
|
|
253
|
+
|
|
254
|
+
1. **`innerText` returns `None` on complex elements** — use `outerText` or break into simple single-property expressions. Avoid chaining DOM traversal inside `JSON.stringify()` on large objects.
|
|
255
|
+
|
|
256
|
+
2. **`goto_url()` returns `ERR_ABORTED`** for producthunt.com in some browser sessions — always use `new_tab("url")` instead.
|
|
257
|
+
|
|
258
|
+
3. **`a[href^="/posts/"]` matches nothing** — Product Hunt uses `/products/` for product URLs, not `/posts/`.
|
|
259
|
+
|
|
260
|
+
4. **Today's votes are always `—`** during the first 4 hours of the day — use yesterday's leaderboard for confirmed vote counts.
|
|
261
|
+
|
|
262
|
+
5. **Homepage does not lazy-load more products on scroll** — 30 items is the fixed set. Use leaderboard pages for complete daily listings.
|
|
263
|
+
|
|
264
|
+
6. **JSON.stringify of DOM-heavy objects returns `None`** — serialize only primitives (strings, numbers) not live DOM node properties.
|
|
265
|
+
|
|
266
|
+
7. **Ranked product names contain rank prefix** — `"1. Claude Opus 4.7"` — strip with regex `re.sub(r'^\d+\.\s+', '', name)`.
|
|
267
|
+
|
|
268
|
+
8. **`wait(3)` required after `wait_for_load()`** — the React SPA continues rendering after the load event.
|
|
269
|
+
|
|
270
|
+
---
|
|
271
|
+
|
|
272
|
+
## Recommended Workflow for Scraping Today's Launches
|
|
273
|
+
|
|
274
|
+
```python
|
|
275
|
+
# 1. Open Product Hunt in a new tab
|
|
276
|
+
new_tab("https://www.producthunt.com/leaderboard/daily/2026/4/18")
|
|
277
|
+
wait(4)
|
|
278
|
+
|
|
279
|
+
# 2. Extract all products with metadata
|
|
280
|
+
products = js("""
|
|
281
|
+
JSON.stringify(
|
|
282
|
+
Array.from(document.querySelectorAll('[data-test^="post-item-"]')).map(el => {
|
|
283
|
+
var id = el.getAttribute('data-test').replace('post-item-', '');
|
|
284
|
+
var nameEl = el.querySelector('[data-test^="post-name-"]');
|
|
285
|
+
var productLink = el.querySelector('a[href^="/products/"]');
|
|
286
|
+
var voteBtn = el.querySelector('[data-test="vote-button"]');
|
|
287
|
+
var topicLinks = Array.from(el.querySelectorAll('a[href^="/topics/"]')).map(a => a.textContent.trim());
|
|
288
|
+
var name = nameEl ? nameEl.textContent.trim() : '';
|
|
289
|
+
var lines = el.innerText.split('\\n').map(l => l.trim()).filter(l => l);
|
|
290
|
+
var tagline = lines.find(l => l !== name && !topicLinks.includes(l) && l !== '•' && !/^[0-9—]/.test(l) && l.length > 5);
|
|
291
|
+
return {
|
|
292
|
+
id: id,
|
|
293
|
+
name: name,
|
|
294
|
+
slug: productLink ? productLink.getAttribute('href') : null,
|
|
295
|
+
votes: voteBtn ? voteBtn.textContent.trim() : null,
|
|
296
|
+
topics: topicLinks,
|
|
297
|
+
tagline: tagline || null
|
|
298
|
+
};
|
|
299
|
+
})
|
|
300
|
+
)
|
|
301
|
+
""")
|
|
302
|
+
import json
|
|
303
|
+
data = json.loads(products)
|
|
304
|
+
print(f"Found {len(data)} products")
|
|
305
|
+
for p in data:
|
|
306
|
+
print(f" {p['name']} — {p['votes']} votes — {p['tagline']}")
|
|
307
|
+
```
|