@pencil-agent/nano-pencil 2.0.1 → 2.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. package/README.md +267 -267
  2. package/dist/build-meta.json +3 -3
  3. package/dist/core/export-html/AGENT.md +11 -11
  4. package/dist/core/export-html/template.css +971 -971
  5. package/dist/core/export-html/template.html +54 -54
  6. package/dist/core/model/custom-providers.js +1 -1
  7. package/dist/core/model-registry.js +5 -5
  8. package/dist/extensions/builtin/AGENT.md +115 -115
  9. package/dist/extensions/builtin/browser/AGENT.md +17 -17
  10. package/dist/extensions/builtin/browser/agent-workspace/agent_helpers.py +12 -12
  11. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/amazon/product-search.md +198 -198
  12. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/archive-org/scraping.md +341 -341
  13. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/arxiv/scraping.md +311 -311
  14. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/arxiv-bulk/scraping.md +333 -333
  15. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/atlas/overview.md +70 -70
  16. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/booking-com/scraping.md +578 -578
  17. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/capterra/scraping.md +440 -440
  18. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/centilebrain/generate-estimates.md +110 -110
  19. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/coingecko/scraping.md +325 -325
  20. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/coinmarketcap/scraping.md +463 -463
  21. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/coursera/scraping.md +360 -360
  22. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/craigslist/scraping.md +390 -390
  23. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/crossref/scraping.md +568 -568
  24. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/dev-to/scraping.md +323 -323
  25. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/duckduckgo/scraping.md +349 -349
  26. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/ebay/scraping.md +435 -435
  27. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/etsy/scraping.md +506 -506
  28. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/eventbrite/scraping.md +363 -363
  29. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/expedia/automation.md +168 -168
  30. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/facebook/groups.md +236 -236
  31. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/facebook/pages.md +295 -295
  32. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/framer/editor.md +108 -108
  33. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/fred/scraping.md +493 -493
  34. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/g2/scraping.md +580 -580
  35. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/genius/scraping.md +511 -511
  36. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/github/repo-actions.md +65 -65
  37. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/github/scraping.md +184 -184
  38. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/glassdoor/scraping.md +543 -543
  39. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/gmail/compose.md +122 -122
  40. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/goodreads/scraping.md +461 -461
  41. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/gutenberg/scraping.md +383 -383
  42. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/hackernews/scraping.md +243 -243
  43. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/howlongtobeat/scraping.md +473 -473
  44. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/imdb/scraping.md +271 -271
  45. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/itch-io/scraping.md +436 -436
  46. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/job-boards/indeed-glassdoor.md +1021 -1021
  47. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/letterboxd/scraping.md +349 -349
  48. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/linkedin/invitation-manager.md +109 -109
  49. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/loom/folder-enumeration.md +170 -170
  50. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/macrotrends/scraping.md +537 -537
  51. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/medium/article-hydration.md +120 -120
  52. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/medium/scraping.md +414 -414
  53. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/metacritic/scraping.md +477 -477
  54. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/musicbrainz/scraping.md +478 -478
  55. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/nasa/scraping.md +339 -339
  56. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/news-aggregation/multi-source.md +205 -205
  57. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/open-library/scraping.md +472 -472
  58. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/openalex/scraping.md +470 -470
  59. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/openstreetmap/scraping.md +490 -490
  60. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/package-registries/npm-pypi.md +478 -478
  61. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/polymarket/scraping.md +234 -234
  62. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/producthunt/scraping.md +307 -307
  63. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/pubmed/scraping.md +421 -421
  64. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/quora/scraping.md +364 -364
  65. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/rawg/scraping.md +352 -352
  66. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/reddit/scraping.md +124 -124
  67. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/rest-countries/scraping.md +233 -233
  68. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/sec-edgar/scraping.md +361 -361
  69. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/shopify-admin/README.md +36 -36
  70. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/shopify-admin/embedded-apps.md +72 -72
  71. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/shopify-admin/knowledge-base.md +109 -109
  72. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/shopify-admin/polaris-inputs.md +137 -137
  73. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/soundcloud/scraping.md +362 -362
  74. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/spotify/scraping.md +339 -339
  75. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/stackoverflow/scraping.md +435 -435
  76. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/steam/scraping.md +575 -575
  77. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/substack/scraping.md +338 -338
  78. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/thetechgeeks/pricing.md +52 -52
  79. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/tiktok/upload.md +107 -107
  80. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/tradingview/scraping.md +309 -309
  81. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/trello/boards-and-lists.md +88 -88
  82. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/trustpilot/scraping.md +375 -375
  83. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/walmart/scraping.md +444 -444
  84. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/wayback-machine/scraping.md +306 -306
  85. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/weather/scraping.md +398 -398
  86. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/wellfound/scraping.md +596 -596
  87. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/world-bank/scraping.md +356 -356
  88. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/xiaohongshu/scraping.md +84 -84
  89. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/youtube/scraping.md +418 -418
  90. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/zillow/scraping.md +433 -433
  91. package/dist/extensions/builtin/browser/browser.md +73 -73
  92. package/dist/extensions/builtin/browser/install.md +142 -142
  93. package/dist/extensions/builtin/browser/interaction-skills/connection.md +48 -48
  94. package/dist/extensions/builtin/browser/interaction-skills/cookies.md +3 -3
  95. package/dist/extensions/builtin/browser/interaction-skills/cross-origin-iframes.md +3 -3
  96. package/dist/extensions/builtin/browser/interaction-skills/dialogs.md +64 -64
  97. package/dist/extensions/builtin/browser/interaction-skills/downloads.md +3 -3
  98. package/dist/extensions/builtin/browser/interaction-skills/drag-and-drop.md +3 -3
  99. package/dist/extensions/builtin/browser/interaction-skills/dropdowns.md +3 -3
  100. package/dist/extensions/builtin/browser/interaction-skills/iframes.md +3 -3
  101. package/dist/extensions/builtin/browser/interaction-skills/network-requests.md +3 -3
  102. package/dist/extensions/builtin/browser/interaction-skills/print-as-pdf.md +3 -3
  103. package/dist/extensions/builtin/browser/interaction-skills/profile-sync.md +90 -90
  104. package/dist/extensions/builtin/browser/interaction-skills/screenshots.md +17 -17
  105. package/dist/extensions/builtin/browser/interaction-skills/scrolling.md +3 -3
  106. package/dist/extensions/builtin/browser/interaction-skills/shadow-dom.md +3 -3
  107. package/dist/extensions/builtin/browser/interaction-skills/tabs.md +69 -69
  108. package/dist/extensions/builtin/browser/interaction-skills/uploads.md +1 -1
  109. package/dist/extensions/builtin/browser/interaction-skills/viewport.md +3 -3
  110. package/dist/extensions/builtin/browser/src/browser_harness/AGENT.md +15 -15
  111. package/dist/extensions/builtin/browser/src/browser_harness/__init__.py +8 -8
  112. package/dist/extensions/builtin/browser/src/browser_harness/_ipc.py +90 -90
  113. package/dist/extensions/builtin/browser/src/browser_harness/admin.py +722 -722
  114. package/dist/extensions/builtin/browser/src/browser_harness/daemon.py +328 -328
  115. package/dist/extensions/builtin/browser/src/browser_harness/helpers.py +396 -396
  116. package/dist/extensions/builtin/browser/src/browser_harness/run.py +103 -103
  117. package/dist/extensions/builtin/debug/index.js +9 -9
  118. package/dist/extensions/builtin/discipline/skills/brainstorming/SKILL.md +33 -33
  119. package/dist/extensions/builtin/discipline/skills/executing-plans/SKILL.md +25 -25
  120. package/dist/extensions/builtin/discipline/skills/finishing-development-branch/SKILL.md +25 -25
  121. package/dist/extensions/builtin/discipline/skills/receiving-code-review/SKILL.md +22 -22
  122. package/dist/extensions/builtin/discipline/skills/requesting-code-review/SKILL.md +31 -31
  123. package/dist/extensions/builtin/discipline/skills/systematic-debugging/SKILL.md +28 -28
  124. package/dist/extensions/builtin/discipline/skills/test-driven-development/SKILL.md +32 -32
  125. package/dist/extensions/builtin/discipline/skills/using-git-worktrees/SKILL.md +25 -25
  126. package/dist/extensions/builtin/discipline/skills/verification-before-completion/SKILL.md +27 -27
  127. package/dist/extensions/builtin/discipline/skills/writing-plans/SKILL.md +26 -26
  128. package/dist/extensions/builtin/goal/README.md +67 -67
  129. package/dist/extensions/builtin/goal/index.js +6 -6
  130. package/dist/extensions/builtin/grub/README.md +112 -112
  131. package/dist/extensions/builtin/link-world/agent-workspace/README.md +16 -16
  132. package/dist/extensions/builtin/link-world/internet-search/internet-search.md +65 -65
  133. package/dist/extensions/builtin/link-world/link-world-agent.md +82 -82
  134. package/dist/extensions/builtin/link-world/linkworld.md +313 -313
  135. package/dist/extensions/builtin/link-world/network-routing/network-routing.md +67 -67
  136. package/dist/extensions/builtin/loop/README.md +92 -92
  137. package/dist/extensions/builtin/mcp/figma-design.md +68 -68
  138. package/dist/extensions/builtin/mcp/mcp-management.md +85 -85
  139. package/dist/extensions/builtin/recap/AGENT.md +15 -15
  140. package/dist/extensions/builtin/sal/README.md +72 -72
  141. package/dist/extensions/builtin/security-audit/README.md +289 -289
  142. package/dist/extensions/builtin/team/AGENT.md +112 -112
  143. package/dist/extensions/builtin/team/TESTING.md +299 -299
  144. package/dist/extensions/builtin/token-save/README.md +56 -56
  145. package/dist/extensions/optional/AGENT.md +10 -10
  146. package/dist/modes/interactive/controllers/input-submit-controller.js +2 -2
  147. package/dist/modes/interactive/controllers/stream-render-controller.js +2 -2
  148. package/dist/modes/interactive/interactive-mode.js +19 -19
  149. package/dist/modes/interactive/theme/dark.json +85 -85
  150. package/dist/modes/interactive/theme/light.json +84 -84
  151. package/dist/modes/interactive/theme/theme-schema.json +335 -335
  152. package/dist/modes/interactive/theme/warm.json +81 -81
  153. package/dist/node_modules/@pencil-agent/ai/dist/cli.js +0 -0
  154. package/dist/node_modules/@pencil-agent/ai/dist/models.generated.js +1 -1
  155. package/docs/ACP/345/215/217/350/256/256/351/233/206/346/210/220/345/274/200/345/217/221/346/226/207/346/241/243.md +851 -0
  156. package/docs/SDK-TESTING.md +364 -0
  157. package/docs/codex-goal-command-impl.md +1055 -1055
  158. package/docs/codex-goal-vs-grub.md +500 -500
  159. package/docs/custom-provider.md +27 -27
  160. package/docs/extensions.md +27 -27
  161. package/docs/keybindings.md +27 -27
  162. package/docs/loop /351/207/215/346/236/204/345/256/214/346/210/220/346/200/273/347/273/223.md" +250 -250
  163. package/docs/loop /351/207/215/346/236/204/345/256/214/346/210/220/346/212/245/345/221/212.md" +122 -122
  164. package/docs/loop /351/207/215/346/236/204/346/226/271/346/241/210.md" +1222 -1222
  165. package/docs/loop /351/207/215/346/236/204/346/226/271/346/241/210/345/256/236/347/216/260/346/212/245/345/221/212.md" +158 -158
  166. package/docs/loop /351/207/215/346/236/204/346/226/271/346/241/210/345/257/271/346/257/224/345/210/206/346/236/220.md" +128 -128
  167. package/docs/loop /351/207/215/346/236/204/350/256/241/345/210/222.md" +320 -320
  168. package/docs/loop-usage-examples.md +214 -214
  169. package/docs/mem-core/346/212/200/346/234/257/346/226/207/346/241/243.md +593 -0
  170. package/docs/models.md +27 -27
  171. package/docs/packages.md +27 -27
  172. package/docs/pi-design-philosophy.md +457 -457
  173. package/docs/planmode.md +1987 -1987
  174. package/docs/prompt-templates.md +27 -27
  175. package/docs/providers.md +27 -27
  176. package/docs/sdk.md +27 -27
  177. package/docs/skills.md +27 -27
  178. package/docs/startup-performance-optimization.md +301 -0
  179. package/docs/themes.md +27 -27
  180. package/docs/tui.md +27 -27
  181. package/docs//350/256/244/347/237/245/345/234/260/345/233/276.md +47 -0
  182. package/package.json +190 -190
  183. package/docs/cc-agent-design.md +0 -1297
  184. package/docs/cc-tui-design.md +0 -1333
  185. package/docs/nanoPencil-/345/255/246/344/271/240/350/256/241/345/210/222.md +0 -170
  186. package/docs/scan-report.md +0 -3820
  187. package/docs//345/257/271/346/240/207Claude-Code.md +0 -1775
  188. package/docs//351/230/277/351/207/214/345/267/264/345/267/264/350/264/242/346/212/245/345/210/206/346/236/220/344/271/246.md +0 -261
@@ -1,307 +1,307 @@
1
- # Product Hunt Scraping Skills
2
-
3
- Field-tested against https://www.producthunt.com on 2026-04-18.
4
- All selectors verified with actual browser runs.
5
-
6
- ---
7
-
8
- ## Page Structure Overview
9
-
10
- Product Hunt is a React SPA. Key structural facts discovered:
11
-
12
- - **No login wall** — all product data is accessible without signing in
13
- - **No cookie banner** — page loads cleanly with no consent dialogs
14
- - **Product URLs use `/products/` not `/posts/`** — the `a[href^="/posts/"]` selector matches nothing
15
- - **`data-test` attributes** are the most reliable selectors throughout the site
16
- - **4 homepage sections**: today, yesterday, last week, last month (5 products each, plus "see all")
17
- - **Today's votes are hidden** for the first 4 hours of each day (`—` instead of count)
18
- - **Homepage has 30 fixed post-items** — scrolling does NOT load more
19
- - **`goto_url()` may return `ERR_ABORTED`** for producthunt.com in some browser sessions — use `new_tab()` instead
20
-
21
- ---
22
-
23
- ## Navigation Pattern
24
-
25
- ```python
26
- # goto_url() may fail on Product Hunt — use new_tab() reliably
27
- tid = new_tab("https://www.producthunt.com")
28
- wait(4) # React SPA needs time; wait_for_load() alone is insufficient
29
- page = page_info()
30
- # Verify: url should be 'https://www.producthunt.com/'
31
- ```
32
-
33
- ---
34
-
35
- ## Homepage — Extract Daily Product Feed
36
-
37
- The homepage shows today's launches plus rolling sections for yesterday, last week, last month.
38
-
39
- ### Working selector: `[data-test^="post-item-"]`
40
-
41
- ```python
42
- # Full extraction with name, tagline, slug, votes, topics
43
- products = js("""
44
- JSON.stringify(
45
- Array.from(document.querySelectorAll('[data-test^="post-item-"]')).map(el => {
46
- var id = el.getAttribute('data-test').replace('post-item-', '');
47
- var nameEl = el.querySelector('[data-test^="post-name-"]');
48
- var productLink = el.querySelector('a[href^="/products/"]');
49
- var voteBtn = el.querySelector('[data-test="vote-button"]');
50
- var voteCount = voteBtn ? voteBtn.textContent.trim() : null;
51
- var topicLinks = Array.from(el.querySelectorAll('a[href^="/topics/"]')).map(a => a.textContent.trim());
52
- var name = nameEl ? nameEl.textContent.trim() : '';
53
- var lines = el.innerText.split('\\n').map(l => l.trim()).filter(l => l);
54
- var tagline = lines.find(l => l !== name && !topicLinks.includes(l) && l !== '•' && !/^[0-9—]/.test(l) && l.length > 5);
55
- return {
56
- id: id,
57
- name: name,
58
- slug: productLink ? productLink.getAttribute('href') : null,
59
- votes: voteCount,
60
- topics: topicLinks,
61
- tagline: tagline || null
62
- };
63
- })
64
- )
65
- """)
66
- ```
67
-
68
- **Sample output:**
69
- ```json
70
- [
71
- {"id":"1126372","name":"Vercel Flags","slug":"/products/vercel","votes":"—","topics":["Software Engineering","Developer Tools"],"tagline":"Feature flags, targeting rules, rollouts. All from Vercel."},
72
- {"id":"1125388","name":"1. Claude Opus 4.7","slug":"/products/claude-opus-4-7","votes":"466","topics":["API","Artificial Intelligence","Development"],"tagline":"Claude's most capable model for reasoning and agentic coding"}
73
- ]
74
- ```
75
-
76
- **Votes:**
77
- - `"—"` = vote count hidden (today's products during first 4 hours)
78
- - `"466"` = numeric string (yesterday/older products)
79
- - `"2,152"` = comma-formatted for large counts (parse: `voteCount.replace(',', '')`)
80
-
81
- **Name prefix**: Ranked products show rank in name: `"1. Claude Opus 4.7"` — strip with `re.sub(r'^\d+\. ', '', name)`.
82
-
83
- ---
84
-
85
- ## Daily Leaderboard — Best URL for Complete Daily Lists
86
-
87
- The leaderboard shows all products for any given day with actual vote counts.
88
-
89
- ```
90
- URL: https://www.producthunt.com/leaderboard/daily/YYYY/M/D
91
- Example: https://www.producthunt.com/leaderboard/daily/2026/4/18
92
- ```
93
-
94
- - Uses zero-padded-free month/day (April = `4`, not `04`)
95
- - Uses same `[data-test^="post-item-"]` selector
96
- - Shows 12–19 products per day
97
- - Same extraction JS as homepage works identically
98
-
99
- **Yesterday's results with real vote counts:**
100
- ```json
101
- {"id":"1125388","name":"1. Claude Opus 4.7","slug":"/products/claude-opus-4-7","votes":"466","tagline":"Claude's most capable model for reasoning and agentic coding"}
102
- {"id":"1118208","name":"2. Build Check","slug":"/products/build-check-for-outsiders","votes":"396"}
103
- ```
104
-
105
- ---
106
-
107
- ## Weekly Leaderboard
108
-
109
- ```
110
- URL: https://www.producthunt.com/leaderboard/weekly/YYYY/WW
111
- Example: https://www.producthunt.com/leaderboard/weekly/2026/16
112
- ```
113
-
114
- - Week number is ISO week (week 16 = April 13–19, 2026)
115
- - Current week may return 0 items until the week ends
116
- - Same `[data-test^="post-item-"]` selector
117
-
118
- ---
119
-
120
- ## Monthly Leaderboard
121
-
122
- ```
123
- URL: https://www.producthunt.com/leaderboard/monthly/YYYY/M
124
- Example: https://www.producthunt.com/leaderboard/monthly/2026/4
125
- ```
126
-
127
- ---
128
-
129
- ## Topic Page
130
-
131
- URL: `https://www.producthunt.com/topics/developer-tools`
132
-
133
- Selector changes on topic pages — uses `[data-test^="product:"]` not `post-item-`.
134
-
135
- ```python
136
- # Navigate to topic
137
- new_tab("https://www.producthunt.com/topics/developer-tools")
138
- wait(3)
139
-
140
- products = js("""
141
- JSON.stringify(
142
- Array.from(document.querySelectorAll('[data-test^="product:"]')).map(el => {
143
- var slug = el.getAttribute('data-test').replace('product:', '');
144
- var link = el.querySelector('a[href^="/products/"]');
145
- return {
146
- slug: slug,
147
- href: link ? link.getAttribute('href') : null,
148
- text: el.outerText.trim().substring(0, 200)
149
- };
150
- })
151
- )
152
- """)
153
- ```
154
-
155
- **Sample output:**
156
- ```json
157
- {"slug":"figma","href":"/products/figma","text":"FigmaThe collaborative interface design tool4.9 (1.4K reviews)..."}
158
- ```
159
-
160
- Returns ~15 top-rated products in the topic, not recent launches.
161
-
162
- ---
163
-
164
- ## Category Page
165
-
166
- URL: `https://www.producthunt.com/categories/ai-agents`
167
-
168
- Same `[data-test^="product:"]` selector as topics. Returns 15 top-reviewed products in that category.
169
-
170
- ```json
171
- {"slug":"elevenlabs","href":"/products/elevenlabs","text":"ElevenLabs\nCreate natural AI voices instantly...\n4.9 (165 reviews)"}
172
- ```
173
-
174
- ---
175
-
176
- ## Product Detail Page
177
-
178
- URL: `https://www.producthunt.com/products/claude-opus-4-7`
179
-
180
- ### Get total vote count (sidebar button)
181
- ```python
182
- # Use [data-test="vote-button"] — different from [data-test="action-bar-vote-button"]
183
- vote_text = js("document.querySelector('[data-test=\"vote-button\"]').outerText.trim().replace(/\\s+/g, ' ')")
184
- # Returns: "Upvote • 466 points"
185
- # Parse votes: vote_text.split('•')[1].strip().replace(' points', '').replace(',', '')
186
- ```
187
-
188
- ### Get review count and rating
189
- ```python
190
- review_link = js("JSON.stringify(Array.from(document.querySelectorAll('a')).filter(a => a.href && a.href.includes('/reviews') && a.outerText.includes('review')).map(a => a.outerText.trim()).slice(0, 1))")
191
- # Returns: ["1 review"] or ["5.0\n(731 reviews)"]
192
- ```
193
-
194
- ### Get day rank (sidebar shows "#1 Day Rank")
195
- No dedicated `data-test` for rank — parse from sidebar context or use leaderboard position.
196
-
197
- ### Comments (action-bar-vote-button)
198
- Each comment has its own `[data-test="action-bar-vote-button"]` with text like `"Upvote (13)"`.
199
-
200
- ---
201
-
202
- ## Search Results
203
-
204
- URL: `https://www.producthunt.com/search?q=AI+agent`
205
-
206
- Selector: `[data-test^="spotlight-result-product-"]`
207
-
208
- ```python
209
- new_tab("https://www.producthunt.com/search?q=AI+agent")
210
- wait(3)
211
-
212
- results = js("""
213
- JSON.stringify(
214
- Array.from(document.querySelectorAll('[data-test^="spotlight-result-product-"]')).map(el => {
215
- var id = el.getAttribute('data-test').replace('spotlight-result-product-', '');
216
- var lines = el.outerText.trim().split('\\n').map(l => l.trim()).filter(l => l);
217
- return {
218
- id: id,
219
- name: lines[0] || null,
220
- tagline: lines[1] || null,
221
- review_text: lines[2] || null
222
- };
223
- })
224
- )
225
- """)
226
- ```
227
-
228
- **Note:** Search result elements are `<button>` elements (not `<a>` links), so there is no `href` in the DOM. Product URL must be constructed: `https://www.producthunt.com/products/<slug>` where slug must be derived by other means. The element's `data-test` ID matches the numeric product ID, not the slug.
229
-
230
- **Sample output:**
231
- ```json
232
- {"id":"526014","name":"/ai","tagline":"Access ChatGPT anywhere you type '/ai'","review_text":"2 reviews"}
233
- {"id":"991302","name":"Naoma AI Demo Agent","tagline":"The first video agent that runs conversational product demos","review_text":"5 reviews"}
234
- ```
235
-
236
- ---
237
-
238
- ## Key Selector Reference
239
-
240
- | Page | Selector | Count | Notes |
241
- |------|----------|-------|-------|
242
- | Homepage | `[data-test^="post-item-"]` | 30 | 4 sections × ~5–7 products |
243
- | Homepage | `[data-test^="post-name-"]` | 30 | Product name elements |
244
- | Homepage/Leaderboard | `[data-test="vote-button"]` | varies | `—` for hidden; numeric for visible |
245
- | Topics/Categories | `[data-test^="product:"]` | ~15 | Top-rated products |
246
- | Search | `[data-test^="spotlight-result-product-"]` | 10 | Button elements, no href |
247
- | Product detail | `[data-test="vote-button"]` | 1 | Main vote: "Upvote • N points" |
248
- | Product detail | `[data-test="action-bar-vote-button"]` | many | Comment upvotes: "Upvote (N)" |
249
-
250
- ---
251
-
252
- ## Common Pitfalls
253
-
254
- 1. **`innerText` returns `None` on complex elements** — use `outerText` or break into simple single-property expressions. Avoid chaining DOM traversal inside `JSON.stringify()` on large objects.
255
-
256
- 2. **`goto_url()` returns `ERR_ABORTED`** for producthunt.com in some browser sessions — always use `new_tab("url")` instead.
257
-
258
- 3. **`a[href^="/posts/"]` matches nothing** — Product Hunt uses `/products/` for product URLs, not `/posts/`.
259
-
260
- 4. **Today's votes are always `—`** during the first 4 hours of the day — use yesterday's leaderboard for confirmed vote counts.
261
-
262
- 5. **Homepage does not lazy-load more products on scroll** — 30 items is the fixed set. Use leaderboard pages for complete daily listings.
263
-
264
- 6. **JSON.stringify of DOM-heavy objects returns `None`** — serialize only primitives (strings, numbers) not live DOM node properties.
265
-
266
- 7. **Ranked product names contain rank prefix** — `"1. Claude Opus 4.7"` — strip with regex `re.sub(r'^\d+\.\s+', '', name)`.
267
-
268
- 8. **`wait(3)` required after `wait_for_load()`** — the React SPA continues rendering after the load event.
269
-
270
- ---
271
-
272
- ## Recommended Workflow for Scraping Today's Launches
273
-
274
- ```python
275
- # 1. Open Product Hunt in a new tab
276
- new_tab("https://www.producthunt.com/leaderboard/daily/2026/4/18")
277
- wait(4)
278
-
279
- # 2. Extract all products with metadata
280
- products = js("""
281
- JSON.stringify(
282
- Array.from(document.querySelectorAll('[data-test^="post-item-"]')).map(el => {
283
- var id = el.getAttribute('data-test').replace('post-item-', '');
284
- var nameEl = el.querySelector('[data-test^="post-name-"]');
285
- var productLink = el.querySelector('a[href^="/products/"]');
286
- var voteBtn = el.querySelector('[data-test="vote-button"]');
287
- var topicLinks = Array.from(el.querySelectorAll('a[href^="/topics/"]')).map(a => a.textContent.trim());
288
- var name = nameEl ? nameEl.textContent.trim() : '';
289
- var lines = el.innerText.split('\\n').map(l => l.trim()).filter(l => l);
290
- var tagline = lines.find(l => l !== name && !topicLinks.includes(l) && l !== '•' && !/^[0-9—]/.test(l) && l.length > 5);
291
- return {
292
- id: id,
293
- name: name,
294
- slug: productLink ? productLink.getAttribute('href') : null,
295
- votes: voteBtn ? voteBtn.textContent.trim() : null,
296
- topics: topicLinks,
297
- tagline: tagline || null
298
- };
299
- })
300
- )
301
- """)
302
- import json
303
- data = json.loads(products)
304
- print(f"Found {len(data)} products")
305
- for p in data:
306
- print(f" {p['name']} — {p['votes']} votes — {p['tagline']}")
307
- ```
1
+ # Product Hunt Scraping Skills
2
+
3
+ Field-tested against https://www.producthunt.com on 2026-04-18.
4
+ All selectors verified with actual browser runs.
5
+
6
+ ---
7
+
8
+ ## Page Structure Overview
9
+
10
+ Product Hunt is a React SPA. Key structural facts discovered:
11
+
12
+ - **No login wall** — all product data is accessible without signing in
13
+ - **No cookie banner** — page loads cleanly with no consent dialogs
14
+ - **Product URLs use `/products/` not `/posts/`** — the `a[href^="/posts/"]` selector matches nothing
15
+ - **`data-test` attributes** are the most reliable selectors throughout the site
16
+ - **4 homepage sections**: today, yesterday, last week, last month (5 products each, plus "see all")
17
+ - **Today's votes are hidden** for the first 4 hours of each day (`—` instead of count)
18
+ - **Homepage has 30 fixed post-items** — scrolling does NOT load more
19
+ - **`goto_url()` may return `ERR_ABORTED`** for producthunt.com in some browser sessions — use `new_tab()` instead
20
+
21
+ ---
22
+
23
+ ## Navigation Pattern
24
+
25
+ ```python
26
+ # goto_url() may fail on Product Hunt — use new_tab() reliably
27
+ tid = new_tab("https://www.producthunt.com")
28
+ wait(4) # React SPA needs time; wait_for_load() alone is insufficient
29
+ page = page_info()
30
+ # Verify: url should be 'https://www.producthunt.com/'
31
+ ```
32
+
33
+ ---
34
+
35
+ ## Homepage — Extract Daily Product Feed
36
+
37
+ The homepage shows today's launches plus rolling sections for yesterday, last week, last month.
38
+
39
+ ### Working selector: `[data-test^="post-item-"]`
40
+
41
+ ```python
42
+ # Full extraction with name, tagline, slug, votes, topics
43
+ products = js("""
44
+ JSON.stringify(
45
+ Array.from(document.querySelectorAll('[data-test^="post-item-"]')).map(el => {
46
+ var id = el.getAttribute('data-test').replace('post-item-', '');
47
+ var nameEl = el.querySelector('[data-test^="post-name-"]');
48
+ var productLink = el.querySelector('a[href^="/products/"]');
49
+ var voteBtn = el.querySelector('[data-test="vote-button"]');
50
+ var voteCount = voteBtn ? voteBtn.textContent.trim() : null;
51
+ var topicLinks = Array.from(el.querySelectorAll('a[href^="/topics/"]')).map(a => a.textContent.trim());
52
+ var name = nameEl ? nameEl.textContent.trim() : '';
53
+ var lines = el.innerText.split('\\n').map(l => l.trim()).filter(l => l);
54
+ var tagline = lines.find(l => l !== name && !topicLinks.includes(l) && l !== '•' && !/^[0-9—]/.test(l) && l.length > 5);
55
+ return {
56
+ id: id,
57
+ name: name,
58
+ slug: productLink ? productLink.getAttribute('href') : null,
59
+ votes: voteCount,
60
+ topics: topicLinks,
61
+ tagline: tagline || null
62
+ };
63
+ })
64
+ )
65
+ """)
66
+ ```
67
+
68
+ **Sample output:**
69
+ ```json
70
+ [
71
+ {"id":"1126372","name":"Vercel Flags","slug":"/products/vercel","votes":"—","topics":["Software Engineering","Developer Tools"],"tagline":"Feature flags, targeting rules, rollouts. All from Vercel."},
72
+ {"id":"1125388","name":"1. Claude Opus 4.7","slug":"/products/claude-opus-4-7","votes":"466","topics":["API","Artificial Intelligence","Development"],"tagline":"Claude's most capable model for reasoning and agentic coding"}
73
+ ]
74
+ ```
75
+
76
+ **Votes:**
77
+ - `"—"` = vote count hidden (today's products during first 4 hours)
78
+ - `"466"` = numeric string (yesterday/older products)
79
+ - `"2,152"` = comma-formatted for large counts (parse: `voteCount.replace(',', '')`)
80
+
81
+ **Name prefix**: Ranked products show rank in name: `"1. Claude Opus 4.7"` — strip with `re.sub(r'^\d+\. ', '', name)`.
82
+
83
+ ---
84
+
85
+ ## Daily Leaderboard — Best URL for Complete Daily Lists
86
+
87
+ The leaderboard shows all products for any given day with actual vote counts.
88
+
89
+ ```
90
+ URL: https://www.producthunt.com/leaderboard/daily/YYYY/M/D
91
+ Example: https://www.producthunt.com/leaderboard/daily/2026/4/18
92
+ ```
93
+
94
+ - Uses zero-padded-free month/day (April = `4`, not `04`)
95
+ - Uses same `[data-test^="post-item-"]` selector
96
+ - Shows 12–19 products per day
97
+ - Same extraction JS as homepage works identically
98
+
99
+ **Yesterday's results with real vote counts:**
100
+ ```json
101
+ {"id":"1125388","name":"1. Claude Opus 4.7","slug":"/products/claude-opus-4-7","votes":"466","tagline":"Claude's most capable model for reasoning and agentic coding"}
102
+ {"id":"1118208","name":"2. Build Check","slug":"/products/build-check-for-outsiders","votes":"396"}
103
+ ```
104
+
105
+ ---
106
+
107
+ ## Weekly Leaderboard
108
+
109
+ ```
110
+ URL: https://www.producthunt.com/leaderboard/weekly/YYYY/WW
111
+ Example: https://www.producthunt.com/leaderboard/weekly/2026/16
112
+ ```
113
+
114
+ - Week number is ISO week (week 16 = April 13–19, 2026)
115
+ - Current week may return 0 items until the week ends
116
+ - Same `[data-test^="post-item-"]` selector
117
+
118
+ ---
119
+
120
+ ## Monthly Leaderboard
121
+
122
+ ```
123
+ URL: https://www.producthunt.com/leaderboard/monthly/YYYY/M
124
+ Example: https://www.producthunt.com/leaderboard/monthly/2026/4
125
+ ```
126
+
127
+ ---
128
+
129
+ ## Topic Page
130
+
131
+ URL: `https://www.producthunt.com/topics/developer-tools`
132
+
133
+ Selector changes on topic pages — uses `[data-test^="product:"]` not `post-item-`.
134
+
135
+ ```python
136
+ # Navigate to topic
137
+ new_tab("https://www.producthunt.com/topics/developer-tools")
138
+ wait(3)
139
+
140
+ products = js("""
141
+ JSON.stringify(
142
+ Array.from(document.querySelectorAll('[data-test^="product:"]')).map(el => {
143
+ var slug = el.getAttribute('data-test').replace('product:', '');
144
+ var link = el.querySelector('a[href^="/products/"]');
145
+ return {
146
+ slug: slug,
147
+ href: link ? link.getAttribute('href') : null,
148
+ text: el.outerText.trim().substring(0, 200)
149
+ };
150
+ })
151
+ )
152
+ """)
153
+ ```
154
+
155
+ **Sample output:**
156
+ ```json
157
+ {"slug":"figma","href":"/products/figma","text":"FigmaThe collaborative interface design tool4.9 (1.4K reviews)..."}
158
+ ```
159
+
160
+ Returns ~15 top-rated products in the topic, not recent launches.
161
+
162
+ ---
163
+
164
+ ## Category Page
165
+
166
+ URL: `https://www.producthunt.com/categories/ai-agents`
167
+
168
+ Same `[data-test^="product:"]` selector as topics. Returns 15 top-reviewed products in that category.
169
+
170
+ ```json
171
+ {"slug":"elevenlabs","href":"/products/elevenlabs","text":"ElevenLabs\nCreate natural AI voices instantly...\n4.9 (165 reviews)"}
172
+ ```
173
+
174
+ ---
175
+
176
+ ## Product Detail Page
177
+
178
+ URL: `https://www.producthunt.com/products/claude-opus-4-7`
179
+
180
+ ### Get total vote count (sidebar button)
181
+ ```python
182
+ # Use [data-test="vote-button"] — different from [data-test="action-bar-vote-button"]
183
+ vote_text = js("document.querySelector('[data-test=\"vote-button\"]').outerText.trim().replace(/\\s+/g, ' ')")
184
+ # Returns: "Upvote • 466 points"
185
+ # Parse votes: vote_text.split('•')[1].strip().replace(' points', '').replace(',', '')
186
+ ```
187
+
188
+ ### Get review count and rating
189
+ ```python
190
+ review_link = js("JSON.stringify(Array.from(document.querySelectorAll('a')).filter(a => a.href && a.href.includes('/reviews') && a.outerText.includes('review')).map(a => a.outerText.trim()).slice(0, 1))")
191
+ # Returns: ["1 review"] or ["5.0\n(731 reviews)"]
192
+ ```
193
+
194
+ ### Get day rank (sidebar shows "#1 Day Rank")
195
+ No dedicated `data-test` for rank — parse from sidebar context or use leaderboard position.
196
+
197
+ ### Comments (action-bar-vote-button)
198
+ Each comment has its own `[data-test="action-bar-vote-button"]` with text like `"Upvote (13)"`.
199
+
200
+ ---
201
+
202
+ ## Search Results
203
+
204
+ URL: `https://www.producthunt.com/search?q=AI+agent`
205
+
206
+ Selector: `[data-test^="spotlight-result-product-"]`
207
+
208
+ ```python
209
+ new_tab("https://www.producthunt.com/search?q=AI+agent")
210
+ wait(3)
211
+
212
+ results = js("""
213
+ JSON.stringify(
214
+ Array.from(document.querySelectorAll('[data-test^="spotlight-result-product-"]')).map(el => {
215
+ var id = el.getAttribute('data-test').replace('spotlight-result-product-', '');
216
+ var lines = el.outerText.trim().split('\\n').map(l => l.trim()).filter(l => l);
217
+ return {
218
+ id: id,
219
+ name: lines[0] || null,
220
+ tagline: lines[1] || null,
221
+ review_text: lines[2] || null
222
+ };
223
+ })
224
+ )
225
+ """)
226
+ ```
227
+
228
+ **Note:** Search result elements are `<button>` elements (not `<a>` links), so there is no `href` in the DOM. Product URL must be constructed: `https://www.producthunt.com/products/<slug>` where slug must be derived by other means. The element's `data-test` ID matches the numeric product ID, not the slug.
229
+
230
+ **Sample output:**
231
+ ```json
232
+ {"id":"526014","name":"/ai","tagline":"Access ChatGPT anywhere you type '/ai'","review_text":"2 reviews"}
233
+ {"id":"991302","name":"Naoma AI Demo Agent","tagline":"The first video agent that runs conversational product demos","review_text":"5 reviews"}
234
+ ```
235
+
236
+ ---
237
+
238
+ ## Key Selector Reference
239
+
240
+ | Page | Selector | Count | Notes |
241
+ |------|----------|-------|-------|
242
+ | Homepage | `[data-test^="post-item-"]` | 30 | 4 sections × ~5–7 products |
243
+ | Homepage | `[data-test^="post-name-"]` | 30 | Product name elements |
244
+ | Homepage/Leaderboard | `[data-test="vote-button"]` | varies | `—` for hidden; numeric for visible |
245
+ | Topics/Categories | `[data-test^="product:"]` | ~15 | Top-rated products |
246
+ | Search | `[data-test^="spotlight-result-product-"]` | 10 | Button elements, no href |
247
+ | Product detail | `[data-test="vote-button"]` | 1 | Main vote: "Upvote • N points" |
248
+ | Product detail | `[data-test="action-bar-vote-button"]` | many | Comment upvotes: "Upvote (N)" |
249
+
250
+ ---
251
+
252
+ ## Common Pitfalls
253
+
254
+ 1. **`innerText` returns `None` on complex elements** — use `outerText` or break into simple single-property expressions. Avoid chaining DOM traversal inside `JSON.stringify()` on large objects.
255
+
256
+ 2. **`goto_url()` returns `ERR_ABORTED`** for producthunt.com in some browser sessions — always use `new_tab("url")` instead.
257
+
258
+ 3. **`a[href^="/posts/"]` matches nothing** — Product Hunt uses `/products/` for product URLs, not `/posts/`.
259
+
260
+ 4. **Today's votes are always `—`** during the first 4 hours of the day — use yesterday's leaderboard for confirmed vote counts.
261
+
262
+ 5. **Homepage does not lazy-load more products on scroll** — 30 items is the fixed set. Use leaderboard pages for complete daily listings.
263
+
264
+ 6. **JSON.stringify of DOM-heavy objects returns `None`** — serialize only primitives (strings, numbers) not live DOM node properties.
265
+
266
+ 7. **Ranked product names contain rank prefix** — `"1. Claude Opus 4.7"` — strip with regex `re.sub(r'^\d+\.\s+', '', name)`.
267
+
268
+ 8. **`wait(3)` required after `wait_for_load()`** — the React SPA continues rendering after the load event.
269
+
270
+ ---
271
+
272
+ ## Recommended Workflow for Scraping Today's Launches
273
+
274
+ ```python
275
+ # 1. Open Product Hunt in a new tab
276
+ new_tab("https://www.producthunt.com/leaderboard/daily/2026/4/18")
277
+ wait(4)
278
+
279
+ # 2. Extract all products with metadata
280
+ products = js("""
281
+ JSON.stringify(
282
+ Array.from(document.querySelectorAll('[data-test^="post-item-"]')).map(el => {
283
+ var id = el.getAttribute('data-test').replace('post-item-', '');
284
+ var nameEl = el.querySelector('[data-test^="post-name-"]');
285
+ var productLink = el.querySelector('a[href^="/products/"]');
286
+ var voteBtn = el.querySelector('[data-test="vote-button"]');
287
+ var topicLinks = Array.from(el.querySelectorAll('a[href^="/topics/"]')).map(a => a.textContent.trim());
288
+ var name = nameEl ? nameEl.textContent.trim() : '';
289
+ var lines = el.innerText.split('\\n').map(l => l.trim()).filter(l => l);
290
+ var tagline = lines.find(l => l !== name && !topicLinks.includes(l) && l !== '•' && !/^[0-9—]/.test(l) && l.length > 5);
291
+ return {
292
+ id: id,
293
+ name: name,
294
+ slug: productLink ? productLink.getAttribute('href') : null,
295
+ votes: voteBtn ? voteBtn.textContent.trim() : null,
296
+ topics: topicLinks,
297
+ tagline: tagline || null
298
+ };
299
+ })
300
+ )
301
+ """)
302
+ import json
303
+ data = json.loads(products)
304
+ print(f"Found {len(data)} products")
305
+ for p in data:
306
+ print(f" {p['name']} — {p['votes']} votes — {p['tagline']}")
307
+ ```