@pencil-agent/nano-pencil 2.0.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. package/README.md +267 -267
  2. package/dist/build-meta.json +3 -3
  3. package/dist/core/export-html/AGENT.md +11 -11
  4. package/dist/core/export-html/template.css +971 -971
  5. package/dist/core/export-html/template.html +54 -54
  6. package/dist/core/mcp/mcp-client.d.ts +3 -1
  7. package/dist/core/mcp/mcp-client.js +6 -6
  8. package/dist/core/mcp/mcp-config.d.ts +3 -3
  9. package/dist/core/mcp/mcp-config.js +1 -1
  10. package/dist/core/mcp/mcp-manager.d.ts +5 -1
  11. package/dist/core/mcp/mcp-manager.js +1 -1
  12. package/dist/core/platform/config/resource-loader.d.ts +2 -0
  13. package/dist/core/platform/config/resource-loader.js +2 -2
  14. package/dist/core/runtime/agent-session.d.ts +12 -0
  15. package/dist/core/runtime/agent-session.js +8 -8
  16. package/dist/core/runtime/sdk.d.ts +8 -0
  17. package/dist/core/runtime/sdk.js +1 -1
  18. package/dist/extensions/builtin/AGENT.md +115 -115
  19. package/dist/extensions/builtin/browser/AGENT.md +17 -17
  20. package/dist/extensions/builtin/browser/agent-workspace/agent_helpers.py +12 -12
  21. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/amazon/product-search.md +198 -198
  22. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/archive-org/scraping.md +341 -341
  23. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/arxiv/scraping.md +311 -311
  24. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/arxiv-bulk/scraping.md +333 -333
  25. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/atlas/overview.md +70 -70
  26. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/booking-com/scraping.md +578 -578
  27. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/capterra/scraping.md +440 -440
  28. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/centilebrain/generate-estimates.md +110 -110
  29. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/coingecko/scraping.md +325 -325
  30. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/coinmarketcap/scraping.md +463 -463
  31. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/coursera/scraping.md +360 -360
  32. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/craigslist/scraping.md +390 -390
  33. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/crossref/scraping.md +568 -568
  34. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/dev-to/scraping.md +323 -323
  35. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/duckduckgo/scraping.md +349 -349
  36. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/ebay/scraping.md +435 -435
  37. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/etsy/scraping.md +506 -506
  38. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/eventbrite/scraping.md +363 -363
  39. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/expedia/automation.md +168 -168
  40. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/facebook/groups.md +236 -236
  41. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/facebook/pages.md +295 -295
  42. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/framer/editor.md +108 -108
  43. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/fred/scraping.md +493 -493
  44. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/g2/scraping.md +580 -580
  45. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/genius/scraping.md +511 -511
  46. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/github/repo-actions.md +65 -65
  47. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/github/scraping.md +184 -184
  48. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/glassdoor/scraping.md +543 -543
  49. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/gmail/compose.md +122 -122
  50. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/goodreads/scraping.md +461 -461
  51. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/gutenberg/scraping.md +383 -383
  52. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/hackernews/scraping.md +243 -243
  53. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/howlongtobeat/scraping.md +473 -473
  54. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/imdb/scraping.md +271 -271
  55. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/itch-io/scraping.md +436 -436
  56. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/job-boards/indeed-glassdoor.md +1021 -1021
  57. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/letterboxd/scraping.md +349 -349
  58. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/linkedin/invitation-manager.md +109 -109
  59. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/loom/folder-enumeration.md +170 -170
  60. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/macrotrends/scraping.md +537 -537
  61. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/medium/article-hydration.md +120 -120
  62. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/medium/scraping.md +414 -414
  63. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/metacritic/scraping.md +477 -477
  64. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/musicbrainz/scraping.md +478 -478
  65. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/nasa/scraping.md +339 -339
  66. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/news-aggregation/multi-source.md +205 -205
  67. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/open-library/scraping.md +472 -472
  68. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/openalex/scraping.md +470 -470
  69. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/openstreetmap/scraping.md +490 -490
  70. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/package-registries/npm-pypi.md +478 -478
  71. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/polymarket/scraping.md +234 -234
  72. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/producthunt/scraping.md +307 -307
  73. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/pubmed/scraping.md +421 -421
  74. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/quora/scraping.md +364 -364
  75. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/rawg/scraping.md +352 -352
  76. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/reddit/scraping.md +124 -124
  77. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/rest-countries/scraping.md +233 -233
  78. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/sec-edgar/scraping.md +361 -361
  79. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/shopify-admin/README.md +36 -36
  80. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/shopify-admin/embedded-apps.md +72 -72
  81. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/shopify-admin/knowledge-base.md +109 -109
  82. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/shopify-admin/polaris-inputs.md +137 -137
  83. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/soundcloud/scraping.md +362 -362
  84. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/spotify/scraping.md +339 -339
  85. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/stackoverflow/scraping.md +435 -435
  86. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/steam/scraping.md +575 -575
  87. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/substack/scraping.md +338 -338
  88. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/thetechgeeks/pricing.md +52 -52
  89. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/tiktok/upload.md +107 -107
  90. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/tradingview/scraping.md +309 -309
  91. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/trello/boards-and-lists.md +88 -88
  92. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/trustpilot/scraping.md +375 -375
  93. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/walmart/scraping.md +444 -444
  94. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/wayback-machine/scraping.md +306 -306
  95. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/weather/scraping.md +398 -398
  96. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/wellfound/scraping.md +596 -596
  97. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/world-bank/scraping.md +356 -356
  98. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/xiaohongshu/scraping.md +84 -84
  99. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/youtube/scraping.md +418 -418
  100. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/zillow/scraping.md +433 -433
  101. package/dist/extensions/builtin/browser/browser.md +73 -73
  102. package/dist/extensions/builtin/browser/install.md +142 -142
  103. package/dist/extensions/builtin/browser/interaction-skills/connection.md +48 -48
  104. package/dist/extensions/builtin/browser/interaction-skills/cookies.md +3 -3
  105. package/dist/extensions/builtin/browser/interaction-skills/cross-origin-iframes.md +3 -3
  106. package/dist/extensions/builtin/browser/interaction-skills/dialogs.md +64 -64
  107. package/dist/extensions/builtin/browser/interaction-skills/downloads.md +3 -3
  108. package/dist/extensions/builtin/browser/interaction-skills/drag-and-drop.md +3 -3
  109. package/dist/extensions/builtin/browser/interaction-skills/dropdowns.md +3 -3
  110. package/dist/extensions/builtin/browser/interaction-skills/iframes.md +3 -3
  111. package/dist/extensions/builtin/browser/interaction-skills/network-requests.md +3 -3
  112. package/dist/extensions/builtin/browser/interaction-skills/print-as-pdf.md +3 -3
  113. package/dist/extensions/builtin/browser/interaction-skills/profile-sync.md +90 -90
  114. package/dist/extensions/builtin/browser/interaction-skills/screenshots.md +17 -17
  115. package/dist/extensions/builtin/browser/interaction-skills/scrolling.md +3 -3
  116. package/dist/extensions/builtin/browser/interaction-skills/shadow-dom.md +3 -3
  117. package/dist/extensions/builtin/browser/interaction-skills/tabs.md +69 -69
  118. package/dist/extensions/builtin/browser/interaction-skills/uploads.md +1 -1
  119. package/dist/extensions/builtin/browser/interaction-skills/viewport.md +3 -3
  120. package/dist/extensions/builtin/browser/src/browser_harness/AGENT.md +15 -15
  121. package/dist/extensions/builtin/browser/src/browser_harness/__init__.py +8 -8
  122. package/dist/extensions/builtin/browser/src/browser_harness/_ipc.py +90 -90
  123. package/dist/extensions/builtin/browser/src/browser_harness/admin.py +722 -722
  124. package/dist/extensions/builtin/browser/src/browser_harness/daemon.py +328 -328
  125. package/dist/extensions/builtin/browser/src/browser_harness/helpers.py +396 -396
  126. package/dist/extensions/builtin/browser/src/browser_harness/run.py +103 -103
  127. package/dist/extensions/builtin/discipline/skills/brainstorming/SKILL.md +33 -33
  128. package/dist/extensions/builtin/discipline/skills/executing-plans/SKILL.md +25 -25
  129. package/dist/extensions/builtin/discipline/skills/finishing-development-branch/SKILL.md +25 -25
  130. package/dist/extensions/builtin/discipline/skills/receiving-code-review/SKILL.md +22 -22
  131. package/dist/extensions/builtin/discipline/skills/requesting-code-review/SKILL.md +31 -31
  132. package/dist/extensions/builtin/discipline/skills/systematic-debugging/SKILL.md +28 -28
  133. package/dist/extensions/builtin/discipline/skills/test-driven-development/SKILL.md +32 -32
  134. package/dist/extensions/builtin/discipline/skills/using-git-worktrees/SKILL.md +25 -25
  135. package/dist/extensions/builtin/discipline/skills/verification-before-completion/SKILL.md +27 -27
  136. package/dist/extensions/builtin/discipline/skills/writing-plans/SKILL.md +26 -26
  137. package/dist/extensions/builtin/goal/README.md +67 -67
  138. package/dist/extensions/builtin/grub/README.md +112 -112
  139. package/dist/extensions/builtin/link-world/agent-workspace/README.md +16 -16
  140. package/dist/extensions/builtin/link-world/internet-search/internet-search.md +65 -65
  141. package/dist/extensions/builtin/link-world/link-world-agent.md +82 -82
  142. package/dist/extensions/builtin/link-world/linkworld.md +313 -313
  143. package/dist/extensions/builtin/link-world/network-routing/network-routing.md +67 -67
  144. package/dist/extensions/builtin/loop/README.md +92 -92
  145. package/dist/extensions/builtin/mcp/figma-design.md +68 -68
  146. package/dist/extensions/builtin/mcp/mcp-management.md +85 -85
  147. package/dist/extensions/builtin/recap/AGENT.md +15 -15
  148. package/dist/extensions/builtin/sal/README.md +72 -72
  149. package/dist/extensions/builtin/security-audit/README.md +289 -289
  150. package/dist/extensions/builtin/team/AGENT.md +112 -112
  151. package/dist/extensions/builtin/team/TESTING.md +299 -299
  152. package/dist/extensions/builtin/token-save/README.md +56 -56
  153. package/dist/extensions/optional/AGENT.md +10 -10
  154. package/dist/modes/interactive/interactive-mode.js +36 -36
  155. package/dist/modes/interactive/theme/dark.json +85 -85
  156. package/dist/modes/interactive/theme/light.json +84 -84
  157. package/dist/modes/interactive/theme/theme-schema.json +335 -335
  158. package/dist/modes/interactive/theme/warm.json +81 -81
  159. package/dist/node_modules/@pencil-agent/agent-core/dist/agent-loop.js +3 -2
  160. package/dist/node_modules/@pencil-agent/agent-core/dist/structured-adaptive-agent-loop.js +2 -1
  161. package/dist/node_modules/@pencil-agent/ai/dist/cli.js +0 -0
  162. package/docs/cc-agent-design.md +1297 -0
  163. package/docs/cc-tui-design.md +1333 -0
  164. package/docs/codex-goal-command-impl.md +1055 -1055
  165. package/docs/codex-goal-vs-grub.md +500 -500
  166. package/docs/custom-provider.md +27 -27
  167. package/docs/extensions.md +27 -27
  168. package/docs/keybindings.md +27 -27
  169. package/docs/loop /351/207/215/346/236/204/345/256/214/346/210/220/346/200/273/347/273/223.md" +250 -250
  170. package/docs/loop /351/207/215/346/236/204/345/256/214/346/210/220/346/212/245/345/221/212.md" +122 -122
  171. package/docs/loop /351/207/215/346/236/204/346/226/271/346/241/210.md" +1222 -1222
  172. package/docs/loop /351/207/215/346/236/204/346/226/271/346/241/210/345/256/236/347/216/260/346/212/245/345/221/212.md" +158 -158
  173. package/docs/loop /351/207/215/346/236/204/346/226/271/346/241/210/345/257/271/346/257/224/345/210/206/346/236/220.md" +128 -128
  174. package/docs/loop /351/207/215/346/236/204/350/256/241/345/210/222.md" +320 -320
  175. package/docs/loop-usage-examples.md +214 -214
  176. package/docs/models.md +27 -27
  177. package/docs/nanoPencil-/345/255/246/344/271/240/350/256/241/345/210/222.md +170 -0
  178. package/docs/packages.md +27 -27
  179. package/docs/pi-design-philosophy.md +457 -457
  180. package/docs/planmode.md +1987 -1987
  181. package/docs/prompt-templates.md +27 -27
  182. package/docs/providers.md +27 -27
  183. package/docs/scan-report.md +3820 -0
  184. package/docs/sdk.md +27 -27
  185. package/docs/skills.md +27 -27
  186. package/docs/themes.md +27 -27
  187. package/docs/tui.md +27 -27
  188. package/docs//345/257/271/346/240/207Claude-Code.md +1775 -0
  189. package/docs//351/230/277/351/207/214/345/267/264/345/267/264/350/264/242/346/212/245/345/210/206/346/236/220/344/271/246.md +261 -0
  190. package/package.json +190 -190
  191. package/docs/ACP/345/215/217/350/256/256/351/233/206/346/210/220/345/274/200/345/217/221/346/226/207/346/241/243.md +0 -851
  192. package/docs/SDK-TESTING.md +0 -364
  193. package/docs/mem-core/346/212/200/346/234/257/346/226/207/346/241/243.md +0 -593
  194. package/docs/startup-performance-optimization.md +0 -301
  195. package/docs//350/256/244/347/237/245/345/234/260/345/233/276.md +0 -47
@@ -1,307 +1,307 @@
1
- # Product Hunt Scraping Skills
2
-
3
- Field-tested against https://www.producthunt.com on 2026-04-18.
4
- All selectors verified with actual browser runs.
5
-
6
- ---
7
-
8
- ## Page Structure Overview
9
-
10
- Product Hunt is a React SPA. Key structural facts discovered:
11
-
12
- - **No login wall** — all product data is accessible without signing in
13
- - **No cookie banner** — page loads cleanly with no consent dialogs
14
- - **Product URLs use `/products/` not `/posts/`** — the `a[href^="/posts/"]` selector matches nothing
15
- - **`data-test` attributes** are the most reliable selectors throughout the site
16
- - **4 homepage sections**: today, yesterday, last week, last month (5 products each, plus "see all")
17
- - **Today's votes are hidden** for the first 4 hours of each day (`—` instead of count)
18
- - **Homepage has 30 fixed post-items** — scrolling does NOT load more
19
- - **`goto_url()` may return `ERR_ABORTED`** for producthunt.com in some browser sessions — use `new_tab()` instead
20
-
21
- ---
22
-
23
- ## Navigation Pattern
24
-
25
- ```python
26
- # goto_url() may fail on Product Hunt — use new_tab() reliably
27
- tid = new_tab("https://www.producthunt.com")
28
- wait(4) # React SPA needs time; wait_for_load() alone is insufficient
29
- page = page_info()
30
- # Verify: url should be 'https://www.producthunt.com/'
31
- ```
32
-
33
- ---
34
-
35
- ## Homepage — Extract Daily Product Feed
36
-
37
- The homepage shows today's launches plus rolling sections for yesterday, last week, last month.
38
-
39
- ### Working selector: `[data-test^="post-item-"]`
40
-
41
- ```python
42
- # Full extraction with name, tagline, slug, votes, topics
43
- products = js("""
44
- JSON.stringify(
45
- Array.from(document.querySelectorAll('[data-test^="post-item-"]')).map(el => {
46
- var id = el.getAttribute('data-test').replace('post-item-', '');
47
- var nameEl = el.querySelector('[data-test^="post-name-"]');
48
- var productLink = el.querySelector('a[href^="/products/"]');
49
- var voteBtn = el.querySelector('[data-test="vote-button"]');
50
- var voteCount = voteBtn ? voteBtn.textContent.trim() : null;
51
- var topicLinks = Array.from(el.querySelectorAll('a[href^="/topics/"]')).map(a => a.textContent.trim());
52
- var name = nameEl ? nameEl.textContent.trim() : '';
53
- var lines = el.innerText.split('\\n').map(l => l.trim()).filter(l => l);
54
- var tagline = lines.find(l => l !== name && !topicLinks.includes(l) && l !== '•' && !/^[0-9—]/.test(l) && l.length > 5);
55
- return {
56
- id: id,
57
- name: name,
58
- slug: productLink ? productLink.getAttribute('href') : null,
59
- votes: voteCount,
60
- topics: topicLinks,
61
- tagline: tagline || null
62
- };
63
- })
64
- )
65
- """)
66
- ```
67
-
68
- **Sample output:**
69
- ```json
70
- [
71
- {"id":"1126372","name":"Vercel Flags","slug":"/products/vercel","votes":"—","topics":["Software Engineering","Developer Tools"],"tagline":"Feature flags, targeting rules, rollouts. All from Vercel."},
72
- {"id":"1125388","name":"1. Claude Opus 4.7","slug":"/products/claude-opus-4-7","votes":"466","topics":["API","Artificial Intelligence","Development"],"tagline":"Claude's most capable model for reasoning and agentic coding"}
73
- ]
74
- ```
75
-
76
- **Votes:**
77
- - `"—"` = vote count hidden (today's products during first 4 hours)
78
- - `"466"` = numeric string (yesterday/older products)
79
- - `"2,152"` = comma-formatted for large counts (parse: `voteCount.replace(',', '')`)
80
-
81
- **Name prefix**: Ranked products show rank in name: `"1. Claude Opus 4.7"` — strip with `re.sub(r'^\d+\. ', '', name)`.
82
-
83
- ---
84
-
85
- ## Daily Leaderboard — Best URL for Complete Daily Lists
86
-
87
- The leaderboard shows all products for any given day with actual vote counts.
88
-
89
- ```
90
- URL: https://www.producthunt.com/leaderboard/daily/YYYY/M/D
91
- Example: https://www.producthunt.com/leaderboard/daily/2026/4/18
92
- ```
93
-
94
- - Uses zero-padded-free month/day (April = `4`, not `04`)
95
- - Uses same `[data-test^="post-item-"]` selector
96
- - Shows 12–19 products per day
97
- - Same extraction JS as homepage works identically
98
-
99
- **Yesterday's results with real vote counts:**
100
- ```json
101
- {"id":"1125388","name":"1. Claude Opus 4.7","slug":"/products/claude-opus-4-7","votes":"466","tagline":"Claude's most capable model for reasoning and agentic coding"}
102
- {"id":"1118208","name":"2. Build Check","slug":"/products/build-check-for-outsiders","votes":"396"}
103
- ```
104
-
105
- ---
106
-
107
- ## Weekly Leaderboard
108
-
109
- ```
110
- URL: https://www.producthunt.com/leaderboard/weekly/YYYY/WW
111
- Example: https://www.producthunt.com/leaderboard/weekly/2026/16
112
- ```
113
-
114
- - Week number is ISO week (week 16 = April 13–19, 2026)
115
- - Current week may return 0 items until the week ends
116
- - Same `[data-test^="post-item-"]` selector
117
-
118
- ---
119
-
120
- ## Monthly Leaderboard
121
-
122
- ```
123
- URL: https://www.producthunt.com/leaderboard/monthly/YYYY/M
124
- Example: https://www.producthunt.com/leaderboard/monthly/2026/4
125
- ```
126
-
127
- ---
128
-
129
- ## Topic Page
130
-
131
- URL: `https://www.producthunt.com/topics/developer-tools`
132
-
133
- Selector changes on topic pages — uses `[data-test^="product:"]` not `post-item-`.
134
-
135
- ```python
136
- # Navigate to topic
137
- new_tab("https://www.producthunt.com/topics/developer-tools")
138
- wait(3)
139
-
140
- products = js("""
141
- JSON.stringify(
142
- Array.from(document.querySelectorAll('[data-test^="product:"]')).map(el => {
143
- var slug = el.getAttribute('data-test').replace('product:', '');
144
- var link = el.querySelector('a[href^="/products/"]');
145
- return {
146
- slug: slug,
147
- href: link ? link.getAttribute('href') : null,
148
- text: el.outerText.trim().substring(0, 200)
149
- };
150
- })
151
- )
152
- """)
153
- ```
154
-
155
- **Sample output:**
156
- ```json
157
- {"slug":"figma","href":"/products/figma","text":"FigmaThe collaborative interface design tool4.9 (1.4K reviews)..."}
158
- ```
159
-
160
- Returns ~15 top-rated products in the topic, not recent launches.
161
-
162
- ---
163
-
164
- ## Category Page
165
-
166
- URL: `https://www.producthunt.com/categories/ai-agents`
167
-
168
- Same `[data-test^="product:"]` selector as topics. Returns 15 top-reviewed products in that category.
169
-
170
- ```json
171
- {"slug":"elevenlabs","href":"/products/elevenlabs","text":"ElevenLabs\nCreate natural AI voices instantly...\n4.9 (165 reviews)"}
172
- ```
173
-
174
- ---
175
-
176
- ## Product Detail Page
177
-
178
- URL: `https://www.producthunt.com/products/claude-opus-4-7`
179
-
180
- ### Get total vote count (sidebar button)
181
- ```python
182
- # Use [data-test="vote-button"] — different from [data-test="action-bar-vote-button"]
183
- vote_text = js("document.querySelector('[data-test=\"vote-button\"]').outerText.trim().replace(/\\s+/g, ' ')")
184
- # Returns: "Upvote • 466 points"
185
- # Parse votes: vote_text.split('•')[1].strip().replace(' points', '').replace(',', '')
186
- ```
187
-
188
- ### Get review count and rating
189
- ```python
190
- review_link = js("JSON.stringify(Array.from(document.querySelectorAll('a')).filter(a => a.href && a.href.includes('/reviews') && a.outerText.includes('review')).map(a => a.outerText.trim()).slice(0, 1))")
191
- # Returns: ["1 review"] or ["5.0\n(731 reviews)"]
192
- ```
193
-
194
- ### Get day rank (sidebar shows "#1 Day Rank")
195
- No dedicated `data-test` for rank — parse from sidebar context or use leaderboard position.
196
-
197
- ### Comments (action-bar-vote-button)
198
- Each comment has its own `[data-test="action-bar-vote-button"]` with text like `"Upvote (13)"`.
199
-
200
- ---
201
-
202
- ## Search Results
203
-
204
- URL: `https://www.producthunt.com/search?q=AI+agent`
205
-
206
- Selector: `[data-test^="spotlight-result-product-"]`
207
-
208
- ```python
209
- new_tab("https://www.producthunt.com/search?q=AI+agent")
210
- wait(3)
211
-
212
- results = js("""
213
- JSON.stringify(
214
- Array.from(document.querySelectorAll('[data-test^="spotlight-result-product-"]')).map(el => {
215
- var id = el.getAttribute('data-test').replace('spotlight-result-product-', '');
216
- var lines = el.outerText.trim().split('\\n').map(l => l.trim()).filter(l => l);
217
- return {
218
- id: id,
219
- name: lines[0] || null,
220
- tagline: lines[1] || null,
221
- review_text: lines[2] || null
222
- };
223
- })
224
- )
225
- """)
226
- ```
227
-
228
- **Note:** Search result elements are `<button>` elements (not `<a>` links), so there is no `href` in the DOM. Product URL must be constructed: `https://www.producthunt.com/products/<slug>` where slug must be derived by other means. The element's `data-test` ID matches the numeric product ID, not the slug.
229
-
230
- **Sample output:**
231
- ```json
232
- {"id":"526014","name":"/ai","tagline":"Access ChatGPT anywhere you type '/ai'","review_text":"2 reviews"}
233
- {"id":"991302","name":"Naoma AI Demo Agent","tagline":"The first video agent that runs conversational product demos","review_text":"5 reviews"}
234
- ```
235
-
236
- ---
237
-
238
- ## Key Selector Reference
239
-
240
- | Page | Selector | Count | Notes |
241
- |------|----------|-------|-------|
242
- | Homepage | `[data-test^="post-item-"]` | 30 | 4 sections × ~5–7 products |
243
- | Homepage | `[data-test^="post-name-"]` | 30 | Product name elements |
244
- | Homepage/Leaderboard | `[data-test="vote-button"]` | varies | `—` for hidden; numeric for visible |
245
- | Topics/Categories | `[data-test^="product:"]` | ~15 | Top-rated products |
246
- | Search | `[data-test^="spotlight-result-product-"]` | 10 | Button elements, no href |
247
- | Product detail | `[data-test="vote-button"]` | 1 | Main vote: "Upvote • N points" |
248
- | Product detail | `[data-test="action-bar-vote-button"]` | many | Comment upvotes: "Upvote (N)" |
249
-
250
- ---
251
-
252
- ## Common Pitfalls
253
-
254
- 1. **`innerText` returns `None` on complex elements** — use `outerText` or break into simple single-property expressions. Avoid chaining DOM traversal inside `JSON.stringify()` on large objects.
255
-
256
- 2. **`goto_url()` returns `ERR_ABORTED`** for producthunt.com in some browser sessions — always use `new_tab("url")` instead.
257
-
258
- 3. **`a[href^="/posts/"]` matches nothing** — Product Hunt uses `/products/` for product URLs, not `/posts/`.
259
-
260
- 4. **Today's votes are always `—`** during the first 4 hours of the day — use yesterday's leaderboard for confirmed vote counts.
261
-
262
- 5. **Homepage does not lazy-load more products on scroll** — 30 items is the fixed set. Use leaderboard pages for complete daily listings.
263
-
264
- 6. **JSON.stringify of DOM-heavy objects returns `None`** — serialize only primitives (strings, numbers) not live DOM node properties.
265
-
266
- 7. **Ranked product names contain rank prefix** — `"1. Claude Opus 4.7"` — strip with regex `re.sub(r'^\d+\.\s+', '', name)`.
267
-
268
- 8. **`wait(3)` required after `wait_for_load()`** — the React SPA continues rendering after the load event.
269
-
270
- ---
271
-
272
- ## Recommended Workflow for Scraping Today's Launches
273
-
274
- ```python
275
- # 1. Open Product Hunt in a new tab
276
- new_tab("https://www.producthunt.com/leaderboard/daily/2026/4/18")
277
- wait(4)
278
-
279
- # 2. Extract all products with metadata
280
- products = js("""
281
- JSON.stringify(
282
- Array.from(document.querySelectorAll('[data-test^="post-item-"]')).map(el => {
283
- var id = el.getAttribute('data-test').replace('post-item-', '');
284
- var nameEl = el.querySelector('[data-test^="post-name-"]');
285
- var productLink = el.querySelector('a[href^="/products/"]');
286
- var voteBtn = el.querySelector('[data-test="vote-button"]');
287
- var topicLinks = Array.from(el.querySelectorAll('a[href^="/topics/"]')).map(a => a.textContent.trim());
288
- var name = nameEl ? nameEl.textContent.trim() : '';
289
- var lines = el.innerText.split('\\n').map(l => l.trim()).filter(l => l);
290
- var tagline = lines.find(l => l !== name && !topicLinks.includes(l) && l !== '•' && !/^[0-9—]/.test(l) && l.length > 5);
291
- return {
292
- id: id,
293
- name: name,
294
- slug: productLink ? productLink.getAttribute('href') : null,
295
- votes: voteBtn ? voteBtn.textContent.trim() : null,
296
- topics: topicLinks,
297
- tagline: tagline || null
298
- };
299
- })
300
- )
301
- """)
302
- import json
303
- data = json.loads(products)
304
- print(f"Found {len(data)} products")
305
- for p in data:
306
- print(f" {p['name']} — {p['votes']} votes — {p['tagline']}")
307
- ```
1
+ # Product Hunt Scraping Skills
2
+
3
+ Field-tested against https://www.producthunt.com on 2026-04-18.
4
+ All selectors verified with actual browser runs.
5
+
6
+ ---
7
+
8
+ ## Page Structure Overview
9
+
10
+ Product Hunt is a React SPA. Key structural facts discovered:
11
+
12
+ - **No login wall** — all product data is accessible without signing in
13
+ - **No cookie banner** — page loads cleanly with no consent dialogs
14
+ - **Product URLs use `/products/` not `/posts/`** — the `a[href^="/posts/"]` selector matches nothing
15
+ - **`data-test` attributes** are the most reliable selectors throughout the site
16
+ - **4 homepage sections**: today, yesterday, last week, last month (5 products each, plus "see all")
17
+ - **Today's votes are hidden** for the first 4 hours of each day (`—` instead of count)
18
+ - **Homepage has 30 fixed post-items** — scrolling does NOT load more
19
+ - **`goto_url()` may return `ERR_ABORTED`** for producthunt.com in some browser sessions — use `new_tab()` instead
20
+
21
+ ---
22
+
23
+ ## Navigation Pattern
24
+
25
+ ```python
26
+ # goto_url() may fail on Product Hunt — use new_tab() reliably
27
+ tid = new_tab("https://www.producthunt.com")
28
+ wait(4) # React SPA needs time; wait_for_load() alone is insufficient
29
+ page = page_info()
30
+ # Verify: url should be 'https://www.producthunt.com/'
31
+ ```
32
+
33
+ ---
34
+
35
+ ## Homepage — Extract Daily Product Feed
36
+
37
+ The homepage shows today's launches plus rolling sections for yesterday, last week, last month.
38
+
39
+ ### Working selector: `[data-test^="post-item-"]`
40
+
41
+ ```python
42
+ # Full extraction with name, tagline, slug, votes, topics
43
+ products = js("""
44
+ JSON.stringify(
45
+ Array.from(document.querySelectorAll('[data-test^="post-item-"]')).map(el => {
46
+ var id = el.getAttribute('data-test').replace('post-item-', '');
47
+ var nameEl = el.querySelector('[data-test^="post-name-"]');
48
+ var productLink = el.querySelector('a[href^="/products/"]');
49
+ var voteBtn = el.querySelector('[data-test="vote-button"]');
50
+ var voteCount = voteBtn ? voteBtn.textContent.trim() : null;
51
+ var topicLinks = Array.from(el.querySelectorAll('a[href^="/topics/"]')).map(a => a.textContent.trim());
52
+ var name = nameEl ? nameEl.textContent.trim() : '';
53
+ var lines = el.innerText.split('\\n').map(l => l.trim()).filter(l => l);
54
+ var tagline = lines.find(l => l !== name && !topicLinks.includes(l) && l !== '•' && !/^[0-9—]/.test(l) && l.length > 5);
55
+ return {
56
+ id: id,
57
+ name: name,
58
+ slug: productLink ? productLink.getAttribute('href') : null,
59
+ votes: voteCount,
60
+ topics: topicLinks,
61
+ tagline: tagline || null
62
+ };
63
+ })
64
+ )
65
+ """)
66
+ ```
67
+
68
+ **Sample output:**
69
+ ```json
70
+ [
71
+ {"id":"1126372","name":"Vercel Flags","slug":"/products/vercel","votes":"—","topics":["Software Engineering","Developer Tools"],"tagline":"Feature flags, targeting rules, rollouts. All from Vercel."},
72
+ {"id":"1125388","name":"1. Claude Opus 4.7","slug":"/products/claude-opus-4-7","votes":"466","topics":["API","Artificial Intelligence","Development"],"tagline":"Claude's most capable model for reasoning and agentic coding"}
73
+ ]
74
+ ```
75
+
76
+ **Votes:**
77
+ - `"—"` = vote count hidden (today's products during first 4 hours)
78
+ - `"466"` = numeric string (yesterday/older products)
79
+ - `"2,152"` = comma-formatted for large counts (parse: `voteCount.replace(',', '')`)
80
+
81
+ **Name prefix**: Ranked products show rank in name: `"1. Claude Opus 4.7"` — strip with `re.sub(r'^\d+\. ', '', name)`.
82
+
83
+ ---
84
+
85
+ ## Daily Leaderboard — Best URL for Complete Daily Lists
86
+
87
+ The leaderboard shows all products for any given day with actual vote counts.
88
+
89
+ ```
90
+ URL: https://www.producthunt.com/leaderboard/daily/YYYY/M/D
91
+ Example: https://www.producthunt.com/leaderboard/daily/2026/4/18
92
+ ```
93
+
94
+ - Uses zero-padded-free month/day (April = `4`, not `04`)
95
+ - Uses same `[data-test^="post-item-"]` selector
96
+ - Shows 12–19 products per day
97
+ - Same extraction JS as homepage works identically
98
+
99
+ **Yesterday's results with real vote counts:**
100
+ ```json
101
+ {"id":"1125388","name":"1. Claude Opus 4.7","slug":"/products/claude-opus-4-7","votes":"466","tagline":"Claude's most capable model for reasoning and agentic coding"}
102
+ {"id":"1118208","name":"2. Build Check","slug":"/products/build-check-for-outsiders","votes":"396"}
103
+ ```
104
+
105
+ ---
106
+
107
+ ## Weekly Leaderboard
108
+
109
+ ```
110
+ URL: https://www.producthunt.com/leaderboard/weekly/YYYY/WW
111
+ Example: https://www.producthunt.com/leaderboard/weekly/2026/16
112
+ ```
113
+
114
+ - Week number is ISO week (week 16 = April 13–19, 2026)
115
+ - Current week may return 0 items until the week ends
116
+ - Same `[data-test^="post-item-"]` selector
117
+
118
+ ---
119
+
120
+ ## Monthly Leaderboard
121
+
122
+ ```
123
+ URL: https://www.producthunt.com/leaderboard/monthly/YYYY/M
124
+ Example: https://www.producthunt.com/leaderboard/monthly/2026/4
125
+ ```
126
+
127
+ ---
128
+
129
+ ## Topic Page
130
+
131
+ URL: `https://www.producthunt.com/topics/developer-tools`
132
+
133
+ Selector changes on topic pages — uses `[data-test^="product:"]` not `post-item-`.
134
+
135
+ ```python
136
+ # Navigate to topic
137
+ new_tab("https://www.producthunt.com/topics/developer-tools")
138
+ wait(3)
139
+
140
+ products = js("""
141
+ JSON.stringify(
142
+ Array.from(document.querySelectorAll('[data-test^="product:"]')).map(el => {
143
+ var slug = el.getAttribute('data-test').replace('product:', '');
144
+ var link = el.querySelector('a[href^="/products/"]');
145
+ return {
146
+ slug: slug,
147
+ href: link ? link.getAttribute('href') : null,
148
+ text: el.outerText.trim().substring(0, 200)
149
+ };
150
+ })
151
+ )
152
+ """)
153
+ ```
154
+
155
+ **Sample output:**
156
+ ```json
157
+ {"slug":"figma","href":"/products/figma","text":"FigmaThe collaborative interface design tool4.9 (1.4K reviews)..."}
158
+ ```
159
+
160
+ Returns ~15 top-rated products in the topic, not recent launches.
161
+
162
+ ---
163
+
164
+ ## Category Page
165
+
166
+ URL: `https://www.producthunt.com/categories/ai-agents`
167
+
168
+ Same `[data-test^="product:"]` selector as topics. Returns 15 top-reviewed products in that category.
169
+
170
+ ```json
171
+ {"slug":"elevenlabs","href":"/products/elevenlabs","text":"ElevenLabs\nCreate natural AI voices instantly...\n4.9 (165 reviews)"}
172
+ ```
173
+
174
+ ---
175
+
176
+ ## Product Detail Page
177
+
178
+ URL: `https://www.producthunt.com/products/claude-opus-4-7`
179
+
180
+ ### Get total vote count (sidebar button)
181
+ ```python
182
+ # Use [data-test="vote-button"] — different from [data-test="action-bar-vote-button"]
183
+ vote_text = js("document.querySelector('[data-test=\"vote-button\"]').outerText.trim().replace(/\\s+/g, ' ')")
184
+ # Returns: "Upvote • 466 points"
185
+ # Parse votes: vote_text.split('•')[1].strip().replace(' points', '').replace(',', '')
186
+ ```
187
+
188
+ ### Get review count and rating
189
+ ```python
190
+ review_link = js("JSON.stringify(Array.from(document.querySelectorAll('a')).filter(a => a.href && a.href.includes('/reviews') && a.outerText.includes('review')).map(a => a.outerText.trim()).slice(0, 1))")
191
+ # Returns: ["1 review"] or ["5.0\n(731 reviews)"]
192
+ ```
193
+
194
+ ### Get day rank (sidebar shows "#1 Day Rank")
195
+ No dedicated `data-test` for rank — parse from sidebar context or use leaderboard position.
196
+
197
+ ### Comments (action-bar-vote-button)
198
+ Each comment has its own `[data-test="action-bar-vote-button"]` with text like `"Upvote (13)"`.
199
+
200
+ ---
201
+
202
+ ## Search Results
203
+
204
+ URL: `https://www.producthunt.com/search?q=AI+agent`
205
+
206
+ Selector: `[data-test^="spotlight-result-product-"]`
207
+
208
+ ```python
209
+ new_tab("https://www.producthunt.com/search?q=AI+agent")
210
+ wait(3)
211
+
212
+ results = js("""
213
+ JSON.stringify(
214
+ Array.from(document.querySelectorAll('[data-test^="spotlight-result-product-"]')).map(el => {
215
+ var id = el.getAttribute('data-test').replace('spotlight-result-product-', '');
216
+ var lines = el.outerText.trim().split('\\n').map(l => l.trim()).filter(l => l);
217
+ return {
218
+ id: id,
219
+ name: lines[0] || null,
220
+ tagline: lines[1] || null,
221
+ review_text: lines[2] || null
222
+ };
223
+ })
224
+ )
225
+ """)
226
+ ```
227
+
228
+ **Note:** Search result elements are `<button>` elements (not `<a>` links), so there is no `href` in the DOM. Product URL must be constructed: `https://www.producthunt.com/products/<slug>` where slug must be derived by other means. The element's `data-test` ID matches the numeric product ID, not the slug.
229
+
230
+ **Sample output:**
231
+ ```json
232
+ {"id":"526014","name":"/ai","tagline":"Access ChatGPT anywhere you type '/ai'","review_text":"2 reviews"}
233
+ {"id":"991302","name":"Naoma AI Demo Agent","tagline":"The first video agent that runs conversational product demos","review_text":"5 reviews"}
234
+ ```
235
+
236
+ ---
237
+
238
+ ## Key Selector Reference
239
+
240
+ | Page | Selector | Count | Notes |
241
+ |------|----------|-------|-------|
242
+ | Homepage | `[data-test^="post-item-"]` | 30 | 4 sections × ~5–7 products |
243
+ | Homepage | `[data-test^="post-name-"]` | 30 | Product name elements |
244
+ | Homepage/Leaderboard | `[data-test="vote-button"]` | varies | `—` for hidden; numeric for visible |
245
+ | Topics/Categories | `[data-test^="product:"]` | ~15 | Top-rated products |
246
+ | Search | `[data-test^="spotlight-result-product-"]` | 10 | Button elements, no href |
247
+ | Product detail | `[data-test="vote-button"]` | 1 | Main vote: "Upvote • N points" |
248
+ | Product detail | `[data-test="action-bar-vote-button"]` | many | Comment upvotes: "Upvote (N)" |
249
+
250
+ ---
251
+
252
+ ## Common Pitfalls
253
+
254
+ 1. **`innerText` returns `None` on complex elements** — use `outerText` or break into simple single-property expressions. Avoid chaining DOM traversal inside `JSON.stringify()` on large objects.
255
+
256
+ 2. **`goto_url()` returns `ERR_ABORTED`** for producthunt.com in some browser sessions — always use `new_tab("url")` instead.
257
+
258
+ 3. **`a[href^="/posts/"]` matches nothing** — Product Hunt uses `/products/` for product URLs, not `/posts/`.
259
+
260
+ 4. **Today's votes are always `—`** during the first 4 hours of the day — use yesterday's leaderboard for confirmed vote counts.
261
+
262
+ 5. **Homepage does not lazy-load more products on scroll** — 30 items is the fixed set. Use leaderboard pages for complete daily listings.
263
+
264
+ 6. **JSON.stringify of DOM-heavy objects returns `None`** — serialize only primitives (strings, numbers) not live DOM node properties.
265
+
266
+ 7. **Ranked product names contain rank prefix** — `"1. Claude Opus 4.7"` — strip with regex `re.sub(r'^\d+\.\s+', '', name)`.
267
+
268
+ 8. **`wait(3)` required after `wait_for_load()`** — the React SPA continues rendering after the load event.
269
+
270
+ ---
271
+
272
+ ## Recommended Workflow for Scraping Today's Launches
273
+
274
+ ```python
275
+ # 1. Open Product Hunt in a new tab
276
+ new_tab("https://www.producthunt.com/leaderboard/daily/2026/4/18")
277
+ wait(4)
278
+
279
+ # 2. Extract all products with metadata
280
+ products = js("""
281
+ JSON.stringify(
282
+ Array.from(document.querySelectorAll('[data-test^="post-item-"]')).map(el => {
283
+ var id = el.getAttribute('data-test').replace('post-item-', '');
284
+ var nameEl = el.querySelector('[data-test^="post-name-"]');
285
+ var productLink = el.querySelector('a[href^="/products/"]');
286
+ var voteBtn = el.querySelector('[data-test="vote-button"]');
287
+ var topicLinks = Array.from(el.querySelectorAll('a[href^="/topics/"]')).map(a => a.textContent.trim());
288
+ var name = nameEl ? nameEl.textContent.trim() : '';
289
+ var lines = el.innerText.split('\\n').map(l => l.trim()).filter(l => l);
290
+ var tagline = lines.find(l => l !== name && !topicLinks.includes(l) && l !== '•' && !/^[0-9—]/.test(l) && l.length > 5);
291
+ return {
292
+ id: id,
293
+ name: name,
294
+ slug: productLink ? productLink.getAttribute('href') : null,
295
+ votes: voteBtn ? voteBtn.textContent.trim() : null,
296
+ topics: topicLinks,
297
+ tagline: tagline || null
298
+ };
299
+ })
300
+ )
301
+ """)
302
+ import json
303
+ data = json.loads(products)
304
+ print(f"Found {len(data)} products")
305
+ for p in data:
306
+ print(f" {p['name']} — {p['votes']} votes — {p['tagline']}")
307
+ ```