@pencil-agent/nano-pencil 2.0.0-beta.8 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (241) hide show
  1. package/README.md +267 -267
  2. package/dist/build-meta.json +3 -3
  3. package/dist/core/export-html/AGENT.md +11 -11
  4. package/dist/core/export-html/template.css +971 -971
  5. package/dist/core/export-html/template.html +54 -54
  6. package/dist/core/extensions-host/index.d.ts +1 -1
  7. package/dist/core/extensions-host/loader.js +1 -1
  8. package/dist/core/extensions-host/runner.d.ts +1 -0
  9. package/dist/core/extensions-host/runner.js +2 -2
  10. package/dist/core/extensions-host/types.d.ts +17 -22
  11. package/dist/core/lib/ai/src/types.d.ts +12 -2
  12. package/dist/core/persona/persona-manager.js +5 -2
  13. package/dist/core/runtime/agent-session.js +3 -3
  14. package/dist/core/runtime/extension-core-bindings.d.ts +1 -0
  15. package/dist/core/runtime/extension-core-bindings.js +2 -2
  16. package/dist/extensions/builtin/AGENT.md +115 -115
  17. package/dist/extensions/builtin/browser/AGENT.md +17 -17
  18. package/dist/extensions/builtin/browser/agent-workspace/agent_helpers.py +12 -12
  19. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/amazon/product-search.md +198 -198
  20. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/archive-org/scraping.md +341 -341
  21. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/arxiv/scraping.md +311 -311
  22. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/arxiv-bulk/scraping.md +333 -333
  23. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/atlas/overview.md +70 -70
  24. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/booking-com/scraping.md +578 -578
  25. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/capterra/scraping.md +440 -440
  26. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/centilebrain/generate-estimates.md +110 -110
  27. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/coingecko/scraping.md +325 -325
  28. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/coinmarketcap/scraping.md +463 -463
  29. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/coursera/scraping.md +360 -360
  30. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/craigslist/scraping.md +390 -390
  31. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/crossref/scraping.md +568 -568
  32. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/dev-to/scraping.md +323 -323
  33. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/duckduckgo/scraping.md +349 -349
  34. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/ebay/scraping.md +435 -435
  35. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/etsy/scraping.md +506 -506
  36. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/eventbrite/scraping.md +363 -363
  37. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/expedia/automation.md +168 -168
  38. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/facebook/groups.md +236 -236
  39. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/facebook/pages.md +295 -295
  40. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/framer/editor.md +108 -108
  41. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/fred/scraping.md +493 -493
  42. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/g2/scraping.md +580 -580
  43. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/genius/scraping.md +511 -511
  44. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/github/repo-actions.md +65 -65
  45. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/github/scraping.md +184 -184
  46. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/glassdoor/scraping.md +543 -543
  47. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/gmail/compose.md +122 -122
  48. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/goodreads/scraping.md +461 -461
  49. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/gutenberg/scraping.md +383 -383
  50. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/hackernews/scraping.md +243 -243
  51. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/howlongtobeat/scraping.md +473 -473
  52. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/imdb/scraping.md +271 -271
  53. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/itch-io/scraping.md +436 -436
  54. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/job-boards/indeed-glassdoor.md +1021 -1021
  55. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/letterboxd/scraping.md +349 -349
  56. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/linkedin/invitation-manager.md +109 -109
  57. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/loom/folder-enumeration.md +170 -170
  58. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/macrotrends/scraping.md +537 -537
  59. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/medium/article-hydration.md +120 -120
  60. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/medium/scraping.md +414 -414
  61. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/metacritic/scraping.md +477 -477
  62. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/musicbrainz/scraping.md +478 -478
  63. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/nasa/scraping.md +339 -339
  64. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/news-aggregation/multi-source.md +205 -205
  65. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/open-library/scraping.md +472 -472
  66. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/openalex/scraping.md +470 -470
  67. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/openstreetmap/scraping.md +490 -490
  68. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/package-registries/npm-pypi.md +478 -478
  69. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/polymarket/scraping.md +234 -234
  70. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/producthunt/scraping.md +307 -307
  71. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/pubmed/scraping.md +421 -421
  72. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/quora/scraping.md +364 -364
  73. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/rawg/scraping.md +352 -352
  74. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/reddit/scraping.md +124 -124
  75. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/rest-countries/scraping.md +233 -233
  76. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/sec-edgar/scraping.md +361 -361
  77. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/shopify-admin/README.md +36 -36
  78. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/shopify-admin/embedded-apps.md +72 -72
  79. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/shopify-admin/knowledge-base.md +109 -109
  80. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/shopify-admin/polaris-inputs.md +137 -137
  81. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/soundcloud/scraping.md +362 -362
  82. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/spotify/scraping.md +339 -339
  83. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/stackoverflow/scraping.md +435 -435
  84. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/steam/scraping.md +575 -575
  85. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/substack/scraping.md +338 -338
  86. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/thetechgeeks/pricing.md +52 -52
  87. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/tiktok/upload.md +107 -107
  88. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/tradingview/scraping.md +309 -309
  89. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/trello/boards-and-lists.md +88 -88
  90. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/trustpilot/scraping.md +375 -375
  91. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/walmart/scraping.md +444 -444
  92. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/wayback-machine/scraping.md +306 -306
  93. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/weather/scraping.md +398 -398
  94. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/wellfound/scraping.md +596 -596
  95. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/world-bank/scraping.md +356 -356
  96. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/xiaohongshu/scraping.md +84 -84
  97. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/youtube/scraping.md +418 -418
  98. package/dist/extensions/builtin/browser/agent-workspace/domain-skills/zillow/scraping.md +433 -433
  99. package/dist/extensions/builtin/browser/browser.md +73 -73
  100. package/dist/extensions/builtin/browser/install.md +142 -142
  101. package/dist/extensions/builtin/browser/interaction-skills/connection.md +48 -48
  102. package/dist/extensions/builtin/browser/interaction-skills/cookies.md +3 -3
  103. package/dist/extensions/builtin/browser/interaction-skills/cross-origin-iframes.md +3 -3
  104. package/dist/extensions/builtin/browser/interaction-skills/dialogs.md +64 -64
  105. package/dist/extensions/builtin/browser/interaction-skills/downloads.md +3 -3
  106. package/dist/extensions/builtin/browser/interaction-skills/drag-and-drop.md +3 -3
  107. package/dist/extensions/builtin/browser/interaction-skills/dropdowns.md +3 -3
  108. package/dist/extensions/builtin/browser/interaction-skills/iframes.md +3 -3
  109. package/dist/extensions/builtin/browser/interaction-skills/network-requests.md +3 -3
  110. package/dist/extensions/builtin/browser/interaction-skills/print-as-pdf.md +3 -3
  111. package/dist/extensions/builtin/browser/interaction-skills/profile-sync.md +90 -90
  112. package/dist/extensions/builtin/browser/interaction-skills/screenshots.md +17 -17
  113. package/dist/extensions/builtin/browser/interaction-skills/scrolling.md +3 -3
  114. package/dist/extensions/builtin/browser/interaction-skills/shadow-dom.md +3 -3
  115. package/dist/extensions/builtin/browser/interaction-skills/tabs.md +69 -69
  116. package/dist/extensions/builtin/browser/interaction-skills/uploads.md +1 -1
  117. package/dist/extensions/builtin/browser/interaction-skills/viewport.md +3 -3
  118. package/dist/extensions/builtin/browser/src/browser_harness/AGENT.md +15 -15
  119. package/dist/extensions/builtin/browser/src/browser_harness/__init__.py +8 -8
  120. package/dist/extensions/builtin/browser/src/browser_harness/_ipc.py +90 -90
  121. package/dist/extensions/builtin/browser/src/browser_harness/admin.py +722 -722
  122. package/dist/extensions/builtin/browser/src/browser_harness/daemon.py +328 -328
  123. package/dist/extensions/builtin/browser/src/browser_harness/helpers.py +396 -396
  124. package/dist/extensions/builtin/browser/src/browser_harness/run.py +103 -103
  125. package/dist/extensions/builtin/discipline/skills/brainstorming/SKILL.md +33 -33
  126. package/dist/extensions/builtin/discipline/skills/executing-plans/SKILL.md +25 -25
  127. package/dist/extensions/builtin/discipline/skills/finishing-development-branch/SKILL.md +25 -25
  128. package/dist/extensions/builtin/discipline/skills/receiving-code-review/SKILL.md +22 -22
  129. package/dist/extensions/builtin/discipline/skills/requesting-code-review/SKILL.md +31 -31
  130. package/dist/extensions/builtin/discipline/skills/systematic-debugging/SKILL.md +28 -28
  131. package/dist/extensions/builtin/discipline/skills/test-driven-development/SKILL.md +32 -32
  132. package/dist/extensions/builtin/discipline/skills/using-git-worktrees/SKILL.md +25 -25
  133. package/dist/extensions/builtin/discipline/skills/verification-before-completion/SKILL.md +27 -27
  134. package/dist/extensions/builtin/discipline/skills/writing-plans/SKILL.md +26 -26
  135. package/dist/extensions/builtin/goal/README.md +67 -67
  136. package/dist/extensions/builtin/goal/goal-controller.d.ts +39 -10
  137. package/dist/extensions/builtin/goal/goal-controller.js +1 -1
  138. package/dist/extensions/builtin/goal/goal-format.js +1 -1
  139. package/dist/extensions/builtin/goal/goal-prompts.d.ts +2 -0
  140. package/dist/extensions/builtin/goal/goal-prompts.js +5 -4
  141. package/dist/extensions/builtin/goal/goal-store.js +1 -1
  142. package/dist/extensions/builtin/goal/index.d.ts +1 -1
  143. package/dist/extensions/builtin/goal/index.js +10 -7
  144. package/dist/extensions/builtin/grub/README.md +112 -112
  145. package/dist/extensions/builtin/link-world/agent-workspace/README.md +16 -16
  146. package/dist/extensions/builtin/link-world/index.js +6 -6
  147. package/dist/extensions/builtin/link-world/internet-search/internet-search.md +65 -65
  148. package/dist/extensions/builtin/link-world/link-world-agent.md +82 -82
  149. package/dist/extensions/builtin/link-world/linkworld.md +313 -313
  150. package/dist/extensions/builtin/link-world/{network-routing.md → network-routing/network-routing.md} +67 -67
  151. package/dist/extensions/builtin/loop/README.md +92 -92
  152. package/dist/extensions/builtin/mcp/figma-design.md +68 -68
  153. package/dist/extensions/builtin/mcp/mcp-management.md +85 -85
  154. package/dist/extensions/builtin/plan/index.js +1 -1
  155. package/dist/extensions/builtin/recap/AGENT.md +15 -15
  156. package/dist/extensions/builtin/sal/README.md +72 -72
  157. package/dist/extensions/builtin/security-audit/README.md +289 -289
  158. package/dist/extensions/builtin/task/task-store.d.ts +4 -0
  159. package/dist/extensions/builtin/task/task-store.js +1 -1
  160. package/dist/extensions/builtin/team/AGENT.md +112 -112
  161. package/dist/extensions/builtin/team/TESTING.md +299 -299
  162. package/dist/extensions/builtin/token-save/README.md +56 -56
  163. package/dist/extensions/optional/AGENT.md +10 -10
  164. package/dist/index.d.ts +5 -30
  165. package/dist/index.js +1 -1
  166. package/dist/models.d.ts +7 -0
  167. package/dist/models.js +1 -0
  168. package/dist/modes/interactive/components/footer.js +1 -1
  169. package/dist/modes/interactive/components/task-status-panel.d.ts +36 -0
  170. package/dist/modes/interactive/components/task-status-panel.js +1 -0
  171. package/dist/modes/interactive/controllers/stream-render-controller.d.ts +7 -0
  172. package/dist/modes/interactive/controllers/stream-render-controller.js +2 -2
  173. package/dist/modes/interactive/interactive-mode.js +40 -40
  174. package/dist/modes/interactive/state/interactive-state.d.ts +2 -0
  175. package/dist/modes/interactive/state/interactive-state.js +1 -1
  176. package/dist/modes/interactive/theme/dark.json +85 -85
  177. package/dist/modes/interactive/theme/light.json +84 -84
  178. package/dist/modes/interactive/theme/theme-schema.json +335 -335
  179. package/dist/modes/interactive/theme/warm.json +81 -81
  180. package/dist/node_modules/@pencil-agent/ai/dist/cli.js +0 -0
  181. package/dist/node_modules/@pencil-agent/ai/dist/models.generated.js +1 -1
  182. package/dist/node_modules/@pencil-agent/ai/dist/providers/anthropic.js +2 -2
  183. package/dist/node_modules/@pencil-agent/ai/dist/providers/openai-completions.js +5 -5
  184. package/dist/node_modules/@pencil-agent/ai/dist/providers/openai-responses.js +1 -1
  185. package/dist/node_modules/@pencil-agent/ai/dist/stream.js +1 -1
  186. package/dist/packages/protocol/src/commands.d.ts +33 -0
  187. package/dist/packages/protocol/src/flags.d.ts +20 -0
  188. package/dist/packages/protocol/src/hooks.d.ts +17 -0
  189. package/dist/packages/protocol/src/hooks.js +0 -0
  190. package/dist/packages/{extension-sdk → protocol}/src/index.d.ts +7 -4
  191. package/dist/packages/protocol/src/index.js +1 -0
  192. package/dist/packages/{extension-sdk → protocol}/src/lifecycle.d.ts +15 -27
  193. package/dist/packages/protocol/src/lifecycle.js +0 -0
  194. package/dist/packages/{extension-sdk → protocol}/src/tools.d.ts +1 -1
  195. package/dist/packages/protocol/src/tools.js +0 -0
  196. package/dist/public-config.d.ts +12 -0
  197. package/dist/public-config.js +1 -0
  198. package/dist/runtime.d.ts +9 -0
  199. package/dist/runtime.js +1 -0
  200. package/dist/session-compaction.d.ts +7 -0
  201. package/dist/session-compaction.js +1 -0
  202. package/dist/session.d.ts +7 -0
  203. package/dist/session.js +1 -0
  204. package/dist/skills.d.ts +7 -0
  205. package/dist/skills.js +1 -0
  206. package/dist/tools.d.ts +7 -0
  207. package/dist/tools.js +1 -0
  208. package/docs/ACP/345/215/217/350/256/256/351/233/206/346/210/220/345/274/200/345/217/221/346/226/207/346/241/243.md +851 -0
  209. package/docs/SDK-TESTING.md +364 -0
  210. package/docs/codex-goal-command-impl.md +1055 -1055
  211. package/docs/codex-goal-vs-grub.md +500 -500
  212. package/docs/custom-provider.md +27 -27
  213. package/docs/extensions.md +27 -27
  214. package/docs/keybindings.md +27 -27
  215. package/docs/loop /351/207/215/346/236/204/345/256/214/346/210/220/346/200/273/347/273/223.md" +250 -250
  216. package/docs/loop /351/207/215/346/236/204/345/256/214/346/210/220/346/212/245/345/221/212.md" +122 -122
  217. package/docs/loop /351/207/215/346/236/204/346/226/271/346/241/210.md" +1222 -1222
  218. package/docs/loop /351/207/215/346/236/204/346/226/271/346/241/210/345/256/236/347/216/260/346/212/245/345/221/212.md" +158 -158
  219. package/docs/loop /351/207/215/346/236/204/346/226/271/346/241/210/345/257/271/346/257/224/345/210/206/346/236/220.md" +128 -128
  220. package/docs/loop /351/207/215/346/236/204/350/256/241/345/210/222.md" +320 -320
  221. package/docs/loop-usage-examples.md +214 -214
  222. package/docs/mem-core/346/212/200/346/234/257/346/226/207/346/241/243.md +593 -0
  223. package/docs/models.md +27 -27
  224. package/docs/packages.md +27 -27
  225. package/docs/pi-design-philosophy.md +457 -457
  226. package/docs/planmode.md +1987 -1987
  227. package/docs/prompt-templates.md +27 -27
  228. package/docs/providers.md +27 -27
  229. package/docs/sdk.md +27 -27
  230. package/docs/skills.md +27 -27
  231. package/docs/startup-performance-optimization.md +301 -0
  232. package/docs/themes.md +27 -27
  233. package/docs/tui.md +27 -27
  234. package/docs//350/256/244/347/237/245/345/234/260/345/233/276.md +47 -0
  235. package/package.json +190 -162
  236. package/dist/packages/extension-sdk/src/index.js +0 -1
  237. package/docs/cc-agent-design.md +0 -1297
  238. package/docs/cc-tui-design.md +0 -1333
  239. package/docs//345/257/271/346/240/207Claude-Code.md +0 -1775
  240. /package/dist/packages/{extension-sdk/src/lifecycle.js → protocol/src/commands.js} +0 -0
  241. /package/dist/packages/{extension-sdk/src/tools.js → protocol/src/flags.js} +0 -0
@@ -1,307 +1,307 @@
1
- # Product Hunt Scraping Skills
2
-
3
- Field-tested against https://www.producthunt.com on 2026-04-18.
4
- All selectors verified with actual browser runs.
5
-
6
- ---
7
-
8
- ## Page Structure Overview
9
-
10
- Product Hunt is a React SPA. Key structural facts discovered:
11
-
12
- - **No login wall** — all product data is accessible without signing in
13
- - **No cookie banner** — page loads cleanly with no consent dialogs
14
- - **Product URLs use `/products/` not `/posts/`** — the `a[href^="/posts/"]` selector matches nothing
15
- - **`data-test` attributes** are the most reliable selectors throughout the site
16
- - **4 homepage sections**: today, yesterday, last week, last month (5 products each, plus "see all")
17
- - **Today's votes are hidden** for the first 4 hours of each day (`—` instead of count)
18
- - **Homepage has 30 fixed post-items** — scrolling does NOT load more
19
- - **`goto_url()` may return `ERR_ABORTED`** for producthunt.com in some browser sessions — use `new_tab()` instead
20
-
21
- ---
22
-
23
- ## Navigation Pattern
24
-
25
- ```python
26
- # goto_url() may fail on Product Hunt — use new_tab() reliably
27
- tid = new_tab("https://www.producthunt.com")
28
- wait(4) # React SPA needs time; wait_for_load() alone is insufficient
29
- page = page_info()
30
- # Verify: url should be 'https://www.producthunt.com/'
31
- ```
32
-
33
- ---
34
-
35
- ## Homepage — Extract Daily Product Feed
36
-
37
- The homepage shows today's launches plus rolling sections for yesterday, last week, last month.
38
-
39
- ### Working selector: `[data-test^="post-item-"]`
40
-
41
- ```python
42
- # Full extraction with name, tagline, slug, votes, topics
43
- products = js("""
44
- JSON.stringify(
45
- Array.from(document.querySelectorAll('[data-test^="post-item-"]')).map(el => {
46
- var id = el.getAttribute('data-test').replace('post-item-', '');
47
- var nameEl = el.querySelector('[data-test^="post-name-"]');
48
- var productLink = el.querySelector('a[href^="/products/"]');
49
- var voteBtn = el.querySelector('[data-test="vote-button"]');
50
- var voteCount = voteBtn ? voteBtn.textContent.trim() : null;
51
- var topicLinks = Array.from(el.querySelectorAll('a[href^="/topics/"]')).map(a => a.textContent.trim());
52
- var name = nameEl ? nameEl.textContent.trim() : '';
53
- var lines = el.innerText.split('\\n').map(l => l.trim()).filter(l => l);
54
- var tagline = lines.find(l => l !== name && !topicLinks.includes(l) && l !== '•' && !/^[0-9—]/.test(l) && l.length > 5);
55
- return {
56
- id: id,
57
- name: name,
58
- slug: productLink ? productLink.getAttribute('href') : null,
59
- votes: voteCount,
60
- topics: topicLinks,
61
- tagline: tagline || null
62
- };
63
- })
64
- )
65
- """)
66
- ```
67
-
68
- **Sample output:**
69
- ```json
70
- [
71
- {"id":"1126372","name":"Vercel Flags","slug":"/products/vercel","votes":"—","topics":["Software Engineering","Developer Tools"],"tagline":"Feature flags, targeting rules, rollouts. All from Vercel."},
72
- {"id":"1125388","name":"1. Claude Opus 4.7","slug":"/products/claude-opus-4-7","votes":"466","topics":["API","Artificial Intelligence","Development"],"tagline":"Claude's most capable model for reasoning and agentic coding"}
73
- ]
74
- ```
75
-
76
- **Votes:**
77
- - `"—"` = vote count hidden (today's products during first 4 hours)
78
- - `"466"` = numeric string (yesterday/older products)
79
- - `"2,152"` = comma-formatted for large counts (parse: `voteCount.replace(',', '')`)
80
-
81
- **Name prefix**: Ranked products show rank in name: `"1. Claude Opus 4.7"` — strip with `re.sub(r'^\d+\. ', '', name)`.
82
-
83
- ---
84
-
85
- ## Daily Leaderboard — Best URL for Complete Daily Lists
86
-
87
- The leaderboard shows all products for any given day with actual vote counts.
88
-
89
- ```
90
- URL: https://www.producthunt.com/leaderboard/daily/YYYY/M/D
91
- Example: https://www.producthunt.com/leaderboard/daily/2026/4/18
92
- ```
93
-
94
- - Uses zero-padded-free month/day (April = `4`, not `04`)
95
- - Uses same `[data-test^="post-item-"]` selector
96
- - Shows 12–19 products per day
97
- - Same extraction JS as homepage works identically
98
-
99
- **Yesterday's results with real vote counts:**
100
- ```json
101
- {"id":"1125388","name":"1. Claude Opus 4.7","slug":"/products/claude-opus-4-7","votes":"466","tagline":"Claude's most capable model for reasoning and agentic coding"}
102
- {"id":"1118208","name":"2. Build Check","slug":"/products/build-check-for-outsiders","votes":"396"}
103
- ```
104
-
105
- ---
106
-
107
- ## Weekly Leaderboard
108
-
109
- ```
110
- URL: https://www.producthunt.com/leaderboard/weekly/YYYY/WW
111
- Example: https://www.producthunt.com/leaderboard/weekly/2026/16
112
- ```
113
-
114
- - Week number is ISO week (week 16 = April 13–19, 2026)
115
- - Current week may return 0 items until the week ends
116
- - Same `[data-test^="post-item-"]` selector
117
-
118
- ---
119
-
120
- ## Monthly Leaderboard
121
-
122
- ```
123
- URL: https://www.producthunt.com/leaderboard/monthly/YYYY/M
124
- Example: https://www.producthunt.com/leaderboard/monthly/2026/4
125
- ```
126
-
127
- ---
128
-
129
- ## Topic Page
130
-
131
- URL: `https://www.producthunt.com/topics/developer-tools`
132
-
133
- Selector changes on topic pages — uses `[data-test^="product:"]` not `post-item-`.
134
-
135
- ```python
136
- # Navigate to topic
137
- new_tab("https://www.producthunt.com/topics/developer-tools")
138
- wait(3)
139
-
140
- products = js("""
141
- JSON.stringify(
142
- Array.from(document.querySelectorAll('[data-test^="product:"]')).map(el => {
143
- var slug = el.getAttribute('data-test').replace('product:', '');
144
- var link = el.querySelector('a[href^="/products/"]');
145
- return {
146
- slug: slug,
147
- href: link ? link.getAttribute('href') : null,
148
- text: el.outerText.trim().substring(0, 200)
149
- };
150
- })
151
- )
152
- """)
153
- ```
154
-
155
- **Sample output:**
156
- ```json
157
- {"slug":"figma","href":"/products/figma","text":"FigmaThe collaborative interface design tool4.9 (1.4K reviews)..."}
158
- ```
159
-
160
- Returns ~15 top-rated products in the topic, not recent launches.
161
-
162
- ---
163
-
164
- ## Category Page
165
-
166
- URL: `https://www.producthunt.com/categories/ai-agents`
167
-
168
- Same `[data-test^="product:"]` selector as topics. Returns 15 top-reviewed products in that category.
169
-
170
- ```json
171
- {"slug":"elevenlabs","href":"/products/elevenlabs","text":"ElevenLabs\nCreate natural AI voices instantly...\n4.9 (165 reviews)"}
172
- ```
173
-
174
- ---
175
-
176
- ## Product Detail Page
177
-
178
- URL: `https://www.producthunt.com/products/claude-opus-4-7`
179
-
180
- ### Get total vote count (sidebar button)
181
- ```python
182
- # Use [data-test="vote-button"] — different from [data-test="action-bar-vote-button"]
183
- vote_text = js("document.querySelector('[data-test=\"vote-button\"]').outerText.trim().replace(/\\s+/g, ' ')")
184
- # Returns: "Upvote • 466 points"
185
- # Parse votes: vote_text.split('•')[1].strip().replace(' points', '').replace(',', '')
186
- ```
187
-
188
- ### Get review count and rating
189
- ```python
190
- review_link = js("JSON.stringify(Array.from(document.querySelectorAll('a')).filter(a => a.href && a.href.includes('/reviews') && a.outerText.includes('review')).map(a => a.outerText.trim()).slice(0, 1))")
191
- # Returns: ["1 review"] or ["5.0\n(731 reviews)"]
192
- ```
193
-
194
- ### Get day rank (sidebar shows "#1 Day Rank")
195
- No dedicated `data-test` for rank — parse from sidebar context or use leaderboard position.
196
-
197
- ### Comments (action-bar-vote-button)
198
- Each comment has its own `[data-test="action-bar-vote-button"]` with text like `"Upvote (13)"`.
199
-
200
- ---
201
-
202
- ## Search Results
203
-
204
- URL: `https://www.producthunt.com/search?q=AI+agent`
205
-
206
- Selector: `[data-test^="spotlight-result-product-"]`
207
-
208
- ```python
209
- new_tab("https://www.producthunt.com/search?q=AI+agent")
210
- wait(3)
211
-
212
- results = js("""
213
- JSON.stringify(
214
- Array.from(document.querySelectorAll('[data-test^="spotlight-result-product-"]')).map(el => {
215
- var id = el.getAttribute('data-test').replace('spotlight-result-product-', '');
216
- var lines = el.outerText.trim().split('\\n').map(l => l.trim()).filter(l => l);
217
- return {
218
- id: id,
219
- name: lines[0] || null,
220
- tagline: lines[1] || null,
221
- review_text: lines[2] || null
222
- };
223
- })
224
- )
225
- """)
226
- ```
227
-
228
- **Note:** Search result elements are `<button>` elements (not `<a>` links), so there is no `href` in the DOM. Product URL must be constructed: `https://www.producthunt.com/products/<slug>` where slug must be derived by other means. The element's `data-test` ID matches the numeric product ID, not the slug.
229
-
230
- **Sample output:**
231
- ```json
232
- {"id":"526014","name":"/ai","tagline":"Access ChatGPT anywhere you type '/ai'","review_text":"2 reviews"}
233
- {"id":"991302","name":"Naoma AI Demo Agent","tagline":"The first video agent that runs conversational product demos","review_text":"5 reviews"}
234
- ```
235
-
236
- ---
237
-
238
- ## Key Selector Reference
239
-
240
- | Page | Selector | Count | Notes |
241
- |------|----------|-------|-------|
242
- | Homepage | `[data-test^="post-item-"]` | 30 | 4 sections × ~5–7 products |
243
- | Homepage | `[data-test^="post-name-"]` | 30 | Product name elements |
244
- | Homepage/Leaderboard | `[data-test="vote-button"]` | varies | `—` for hidden; numeric for visible |
245
- | Topics/Categories | `[data-test^="product:"]` | ~15 | Top-rated products |
246
- | Search | `[data-test^="spotlight-result-product-"]` | 10 | Button elements, no href |
247
- | Product detail | `[data-test="vote-button"]` | 1 | Main vote: "Upvote • N points" |
248
- | Product detail | `[data-test="action-bar-vote-button"]` | many | Comment upvotes: "Upvote (N)" |
249
-
250
- ---
251
-
252
- ## Common Pitfalls
253
-
254
- 1. **`innerText` returns `None` on complex elements** — use `outerText` or break into simple single-property expressions. Avoid chaining DOM traversal inside `JSON.stringify()` on large objects.
255
-
256
- 2. **`goto_url()` returns `ERR_ABORTED`** for producthunt.com in some browser sessions — always use `new_tab("url")` instead.
257
-
258
- 3. **`a[href^="/posts/"]` matches nothing** — Product Hunt uses `/products/` for product URLs, not `/posts/`.
259
-
260
- 4. **Today's votes are always `—`** during the first 4 hours of the day — use yesterday's leaderboard for confirmed vote counts.
261
-
262
- 5. **Homepage does not lazy-load more products on scroll** — 30 items is the fixed set. Use leaderboard pages for complete daily listings.
263
-
264
- 6. **JSON.stringify of DOM-heavy objects returns `None`** — serialize only primitives (strings, numbers) not live DOM node properties.
265
-
266
- 7. **Ranked product names contain rank prefix** — `"1. Claude Opus 4.7"` — strip with regex `re.sub(r'^\d+\.\s+', '', name)`.
267
-
268
- 8. **`wait(3)` required after `wait_for_load()`** — the React SPA continues rendering after the load event.
269
-
270
- ---
271
-
272
- ## Recommended Workflow for Scraping Today's Launches
273
-
274
- ```python
275
- # 1. Open Product Hunt in a new tab
276
- new_tab("https://www.producthunt.com/leaderboard/daily/2026/4/18")
277
- wait(4)
278
-
279
- # 2. Extract all products with metadata
280
- products = js("""
281
- JSON.stringify(
282
- Array.from(document.querySelectorAll('[data-test^="post-item-"]')).map(el => {
283
- var id = el.getAttribute('data-test').replace('post-item-', '');
284
- var nameEl = el.querySelector('[data-test^="post-name-"]');
285
- var productLink = el.querySelector('a[href^="/products/"]');
286
- var voteBtn = el.querySelector('[data-test="vote-button"]');
287
- var topicLinks = Array.from(el.querySelectorAll('a[href^="/topics/"]')).map(a => a.textContent.trim());
288
- var name = nameEl ? nameEl.textContent.trim() : '';
289
- var lines = el.innerText.split('\\n').map(l => l.trim()).filter(l => l);
290
- var tagline = lines.find(l => l !== name && !topicLinks.includes(l) && l !== '•' && !/^[0-9—]/.test(l) && l.length > 5);
291
- return {
292
- id: id,
293
- name: name,
294
- slug: productLink ? productLink.getAttribute('href') : null,
295
- votes: voteBtn ? voteBtn.textContent.trim() : null,
296
- topics: topicLinks,
297
- tagline: tagline || null
298
- };
299
- })
300
- )
301
- """)
302
- import json
303
- data = json.loads(products)
304
- print(f"Found {len(data)} products")
305
- for p in data:
306
- print(f" {p['name']} — {p['votes']} votes — {p['tagline']}")
307
- ```
1
+ # Product Hunt Scraping Skills
2
+
3
+ Field-tested against https://www.producthunt.com on 2026-04-18.
4
+ All selectors verified with actual browser runs.
5
+
6
+ ---
7
+
8
+ ## Page Structure Overview
9
+
10
+ Product Hunt is a React SPA. Key structural facts discovered:
11
+
12
+ - **No login wall** — all product data is accessible without signing in
13
+ - **No cookie banner** — page loads cleanly with no consent dialogs
14
+ - **Product URLs use `/products/` not `/posts/`** — the `a[href^="/posts/"]` selector matches nothing
15
+ - **`data-test` attributes** are the most reliable selectors throughout the site
16
+ - **4 homepage sections**: today, yesterday, last week, last month (5 products each, plus "see all")
17
+ - **Today's votes are hidden** for the first 4 hours of each day (`—` instead of count)
18
+ - **Homepage has 30 fixed post-items** — scrolling does NOT load more
19
+ - **`goto_url()` may return `ERR_ABORTED`** for producthunt.com in some browser sessions — use `new_tab()` instead
20
+
21
+ ---
22
+
23
+ ## Navigation Pattern
24
+
25
+ ```python
26
+ # goto_url() may fail on Product Hunt — use new_tab() reliably
27
+ tid = new_tab("https://www.producthunt.com")
28
+ wait(4) # React SPA needs time; wait_for_load() alone is insufficient
29
+ page = page_info()
30
+ # Verify: url should be 'https://www.producthunt.com/'
31
+ ```
32
+
33
+ ---
34
+
35
+ ## Homepage — Extract Daily Product Feed
36
+
37
+ The homepage shows today's launches plus rolling sections for yesterday, last week, last month.
38
+
39
+ ### Working selector: `[data-test^="post-item-"]`
40
+
41
+ ```python
42
+ # Full extraction with name, tagline, slug, votes, topics
43
+ products = js("""
44
+ JSON.stringify(
45
+ Array.from(document.querySelectorAll('[data-test^="post-item-"]')).map(el => {
46
+ var id = el.getAttribute('data-test').replace('post-item-', '');
47
+ var nameEl = el.querySelector('[data-test^="post-name-"]');
48
+ var productLink = el.querySelector('a[href^="/products/"]');
49
+ var voteBtn = el.querySelector('[data-test="vote-button"]');
50
+ var voteCount = voteBtn ? voteBtn.textContent.trim() : null;
51
+ var topicLinks = Array.from(el.querySelectorAll('a[href^="/topics/"]')).map(a => a.textContent.trim());
52
+ var name = nameEl ? nameEl.textContent.trim() : '';
53
+ var lines = el.innerText.split('\\n').map(l => l.trim()).filter(l => l);
54
+ var tagline = lines.find(l => l !== name && !topicLinks.includes(l) && l !== '•' && !/^[0-9—]/.test(l) && l.length > 5);
55
+ return {
56
+ id: id,
57
+ name: name,
58
+ slug: productLink ? productLink.getAttribute('href') : null,
59
+ votes: voteCount,
60
+ topics: topicLinks,
61
+ tagline: tagline || null
62
+ };
63
+ })
64
+ )
65
+ """)
66
+ ```
67
+
68
+ **Sample output:**
69
+ ```json
70
+ [
71
+ {"id":"1126372","name":"Vercel Flags","slug":"/products/vercel","votes":"—","topics":["Software Engineering","Developer Tools"],"tagline":"Feature flags, targeting rules, rollouts. All from Vercel."},
72
+ {"id":"1125388","name":"1. Claude Opus 4.7","slug":"/products/claude-opus-4-7","votes":"466","topics":["API","Artificial Intelligence","Development"],"tagline":"Claude's most capable model for reasoning and agentic coding"}
73
+ ]
74
+ ```
75
+
76
+ **Votes:**
77
+ - `"—"` = vote count hidden (today's products during first 4 hours)
78
+ - `"466"` = numeric string (yesterday/older products)
79
+ - `"2,152"` = comma-formatted for large counts (parse: `voteCount.replace(',', '')`)
80
+
81
+ **Name prefix**: Ranked products show rank in name: `"1. Claude Opus 4.7"` — strip with `re.sub(r'^\d+\. ', '', name)`.
82
+
83
+ ---
84
+
85
+ ## Daily Leaderboard — Best URL for Complete Daily Lists
86
+
87
+ The leaderboard shows all products for any given day with actual vote counts.
88
+
89
+ ```
90
+ URL: https://www.producthunt.com/leaderboard/daily/YYYY/M/D
91
+ Example: https://www.producthunt.com/leaderboard/daily/2026/4/18
92
+ ```
93
+
94
+ - Uses zero-padded-free month/day (April = `4`, not `04`)
95
+ - Uses same `[data-test^="post-item-"]` selector
96
+ - Shows 12–19 products per day
97
+ - Same extraction JS as homepage works identically
98
+
99
+ **Yesterday's results with real vote counts:**
100
+ ```json
101
+ {"id":"1125388","name":"1. Claude Opus 4.7","slug":"/products/claude-opus-4-7","votes":"466","tagline":"Claude's most capable model for reasoning and agentic coding"}
102
+ {"id":"1118208","name":"2. Build Check","slug":"/products/build-check-for-outsiders","votes":"396"}
103
+ ```
104
+
105
+ ---
106
+
107
+ ## Weekly Leaderboard
108
+
109
+ ```
110
+ URL: https://www.producthunt.com/leaderboard/weekly/YYYY/WW
111
+ Example: https://www.producthunt.com/leaderboard/weekly/2026/16
112
+ ```
113
+
114
+ - Week number is ISO week (week 16 = April 13–19, 2026)
115
+ - Current week may return 0 items until the week ends
116
+ - Same `[data-test^="post-item-"]` selector
117
+
118
+ ---
119
+
120
+ ## Monthly Leaderboard
121
+
122
+ ```
123
+ URL: https://www.producthunt.com/leaderboard/monthly/YYYY/M
124
+ Example: https://www.producthunt.com/leaderboard/monthly/2026/4
125
+ ```
126
+
127
+ ---
128
+
129
+ ## Topic Page
130
+
131
+ URL: `https://www.producthunt.com/topics/developer-tools`
132
+
133
+ Selector changes on topic pages — uses `[data-test^="product:"]` not `post-item-`.
134
+
135
+ ```python
136
+ # Navigate to topic
137
+ new_tab("https://www.producthunt.com/topics/developer-tools")
138
+ wait(3)
139
+
140
+ products = js("""
141
+ JSON.stringify(
142
+ Array.from(document.querySelectorAll('[data-test^="product:"]')).map(el => {
143
+ var slug = el.getAttribute('data-test').replace('product:', '');
144
+ var link = el.querySelector('a[href^="/products/"]');
145
+ return {
146
+ slug: slug,
147
+ href: link ? link.getAttribute('href') : null,
148
+ text: el.outerText.trim().substring(0, 200)
149
+ };
150
+ })
151
+ )
152
+ """)
153
+ ```
154
+
155
+ **Sample output:**
156
+ ```json
157
+ {"slug":"figma","href":"/products/figma","text":"FigmaThe collaborative interface design tool4.9 (1.4K reviews)..."}
158
+ ```
159
+
160
+ Returns ~15 top-rated products in the topic, not recent launches.
161
+
162
+ ---
163
+
164
+ ## Category Page
165
+
166
+ URL: `https://www.producthunt.com/categories/ai-agents`
167
+
168
+ Same `[data-test^="product:"]` selector as topics. Returns 15 top-reviewed products in that category.
169
+
170
+ ```json
171
+ {"slug":"elevenlabs","href":"/products/elevenlabs","text":"ElevenLabs\nCreate natural AI voices instantly...\n4.9 (165 reviews)"}
172
+ ```
173
+
174
+ ---
175
+
176
+ ## Product Detail Page
177
+
178
+ URL: `https://www.producthunt.com/products/claude-opus-4-7`
179
+
180
+ ### Get total vote count (sidebar button)
181
+ ```python
182
+ # Use [data-test="vote-button"] — different from [data-test="action-bar-vote-button"]
183
+ vote_text = js("document.querySelector('[data-test=\"vote-button\"]').outerText.trim().replace(/\\s+/g, ' ')")
184
+ # Returns: "Upvote • 466 points"
185
+ # Parse votes: vote_text.split('•')[1].strip().replace(' points', '').replace(',', '')
186
+ ```
187
+
188
+ ### Get review count and rating
189
+ ```python
190
+ review_link = js("JSON.stringify(Array.from(document.querySelectorAll('a')).filter(a => a.href && a.href.includes('/reviews') && a.outerText.includes('review')).map(a => a.outerText.trim()).slice(0, 1))")
191
+ # Returns: ["1 review"] or ["5.0\n(731 reviews)"]
192
+ ```
193
+
194
+ ### Get day rank (sidebar shows "#1 Day Rank")
195
+ No dedicated `data-test` for rank — parse from sidebar context or use leaderboard position.
196
+
197
+ ### Comments (action-bar-vote-button)
198
+ Each comment has its own `[data-test="action-bar-vote-button"]` with text like `"Upvote (13)"`.
199
+
200
+ ---
201
+
202
+ ## Search Results
203
+
204
+ URL: `https://www.producthunt.com/search?q=AI+agent`
205
+
206
+ Selector: `[data-test^="spotlight-result-product-"]`
207
+
208
+ ```python
209
+ new_tab("https://www.producthunt.com/search?q=AI+agent")
210
+ wait(3)
211
+
212
+ results = js("""
213
+ JSON.stringify(
214
+ Array.from(document.querySelectorAll('[data-test^="spotlight-result-product-"]')).map(el => {
215
+ var id = el.getAttribute('data-test').replace('spotlight-result-product-', '');
216
+ var lines = el.outerText.trim().split('\\n').map(l => l.trim()).filter(l => l);
217
+ return {
218
+ id: id,
219
+ name: lines[0] || null,
220
+ tagline: lines[1] || null,
221
+ review_text: lines[2] || null
222
+ };
223
+ })
224
+ )
225
+ """)
226
+ ```
227
+
228
+ **Note:** Search result elements are `<button>` elements (not `<a>` links), so there is no `href` in the DOM. Product URL must be constructed: `https://www.producthunt.com/products/<slug>` where slug must be derived by other means. The element's `data-test` ID matches the numeric product ID, not the slug.
229
+
230
+ **Sample output:**
231
+ ```json
232
+ {"id":"526014","name":"/ai","tagline":"Access ChatGPT anywhere you type '/ai'","review_text":"2 reviews"}
233
+ {"id":"991302","name":"Naoma AI Demo Agent","tagline":"The first video agent that runs conversational product demos","review_text":"5 reviews"}
234
+ ```
235
+
236
+ ---
237
+
238
+ ## Key Selector Reference
239
+
240
+ | Page | Selector | Count | Notes |
241
+ |------|----------|-------|-------|
242
+ | Homepage | `[data-test^="post-item-"]` | 30 | 4 sections × ~5–7 products |
243
+ | Homepage | `[data-test^="post-name-"]` | 30 | Product name elements |
244
+ | Homepage/Leaderboard | `[data-test="vote-button"]` | varies | `—` for hidden; numeric for visible |
245
+ | Topics/Categories | `[data-test^="product:"]` | ~15 | Top-rated products |
246
+ | Search | `[data-test^="spotlight-result-product-"]` | 10 | Button elements, no href |
247
+ | Product detail | `[data-test="vote-button"]` | 1 | Main vote: "Upvote • N points" |
248
+ | Product detail | `[data-test="action-bar-vote-button"]` | many | Comment upvotes: "Upvote (N)" |
249
+
250
+ ---
251
+
252
+ ## Common Pitfalls
253
+
254
+ 1. **`innerText` returns `None` on complex elements** — use `outerText` or break into simple single-property expressions. Avoid chaining DOM traversal inside `JSON.stringify()` on large objects.
255
+
256
+ 2. **`goto_url()` returns `ERR_ABORTED`** for producthunt.com in some browser sessions — always use `new_tab("url")` instead.
257
+
258
+ 3. **`a[href^="/posts/"]` matches nothing** — Product Hunt uses `/products/` for product URLs, not `/posts/`.
259
+
260
+ 4. **Today's votes are always `—`** during the first 4 hours of the day — use yesterday's leaderboard for confirmed vote counts.
261
+
262
+ 5. **Homepage does not lazy-load more products on scroll** — 30 items is the fixed set. Use leaderboard pages for complete daily listings.
263
+
264
+ 6. **JSON.stringify of DOM-heavy objects returns `None`** — serialize only primitives (strings, numbers) not live DOM node properties.
265
+
266
+ 7. **Ranked product names contain rank prefix** — `"1. Claude Opus 4.7"` — strip with regex `re.sub(r'^\d+\.\s+', '', name)`.
267
+
268
+ 8. **`wait(3)` required after `wait_for_load()`** — the React SPA continues rendering after the load event.
269
+
270
+ ---
271
+
272
+ ## Recommended Workflow for Scraping Today's Launches
273
+
274
+ ```python
275
+ # 1. Open Product Hunt in a new tab
276
+ new_tab("https://www.producthunt.com/leaderboard/daily/2026/4/18")
277
+ wait(4)
278
+
279
+ # 2. Extract all products with metadata
280
+ products = js("""
281
+ JSON.stringify(
282
+ Array.from(document.querySelectorAll('[data-test^="post-item-"]')).map(el => {
283
+ var id = el.getAttribute('data-test').replace('post-item-', '');
284
+ var nameEl = el.querySelector('[data-test^="post-name-"]');
285
+ var productLink = el.querySelector('a[href^="/products/"]');
286
+ var voteBtn = el.querySelector('[data-test="vote-button"]');
287
+ var topicLinks = Array.from(el.querySelectorAll('a[href^="/topics/"]')).map(a => a.textContent.trim());
288
+ var name = nameEl ? nameEl.textContent.trim() : '';
289
+ var lines = el.innerText.split('\\n').map(l => l.trim()).filter(l => l);
290
+ var tagline = lines.find(l => l !== name && !topicLinks.includes(l) && l !== '•' && !/^[0-9—]/.test(l) && l.length > 5);
291
+ return {
292
+ id: id,
293
+ name: name,
294
+ slug: productLink ? productLink.getAttribute('href') : null,
295
+ votes: voteBtn ? voteBtn.textContent.trim() : null,
296
+ topics: topicLinks,
297
+ tagline: tagline || null
298
+ };
299
+ })
300
+ )
301
+ """)
302
+ import json
303
+ data = json.loads(products)
304
+ print(f"Found {len(data)} products")
305
+ for p in data:
306
+ print(f" {p['name']} — {p['votes']} votes — {p['tagline']}")
307
+ ```