@egbe-ai/opencode 0.0.0-stage → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (479) hide show
  1. package/LICENSE +4 -0
  2. package/README.md +124 -2
  3. package/index.js +5 -0
  4. package/package.json +55 -4
  5. package/plugin.js +230 -0
  6. package/session-log.js +315 -0
  7. package/skills/.egbe-catalog/manifest.json +1 -0
  8. package/skills/egbe-os-analytics/SKILL.md +499 -0
  9. package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
  10. package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
  11. package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
  12. package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
  13. package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
  14. package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
  15. package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
  16. package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
  17. package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
  18. package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
  19. package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
  20. package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
  21. package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
  22. package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
  23. package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
  24. package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
  25. package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
  26. package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
  27. package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
  28. package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
  29. package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
  30. package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
  31. package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
  32. package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
  33. package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
  34. package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
  35. package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
  36. package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
  37. package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
  38. package/skills/egbe-os-db/SKILL.md +174 -0
  39. package/skills/egbe-os-deploy/SKILL.md +412 -0
  40. package/skills/egbe-os-design/SKILL.md +182 -0
  41. package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
  42. package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
  43. package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
  44. package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
  45. package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
  46. package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
  47. package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
  48. package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
  49. package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
  50. package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
  51. package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
  52. package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
  53. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
  54. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
  55. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
  56. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
  57. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
  58. package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
  59. package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
  60. package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
  61. package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
  62. package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
  63. package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
  64. package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
  65. package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
  66. package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
  67. package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
  68. package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
  69. package/skills/egbe-os-destination-readiness/references/report.md +116 -0
  70. package/skills/egbe-os-domains/SKILL.md +189 -0
  71. package/skills/egbe-os-errors/SKILL.md +259 -0
  72. package/skills/egbe-os-git/SKILL.md +275 -0
  73. package/skills/egbe-os-git/references/issues.md +33 -0
  74. package/skills/egbe-os-google-ads/SKILL.md +163 -0
  75. package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
  76. package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
  77. package/skills/egbe-os-google-ads/references/conversions.md +153 -0
  78. package/skills/egbe-os-google-ads/references/delivery.md +128 -0
  79. package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
  80. package/skills/egbe-os-google-ads/references/reporting.md +105 -0
  81. package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
  82. package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
  83. package/skills/egbe-os-google-auth/SKILL.md +285 -0
  84. package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
  85. package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
  86. package/skills/egbe-os-images/SKILL.md +163 -0
  87. package/skills/egbe-os-images/references/stock-images.md +21 -0
  88. package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
  89. package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
  90. package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
  91. package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
  92. package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
  93. package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
  94. package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
  95. package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
  96. package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
  97. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
  98. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
  99. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
  100. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
  101. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
  102. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
  103. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
  104. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
  105. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
  106. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
  107. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
  108. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
  109. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
  110. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
  111. package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
  112. package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
  113. package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
  114. package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
  115. package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
  116. package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
  117. package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
  118. package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
  119. package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
  120. package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
  121. package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
  122. package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
  123. package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
  124. package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
  125. package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
  126. package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
  127. package/skills/egbe-os-llm/SKILL.md +160 -0
  128. package/skills/egbe-os-mail/SKILL.md +180 -0
  129. package/skills/egbe-os-me/SKILL.md +119 -0
  130. package/skills/egbe-os-meta-ads/SKILL.md +114 -0
  131. package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
  132. package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
  133. package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
  134. package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
  135. package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
  136. package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
  137. package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
  138. package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
  139. package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
  140. package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
  141. package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
  142. package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
  143. package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
  144. package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
  145. package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
  146. package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
  147. package/skills/egbe-os-notifications/SKILL.md +78 -0
  148. package/skills/egbe-os-openai-ads/SKILL.md +149 -0
  149. package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
  150. package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
  151. package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
  152. package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
  153. package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
  154. package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
  155. package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
  156. package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
  157. package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
  158. package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
  159. package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
  160. package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
  161. package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
  162. package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
  163. package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
  164. package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
  165. package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
  166. package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
  167. package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
  168. package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
  169. package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
  170. package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
  171. package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
  172. package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
  173. package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
  174. package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
  175. package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
  176. package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
  177. package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
  178. package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
  179. package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
  180. package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
  181. package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
  182. package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
  183. package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
  184. package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
  185. package/skills/egbe-os-payments/SKILL.md +215 -0
  186. package/skills/egbe-os-payments/agents/openai.yaml +4 -0
  187. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
  188. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
  189. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
  190. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
  191. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
  192. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
  193. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
  194. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
  195. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
  196. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
  197. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
  198. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
  199. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
  200. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
  201. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
  202. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
  203. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
  204. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
  205. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
  206. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
  207. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
  208. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
  209. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
  210. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
  211. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
  212. package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
  213. package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
  214. package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
  215. package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
  216. package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
  217. package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
  218. package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
  219. package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
  220. package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
  221. package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
  222. package/skills/egbe-os-playbook/SKILL.md +79 -0
  223. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
  224. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
  225. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
  226. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
  227. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
  228. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
  229. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
  230. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
  231. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
  232. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
  233. package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
  234. package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
  235. package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
  236. package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
  237. package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
  238. package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
  239. package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
  240. package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
  241. package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
  242. package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
  243. package/skills/egbe-os-playbook/playbooks/product.md +67 -0
  244. package/skills/egbe-os-playbook/references/catalog.md +127 -0
  245. package/skills/egbe-os-playbook/references/changes.md +167 -0
  246. package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
  247. package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
  248. package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
  249. package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
  250. package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
  251. package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
  252. package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
  253. package/skills/egbe-os-product-auth/SKILL.md +151 -0
  254. package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
  255. package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
  256. package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
  257. package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
  258. package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
  259. package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
  260. package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
  261. package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
  262. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
  263. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
  264. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
  265. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
  266. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
  267. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
  268. package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
  269. package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
  270. package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
  271. package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
  272. package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
  273. package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
  274. package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
  275. package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
  276. package/skills/egbe-os-product-build/SKILL.md +111 -0
  277. package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
  278. package/skills/egbe-os-product-build/references/capabilities.md +25 -0
  279. package/skills/egbe-os-product-build/references/engineering.md +72 -0
  280. package/skills/egbe-os-product-build/references/execution.md +55 -0
  281. package/skills/egbe-os-product-build/references/foundations.md +93 -0
  282. package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
  283. package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
  284. package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
  285. package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
  286. package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
  287. package/skills/egbe-os-product-market-research/SKILL.md +159 -0
  288. package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
  289. package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
  290. package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
  291. package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
  292. package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
  293. package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
  294. package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
  295. package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
  296. package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
  297. package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
  298. package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
  299. package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
  300. package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
  301. package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
  302. package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
  303. package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
  304. package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
  305. package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
  306. package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
  307. package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
  308. package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
  309. package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
  310. package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
  311. package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
  312. package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
  313. package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
  314. package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
  315. package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
  316. package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
  317. package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
  318. package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
  319. package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
  320. package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
  321. package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
  322. package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
  323. package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
  324. package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
  325. package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
  326. package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
  327. package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
  328. package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
  329. package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
  330. package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
  331. package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
  332. package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
  333. package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
  334. package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
  335. package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
  336. package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
  337. package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
  338. package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
  339. package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
  340. package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
  341. package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
  342. package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
  343. package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
  344. package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
  345. package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
  346. package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
  347. package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
  348. package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
  349. package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
  350. package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
  351. package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
  352. package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
  353. package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
  354. package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
  355. package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
  356. package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
  357. package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
  358. package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
  359. package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
  360. package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
  361. package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
  362. package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
  363. package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
  364. package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
  365. package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
  366. package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
  367. package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
  368. package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
  369. package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
  370. package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
  371. package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
  372. package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
  373. package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
  374. package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
  375. package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
  376. package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
  377. package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
  378. package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
  379. package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
  380. package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
  381. package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
  382. package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
  383. package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
  384. package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
  385. package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
  386. package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
  387. package/skills/egbe-os-product-report/SKILL.md +150 -0
  388. package/skills/egbe-os-product-report/references/artifacts.md +84 -0
  389. package/skills/egbe-os-product-report/references/brief.md +34 -0
  390. package/skills/egbe-os-product-report/references/daily.md +76 -0
  391. package/skills/egbe-os-product-report/references/sources.md +68 -0
  392. package/skills/egbe-os-product-routines/SKILL.md +166 -0
  393. package/skills/egbe-os-product-routines/references/catalog.md +22 -0
  394. package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
  395. package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
  396. package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
  397. package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
  398. package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
  399. package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
  400. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
  401. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
  402. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
  403. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
  404. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
  405. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
  406. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
  407. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
  408. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
  409. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
  410. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
  411. package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
  412. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
  413. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
  414. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
  415. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
  416. package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
  417. package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
  418. package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
  419. package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
  420. package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
  421. package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
  422. package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
  423. package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
  424. package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
  425. package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
  426. package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
  427. package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
  428. package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
  429. package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
  430. package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
  431. package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
  432. package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
  433. package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
  434. package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
  435. package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
  436. package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
  437. package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
  438. package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
  439. package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
  440. package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
  441. package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
  442. package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
  443. package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
  444. package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
  445. package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
  446. package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
  447. package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
  448. package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
  449. package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
  450. package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
  451. package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
  452. package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
  453. package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
  454. package/skills/egbe-os-report-ui/SKILL.md +133 -0
  455. package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
  456. package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
  457. package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
  458. package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
  459. package/skills/egbe-os-report-ui/references/components.md +209 -0
  460. package/skills/egbe-os-research/SKILL.md +125 -0
  461. package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
  462. package/skills/egbe-os-scrape/SKILL.md +160 -0
  463. package/skills/egbe-os-search/SKILL.md +182 -0
  464. package/skills/egbe-os-search/references/google-places.md +81 -0
  465. package/skills/egbe-os-support/SKILL.md +149 -0
  466. package/skills/egbe-os-telemetry/SKILL.md +344 -0
  467. package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
  468. package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
  469. package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
  470. package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
  471. package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
  472. package/skills/egbe-os-tools/SKILL.md +180 -0
  473. package/skills/egbe-os-tools/references/theona.md +26 -0
  474. package/skills/egbe-os-wallet/SKILL.md +181 -0
  475. package/skills/egbe-setup/EGBE.md +484 -0
  476. package/skills/egbe-setup/SKILL.md +116 -0
  477. package/skills/egbe-setup/references/project.md +80 -0
  478. package/state.js +37 -0
  479. package/update.js +361 -0
@@ -0,0 +1,1603 @@
1
+ #!/usr/bin/env python3
2
+ """Run the deterministic Community Demand section against community sources.
3
+
4
+ People asking for tools, describing workarounds, or naming prices in public
5
+ communities are demand evidence that precedes any paid channel: the exact
6
+ pain language feeds offers, and the communities themselves are organic and
7
+ paid placements (subreddit ads target subreddits). This runner collects
8
+ both from two sources:
9
+
10
+ - Hacker News via the keyless Algolia search API (stories + comments;
11
+ free, longitudinal);
12
+ - fresh Reddit (last 12 months) via the Apify actor
13
+ `trudax/reddit-scraper-lite` (posts + comments across all of Reddit —
14
+ the official Reddit API is commercially unviable and is never used);
15
+ - YouTube comments under the top videos per query (OS `/tools/youtube`,
16
+ metered per claw against a fleet-shared Google quota) — best-effort:
17
+ a failure is recorded as an excluded source, never a dead section.
18
+
19
+ Quotes are selected by a fixed, versioned marker-regex list (payment
20
+ language, pain language, tool-seeking language) — deterministic heuristics,
21
+ not comprehension. The community observation table is computed over EXACTLY the
22
+ quote rows persisted in the evidence file, so every published number is
23
+ reproducible from it. No LLM anywhere; synthesis into segment/offer/channel
24
+ hypotheses is a downstream concern. Caching, spend provenance, budgets and
25
+ retries mirror the Competitor Reviews runner.
26
+
27
+ Deliberately excluded (recorded as excluded_source rows): Product Hunt
28
+ (its GraphQL API cannot keyword-search launches — live-probed), Arctic
29
+ Shift (its search API cannot keyword-search across all of Reddit), and
30
+ Stack Exchange (v2 scope).
31
+ """
32
+
33
+ from __future__ import annotations
34
+
35
+ import sys as _artifact_sys
36
+ from pathlib import Path as _ArtifactPath
37
+ _artifact_sys.path.insert(0, str(_ArtifactPath(__file__).resolve().parent))
38
+ from egbe_artifacts import artifact_path as _artifact_path
39
+
40
+ import runpy as _egbe_runpy
41
+ from pathlib import Path as _EgbePath
42
+ _egbe_adapter = _EgbePath(__file__).with_name("connection_identity.py")
43
+ _connection_headers = _egbe_runpy.run_path(str(_egbe_adapter))["connection_headers"] if _egbe_adapter.is_file() else lambda _url: {}
44
+
45
+ import argparse
46
+ import hashlib
47
+ import html
48
+ import importlib.util
49
+ import json
50
+ import os
51
+ import re
52
+ import sys
53
+ import time
54
+ import urllib.error
55
+ import urllib.parse
56
+ import urllib.request
57
+ from dataclasses import dataclass
58
+ from datetime import date, datetime, timedelta, timezone
59
+ from pathlib import Path
60
+ from types import ModuleType
61
+ from typing import Any, Callable
62
+
63
+ SCHEMA_VERSION = 1
64
+ SECTION_ID = "community_demand"
65
+ SECTION_HEADING = "Communities analysis"
66
+ # H2s written by bundles that predate the founder-facing rename. The runner
67
+ # folds them into the canonical heading before materializing, so the report
68
+ # never carries this section under two names.
69
+ LEGACY_SECTION_HEADINGS = ("Community demand",)
70
+ DISPLAY_NAME = "Communities Analysis"
71
+
72
+ DEFAULT_OS_URL = "https://os.hz.egbe.dev"
73
+ HN_API_URL = "https://hn.algolia.com/api/v1"
74
+ RUNTIME_REL = Path(_artifact_path("egbe-docs/research/market/work")) / SECTION_ID
75
+ EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/community-demand.json"))
76
+ REPORT_REL = Path(_artifact_path("egbe-docs/research/market/full-report.md"))
77
+
78
+ REDDIT_ACTOR = "trudax/reddit-scraper-lite"
79
+
80
+ # One Reddit actor run, two HN calls, and up to three best-effort YouTube
81
+ # calls (one search + comments of the top videos) per query.
82
+ MAX_QUERIES = 5
83
+ MAX_ACTOR_RUNS_PER_ATTEMPT = 6
84
+ MAX_HN_CALLS_PER_ATTEMPT = 16
85
+ MAX_YT_CALLS_PER_ATTEMPT = 18
86
+ # 60 items fits Apify's 300s sync-endpoint ceiling; 100 exceeded it on a
87
+ # live run (HTTP 408 run-timeout-exceeded). Recall scales via MORE QUERIES,
88
+ # not deeper single runs.
89
+ REDDIT_ITEMS_PER_QUERY = 60
90
+ HN_STORY_HITS = 30
91
+ HN_COMMENT_HITS = 50
92
+ HN_RECENCY_DAYS = 3 * 365
93
+ YT_RECENCY_DAYS = 2 * 365
94
+ YT_SEARCH_RESULTS = 5
95
+ YT_VIDEOS_PER_QUERY = 2
96
+ YT_COMMENTS_PER_VIDEO = 50
97
+ MIN_QUOTE_CHARS = 30
98
+ TEXT_TRUNCATE = 400
99
+ STORED_QUOTES_CAP = 120
100
+ STORED_PER_COMMUNITY_CAP = 15
101
+
102
+ ACTOR_RETRY_DELAY_SECONDS = 20.0
103
+ HN_RETRY_DELAY_SECONDS = 5.0
104
+
105
+ # Versioned deterministic marker list (v1). Quotes are MARKER MATCHES, not
106
+ # classified meaning: the categories name which list matched, nothing more.
107
+ # Payment language must be precise, not broad: a bare dollar amount matches
108
+ # marketing scale-talk ("$100M SaaS") and job listings ("$30-40/hr"), so a
109
+ # money mention counts only NEAR a payment word, and explicit first-person
110
+ # pay phrases count on their own (validated on a live HN pull).
111
+ MARKER_LIST_VERSION = 1
112
+ _PAY_PHRASE_RE = re.compile(
113
+ r"\b(?:would|i'?d|happy to|willing to|gladly|happily)\s+pay\b"
114
+ r"|\bpay(?:ing)?\s+for\b|\bworth paying\b"
115
+ r"|\bpaid\s+(?:plan|version|tier|tool|app)\b"
116
+ r"|\btake my money\b",
117
+ re.IGNORECASE,
118
+ )
119
+ # $100M/$2B are scale brags, not prices — the money marker rejects M/B/k
120
+ # magnitude suffixes.
121
+ _MONEY_RE = re.compile(
122
+ r"\$\s?\d[\d,]*(?:\.\d{1,2})?(?![\dkKmMbB])"
123
+ r"|\b\d+\s?(?:usd|dollars|bucks)\b",
124
+ re.IGNORECASE,
125
+ )
126
+ _PAY_CONTEXT_RE = re.compile(
127
+ r"\bpay(?:s|ing|ment)?\b|\bpaid\b|\bcosts?\b|\bprices?\b|\bpriced\b"
128
+ r"|\bpricing\b|\bcharge[sd]?\b|\bsubscription\b|\bsubscribe[d]?\b"
129
+ r"|\bbill(?:ed|ing)?\b|/mo\b|\bper\s+(?:month|year|seat|user)\b"
130
+ r"|\ba\s+month\b|\ba\s+year\b",
131
+ re.IGNORECASE,
132
+ )
133
+ _MONEY_CONTEXT_WINDOW = 60
134
+
135
+
136
+ def _wtp_marker_positions(text: str) -> list[int]:
137
+ positions = [match.start() for match in _PAY_PHRASE_RE.finditer(text)]
138
+ for money in _MONEY_RE.finditer(text):
139
+ start = max(0, money.start() - _MONEY_CONTEXT_WINDOW)
140
+ end = min(len(text), money.end() + _MONEY_CONTEXT_WINDOW)
141
+ if _PAY_CONTEXT_RE.search(text[start:end]):
142
+ positions.append(money.start())
143
+ return positions
144
+ _PAIN_RE = re.compile(
145
+ r"\bfrustrat\w*|\bannoy\w*|\bhate[sd]?\b|\bpainful\b|\btedious\b"
146
+ r"|\bclunky\b|\bnightmare\b|\bdrives me (?:crazy|nuts|insane)\b"
147
+ r"|\bwaste[sd]? (?:of )?(?:so much )?time\b|\bhours every (?:day|week)\b"
148
+ r"|\bmanually\b|\bstruggl\w*|\bcan'?t stand\b|\btoo (?:slow|complicated)\b"
149
+ r"|\bbuggy\b|\bunusable\b|\bgave up\b",
150
+ re.IGNORECASE,
151
+ )
152
+ _SEEKING_RE = re.compile(
153
+ r"\blooking for (?:a|an|some|any)\b|\bis there (?:a|an|any)\b"
154
+ r"|\bany (?:good )?(?:recommendations?|alternatives?|tools?)\b"
155
+ r"|\brecommend (?:a|an|any|me)\b|\balternative to\b"
156
+ r"|\bwish there (?:was|were)\b|\bi wish (?:it|there|someone)\b"
157
+ r"|\bdoes anyone (?:know|use)\b|\bwhat (?:do you|does everyone) use\b"
158
+ r"|\bhow do you (?:all )?(?:manage|track|handle)\b",
159
+ re.IGNORECASE,
160
+ )
161
+ # Price-sensitivity is its own diagnostic category, not a WTP signal: a
162
+ # segment repeatedly asking for "free/cheap" tools is a monetization
163
+ # warning that must be visible in the tables, not buried inside a seeking
164
+ # quote (the tutors' "ideally free" lesson from the first live run).
165
+ _PRICE_SENS_RE = re.compile(
166
+ r"\bfree\b|\bfreemium\b|\bfree tier\b|\bcheap(?:er|est)?\b"
167
+ r"|\baffordable\b|\bbudget\b|\blow[- ]cost\b|\binexpensive\b"
168
+ r"|\btoo expensive\b|\boverpriced\b|\bpricey\b|\bcan(?:'|no)t afford\b",
169
+ re.IGNORECASE,
170
+ )
171
+ _BOT_USERNAMES = {"automoderator", "sneakpeekbot", "remindmebot", "savevideo"}
172
+ _TAG_RE = re.compile(r"<[^>]+>")
173
+ # Sellers pitching their own product are not demand; separator-formatted
174
+ # classifieds (HN "Who is hiring"/"Who wants to be hired" comments, job
175
+ # posts) are listings, not demand language. All observed dominating a live
176
+ # relevance-sorted pull.
177
+ _PROMO_TITLE_RE = re.compile(r"^\s*(?:show|launch)\s+hn\b", re.IGNORECASE)
178
+ _JOBS_THREAD_RE = re.compile(
179
+ r"who is hiring|who wants to be hired|seeking freelancer", re.IGNORECASE
180
+ )
181
+ # Reddit self-promo posts are sellers, not askers ("wanted to share a
182
+ # little tool me and my SO have been building" — slipped through the first
183
+ # live run as a seeking quote).
184
+ _SELF_PROMO_RE = re.compile(
185
+ r"\b(?:i|we)(?:'ve| have)? (?:built|made|created|launched)\b"
186
+ r"|\bwanted to share\b|\bsharing (?:my|our)\b|\bcheck out (?:my|our)\b"
187
+ r"|\bmy (?:side |latest )?project\b",
188
+ re.IGNORECASE,
189
+ )
190
+ _LISTING_MIN_SEPARATORS = 3
191
+ _LISTING_LABELS = (
192
+ "location:",
193
+ "remote:",
194
+ "willing to relocate:",
195
+ "technologies:",
196
+ "compensation:",
197
+ "salary:",
198
+ "resume:",
199
+ "résumé:",
200
+ )
201
+ # Query tokens too generic to anchor topical relevance on their own.
202
+ _GENERIC_QUERY_TOKENS = {
203
+ "ai",
204
+ "app",
205
+ "apps",
206
+ "best",
207
+ "for",
208
+ "free",
209
+ "online",
210
+ "platform",
211
+ "service",
212
+ "software",
213
+ "the",
214
+ "tool",
215
+ "tools",
216
+ "top",
217
+ "web",
218
+ }
219
+
220
+ HN_COMMUNITY = "news.ycombinator.com"
221
+
222
+
223
+ class CommunityDemandError(RuntimeError):
224
+ """Terminal runner failure; runtime state is preserved for resume."""
225
+
226
+
227
+ def _load_script(filename: str, module_name: str) -> ModuleType:
228
+ path = Path(__file__).resolve().parent / filename
229
+ spec = importlib.util.spec_from_file_location(module_name, path)
230
+ if spec is None or spec.loader is None:
231
+ raise CommunityDemandError(f"cannot load bundled script {filename}")
232
+ module = importlib.util.module_from_spec(spec)
233
+ sys.modules[spec.name] = module
234
+ previous = sys.dont_write_bytecode
235
+ sys.dont_write_bytecode = True
236
+ try:
237
+ spec.loader.exec_module(module)
238
+ finally:
239
+ sys.dont_write_bytecode = previous
240
+ return module
241
+
242
+
243
+ # ── mixed client: paid OS actor runs + free keyless HN calls ───────────────
244
+
245
+ @dataclass
246
+ class OsClient:
247
+ os_url: str
248
+ api_key: str
249
+ cache_dir: Path
250
+ max_actor_runs: int = MAX_ACTOR_RUNS_PER_ATTEMPT
251
+ max_hn_calls: int = MAX_HN_CALLS_PER_ATTEMPT
252
+ max_yt_calls: int = MAX_YT_CALLS_PER_ATTEMPT
253
+ hn_url: str = HN_API_URL
254
+ actor_runs: int = 0
255
+ hn_calls: int = 0
256
+ yt_calls: int = 0
257
+ cache_hits: int = 0
258
+ touched: set[str] = None # type: ignore[assignment] # set in __post_init__
259
+ request: Callable[..., dict[str, Any]] | None = None
260
+ hn_request: Callable[[str], dict[str, Any]] | None = None
261
+ sleep: Callable[[float], None] | None = None
262
+
263
+ def __post_init__(self) -> None:
264
+ if self.touched is None:
265
+ self.touched = set()
266
+
267
+ def _http(self, path: str, payload: dict[str, Any], timeout: float, *, idempotency_key: str | None = None) -> dict[str, Any]:
268
+ url = f"{self.os_url.rstrip('/')}{path}"
269
+ headers = {
270
+ "Authorization": f"Bearer {self.api_key}",
271
+ "Content-Type": "application/json",
272
+ "Accept": "application/json",
273
+ }
274
+ if idempotency_key:
275
+ headers["Idempotency-Key"] = idempotency_key
276
+ req = urllib.request.Request(
277
+ url,
278
+ data=json.dumps(payload).encode("utf-8"),
279
+ headers=headers,
280
+ method="POST",
281
+ )
282
+ try:
283
+ for _name, _value in _connection_headers(req.full_url).items():
284
+ req.add_unredirected_header(_name, _value)
285
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
286
+ envelope = json.loads(resp.read().decode("utf-8"))
287
+ except urllib.error.HTTPError as exc:
288
+ detail = exc.read().decode("utf-8", errors="replace")[:500]
289
+ raise CommunityDemandError(f"OS {path} failed: HTTP {exc.code} {detail}") from exc
290
+ except urllib.error.URLError as exc:
291
+ raise CommunityDemandError(f"OS {path} unreachable: {exc.reason}") from exc
292
+ data = envelope.get("data")
293
+ if envelope.get("code") not in (None, 200) or not isinstance(data, dict):
294
+ raise CommunityDemandError(f"OS {path} returned no data: {str(envelope)[:300]}")
295
+ return data
296
+
297
+ def _hn_http(self, url: str) -> dict[str, Any]:
298
+ req = urllib.request.Request(url, headers={"Accept": "application/json"})
299
+ try:
300
+ for _name, _value in _connection_headers(req.full_url).items():
301
+ req.add_unredirected_header(_name, _value)
302
+ with urllib.request.urlopen(req, timeout=30.0) as resp:
303
+ value = json.loads(resp.read().decode("utf-8"))
304
+ except urllib.error.HTTPError as exc:
305
+ raise CommunityDemandError(f"HN search failed: HTTP {exc.code}") from exc
306
+ except urllib.error.URLError as exc:
307
+ raise CommunityDemandError(f"HN search unreachable: {exc.reason}") from exc
308
+ if not isinstance(value, dict):
309
+ raise CommunityDemandError("HN search returned a non-object response")
310
+ return value
311
+
312
+ def _cache_read(self, cache_file: Path) -> dict[str, Any] | None:
313
+ if not cache_file.is_file():
314
+ return None
315
+ try:
316
+ return json.loads(cache_file.read_text(encoding="utf-8"))
317
+ except (OSError, json.JSONDecodeError):
318
+ # A torn write must not poison replay forever: discard the
319
+ # corrupt entry and refetch.
320
+ cache_file.unlink(missing_ok=True)
321
+ return None
322
+
323
+ def _cache_write(self, cache_file: Path, entry: dict[str, Any]) -> None:
324
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
325
+ tmp_file = cache_file.with_suffix(".json.tmp")
326
+ tmp_file.write_text(json.dumps(entry), encoding="utf-8")
327
+ tmp_file.replace(cache_file)
328
+
329
+ def run_actor(self, actor: str, actor_input: dict[str, Any], limit: int) -> tuple[list[dict[str, Any]], str]:
330
+ payload = {"actor": actor, "input": actor_input, "sync": True, "limit": limit}
331
+ key_raw = json.dumps({"kind": "actor", "payload": payload}, sort_keys=True)
332
+ digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
333
+ cache_file = self.cache_dir / f"{digest}.json"
334
+ self.touched.add(digest)
335
+ entry = self._cache_read(cache_file)
336
+ if entry is not None:
337
+ self.cache_hits += 1
338
+ rows = entry["response"].get("rows")
339
+ return (
340
+ rows if isinstance(rows, list) else []
341
+ ), entry.get("retrieved_on") or date.today().isoformat()
342
+
343
+ transport = self.request or self._http
344
+ attempts = 0
345
+ # Salted ONCE per call, before the retry loop: a within-call
346
+ # transient-5xx retry must reuse the SAME wire key (so the gateway
347
+ # can dedupe an ambiguous sync failure into one upstream run), but a
348
+ # fresh call — a prior attempt having already failed and journaled a
349
+ # start for this digest — must get a NEW key. The cache key (digest)
350
+ # stays the pure payload hash so replay lookups are unaffected.
351
+ wire_key = f"{digest}-{self._prior_start_count(digest)}"
352
+ while True:
353
+ # Paid starts count BEFORE the transport call (fail closed) and
354
+ # are journaled durably: a failed start may still have billed.
355
+ if self.actor_runs >= self.max_actor_runs:
356
+ raise CommunityDemandError(
357
+ f"actor-run budget exhausted ({self.max_actor_runs} per "
358
+ "attempt); narrow the query list or re-run to continue "
359
+ "from cache"
360
+ )
361
+ self.actor_runs += 1
362
+ self._journal_start(digest)
363
+ try:
364
+ response = transport(
365
+ "/scrape/apify/run", payload, 330.0, idempotency_key=wire_key
366
+ )
367
+ break
368
+ except CommunityDemandError as exc:
369
+ if attempts >= 1 or not re.search(r"HTTP 5\d\d", str(exc)):
370
+ raise
371
+ attempts += 1
372
+ (self.sleep or time.sleep)(ACTOR_RETRY_DELAY_SECONDS)
373
+ retrieved_on = date.today().isoformat()
374
+ self._cache_write(
375
+ cache_file,
376
+ {
377
+ "kind": "actor",
378
+ "payload": payload,
379
+ "response": response,
380
+ "retrieved_on": retrieved_on,
381
+ },
382
+ )
383
+ rows = response.get("rows")
384
+ return (rows if isinstance(rows, list) else []), retrieved_on
385
+
386
+ def hn_search(
387
+ self, query: str, tags: str, hits_per_page: int, created_after_epoch: int
388
+ ) -> tuple[list[dict[str, Any]], str]:
389
+ # The recency window lives in the QUERY (and therefore the cache
390
+ # key): an all-time relevance page can fill up with old hits and
391
+ # silently hide newer demand.
392
+ params = urllib.parse.urlencode(
393
+ {
394
+ "query": query,
395
+ "tags": tags,
396
+ "hitsPerPage": hits_per_page,
397
+ "numericFilters": f"created_at_i>{created_after_epoch}",
398
+ }
399
+ )
400
+ url = f"{self.hn_url.rstrip('/')}/search?{params}"
401
+ key_raw = json.dumps({"kind": "hn", "url": url}, sort_keys=True)
402
+ digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
403
+ cache_file = self.cache_dir / f"{digest}.json"
404
+ self.touched.add(digest)
405
+ entry = self._cache_read(cache_file)
406
+ if entry is not None:
407
+ self.cache_hits += 1
408
+ hits = entry["response"].get("hits")
409
+ return (
410
+ hits if isinstance(hits, list) else []
411
+ ), entry.get("retrieved_on") or date.today().isoformat()
412
+
413
+ transport = self.hn_request or self._hn_http
414
+ attempts = 0
415
+ while True:
416
+ # Free calls are still bounded: a broken loop must not hammer a
417
+ # keyless public API.
418
+ if self.hn_calls >= self.max_hn_calls:
419
+ raise CommunityDemandError(
420
+ f"HN call budget exhausted ({self.max_hn_calls} per attempt)"
421
+ )
422
+ self.hn_calls += 1
423
+ try:
424
+ response = transport(url)
425
+ break
426
+ except CommunityDemandError as exc:
427
+ if attempts >= 1 or not re.search(r"HTTP 5\d\d", str(exc)):
428
+ raise
429
+ attempts += 1
430
+ (self.sleep or time.sleep)(HN_RETRY_DELAY_SECONDS)
431
+ retrieved_on = date.today().isoformat()
432
+ self._cache_write(
433
+ cache_file,
434
+ {
435
+ "kind": "hn",
436
+ "url": url,
437
+ "response": response,
438
+ "retrieved_on": retrieved_on,
439
+ },
440
+ )
441
+ hits = response.get("hits")
442
+ return (hits if isinstance(hits, list) else []), retrieved_on
443
+
444
+ def yt_call(self, operation: str, payload: dict[str, Any]) -> tuple[dict[str, Any], str]:
445
+ """One OS `/tools/youtube/<operation>` call. Metered per claw but
446
+ free in money terms; bounded so a broken loop cannot drain the
447
+ fleet-shared Google quota."""
448
+ key_raw = json.dumps(
449
+ {"kind": "yt", "operation": operation, "payload": payload}, sort_keys=True
450
+ )
451
+ digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
452
+ cache_file = self.cache_dir / f"{digest}.json"
453
+ self.touched.add(digest)
454
+ entry = self._cache_read(cache_file)
455
+ if entry is not None:
456
+ self.cache_hits += 1
457
+ response = entry["response"]
458
+ return (
459
+ response if isinstance(response, dict) else {}
460
+ ), entry.get("retrieved_on") or date.today().isoformat()
461
+
462
+ transport = self.request or self._http
463
+ attempts = 0
464
+ while True:
465
+ if self.yt_calls >= self.max_yt_calls:
466
+ raise CommunityDemandError(
467
+ f"YouTube call budget exhausted ({self.max_yt_calls} per attempt)"
468
+ )
469
+ self.yt_calls += 1
470
+ try:
471
+ response = transport(f"/tools/youtube/{operation}", payload, 60.0)
472
+ break
473
+ except CommunityDemandError as exc:
474
+ if attempts >= 1 or not re.search(r"HTTP 5\d\d", str(exc)):
475
+ raise
476
+ attempts += 1
477
+ (self.sleep or time.sleep)(HN_RETRY_DELAY_SECONDS)
478
+ retrieved_on = date.today().isoformat()
479
+ self._cache_write(
480
+ cache_file,
481
+ {
482
+ "kind": "yt",
483
+ "operation": operation,
484
+ "payload": payload,
485
+ "response": response,
486
+ "retrieved_on": retrieved_on,
487
+ },
488
+ )
489
+ return (response if isinstance(response, dict) else {}), retrieved_on
490
+
491
+ def _journal_start(self, digest: str) -> None:
492
+ """Append-only durable record of every PAID start, successful or not
493
+ — the cache holds only successes, but failed sync starts may still
494
+ have billed upstream."""
495
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
496
+ with (self.cache_dir / "starts.log").open("a", encoding="utf-8") as handle:
497
+ handle.write(f"{date.today().isoformat()} {digest}\n")
498
+
499
+ def _prior_start_count(self, digest: str) -> int:
500
+ """Paid starts already journaled for this digest by EARLIER
501
+ invocations — the salt for the wire Idempotency-Key. A failed
502
+ attempt's key must never be replayed verbatim: on the gateway (see
503
+ #3715), a client key can poison a wallet hold, and a byte-identical
504
+ retry of a poisoned key fails identically forever."""
505
+ starts_log = self.cache_dir / "starts.log"
506
+ try:
507
+ lines = starts_log.read_text(encoding="utf-8").splitlines()
508
+ except OSError:
509
+ return 0
510
+ return sum(1 for line in lines if line.endswith(f" {digest}"))
511
+
512
+ def ledger_totals(self) -> dict[str, Any]:
513
+ """Durable provenance over entries THIS execution touched."""
514
+ actor_entries = 0
515
+ hn_entries = 0
516
+ yt_entries = 0
517
+ dates: list[str] = []
518
+ if self.cache_dir.is_dir():
519
+ for digest in sorted(self.touched):
520
+ entry_path = self.cache_dir / f"{digest}.json"
521
+ try:
522
+ entry = json.loads(entry_path.read_text(encoding="utf-8"))
523
+ except (OSError, json.JSONDecodeError):
524
+ continue
525
+ kind = entry.get("kind")
526
+ if kind == "hn":
527
+ hn_entries += 1
528
+ elif kind == "yt":
529
+ yt_entries += 1
530
+ else:
531
+ actor_entries += 1
532
+ if isinstance(entry.get("retrieved_on"), str):
533
+ dates.append(entry["retrieved_on"])
534
+ today = date.today().isoformat()
535
+ starts_log = self.cache_dir / "starts.log"
536
+ try:
537
+ starts_recorded = sum(
538
+ 1 for line in starts_log.read_text(encoding="utf-8").splitlines() if line
539
+ )
540
+ except OSError:
541
+ starts_recorded = 0
542
+ return {
543
+ "actor_runs": actor_entries,
544
+ "provider_calls": actor_entries,
545
+ "hn_calls": hn_entries,
546
+ "youtube_calls": yt_entries,
547
+ "actor_starts_recorded": starts_recorded,
548
+ "first_retrieved_on": min(dates) if dates else today,
549
+ "last_retrieved_on": max(dates) if dates else today,
550
+ }
551
+
552
+
553
+ # ── deterministic helpers ──────────────────────────────────────────────────
554
+
555
+ def _clean(value: Any) -> Any:
556
+ """Apify actors serialize empty fields as the literal string 'None'."""
557
+ if value is None:
558
+ return None
559
+ if isinstance(value, str) and value.strip() in ("", "None", "null"):
560
+ return None
561
+ return value
562
+
563
+
564
+ def _to_int(value: Any) -> int | None:
565
+ value = _clean(value)
566
+ if value is None:
567
+ return None
568
+ try:
569
+ return int(float(value))
570
+ except (TypeError, ValueError):
571
+ return None
572
+
573
+
574
+ def _plain_text(value: Any) -> str:
575
+ """HTML-decode, strip tags (HN comment_text carries markup), collapse
576
+ whitespace."""
577
+ text = html.unescape(str(_clean(value) or ""))
578
+ text = _TAG_RE.sub(" ", text)
579
+ return " ".join(text.split())
580
+
581
+
582
+ def _truncate(text: str, cap: int) -> str:
583
+ if len(text) <= cap:
584
+ return text
585
+ cut = text[:cap].rsplit(" ", 1)[0].rstrip()
586
+ return (cut or text[:cap].rstrip()) + "…"
587
+
588
+
589
+ # A long comment can mention the category once and rant about something
590
+ # else: a marker is topical only NEAR a query anchor token (validated on a
591
+ # live pull where AI-industry rants outnumbered category demand).
592
+ _ANCHOR_PROXIMITY_WINDOW = 120
593
+
594
+
595
+ def _anchor_positions(text: str, query: str) -> list[int] | None:
596
+ """None means the query is fully generic and imposes no anchor."""
597
+ anchors = query_anchor_tokens(query)
598
+ if not anchors:
599
+ return None
600
+ lowered = text.lower()
601
+ return [
602
+ match.start()
603
+ for anchor in anchors
604
+ for match in re.finditer(rf"\b{re.escape(anchor)}", lowered)
605
+ ]
606
+
607
+
608
+ def _near_any(positions: list[int] | None, marker_positions: list[int]) -> bool:
609
+ if not marker_positions:
610
+ return False
611
+ if positions is None:
612
+ return True
613
+ return any(
614
+ abs(marker - anchor) <= _ANCHOR_PROXIMITY_WINDOW
615
+ for marker in marker_positions
616
+ for anchor in positions
617
+ )
618
+
619
+
620
+ def matched_categories(text: str, query: str = "") -> list[str]:
621
+ anchor_positions = _anchor_positions(text, query) if query else None
622
+ if anchor_positions is not None and not anchor_positions:
623
+ return [] # anchored query, text never mentions the category
624
+ categories: list[str] = []
625
+ if _near_any(anchor_positions, _wtp_marker_positions(text)):
626
+ categories.append("wtp")
627
+ if _near_any(
628
+ anchor_positions, [m.start() for m in _SEEKING_RE.finditer(text)]
629
+ ):
630
+ categories.append("seeking")
631
+ if _near_any(
632
+ anchor_positions, [m.start() for m in _PAIN_RE.finditer(text)]
633
+ ):
634
+ categories.append("pain")
635
+ if _near_any(
636
+ anchor_positions, [m.start() for m in _PRICE_SENS_RE.finditer(text)]
637
+ ):
638
+ categories.append("price_sensitivity")
639
+ return categories
640
+
641
+
642
+ def parse_context_tokens(raw: str) -> list[str]:
643
+ tokens: list[str] = []
644
+ for part in re.split(r"[,;\n]", raw or ""):
645
+ token = re.sub(r"[^a-z0-9]", "", part.strip().lower())
646
+ if len(token) >= 2 and token not in tokens:
647
+ tokens.append(token)
648
+ return tokens[:8]
649
+
650
+
651
+ def mentions_product_context(text: str, context_tokens: list[str]) -> bool:
652
+ """Does the text speak to THIS product's mechanism (e.g. 'ai', 'agent')
653
+ rather than the bare category? Short tokens match exact words ('ai'
654
+ must not match 'aim'); longer ones are prefix-tolerant."""
655
+ lowered = text.lower()
656
+ for token in context_tokens:
657
+ pattern = (
658
+ rf"\b{re.escape(token)}\b" if len(token) <= 3 else rf"\b{re.escape(token)}"
659
+ )
660
+ if re.search(pattern, lowered):
661
+ return True
662
+ return False
663
+
664
+
665
+ def is_listing(text: str) -> bool:
666
+ """Separator-formatted classifieds (job posts, 'Who is hiring' entries)
667
+ and labeled hire-me templates."""
668
+ if text.count(" | ") + text.count(" / ") >= _LISTING_MIN_SEPARATORS:
669
+ return True
670
+ lowered = text.lower()
671
+ return sum(1 for label in _LISTING_LABELS if label in lowered) >= 2
672
+
673
+
674
+ def query_anchor_tokens(query: str) -> list[str]:
675
+ return [
676
+ token
677
+ for token in re.findall(r"[a-z0-9]+", query.lower())
678
+ if len(token) >= 3 and token not in _GENERIC_QUERY_TOKENS
679
+ ]
680
+
681
+
682
+
683
+
684
+ def parse_terms(raw: str, cap: int) -> list[str]:
685
+ seen: dict[str, None] = {}
686
+ for part in re.split(r"[,;\n]", raw or ""):
687
+ term = " ".join(part.strip().lower().split())
688
+ if term and term not in seen:
689
+ seen[term] = None
690
+ return list(seen)[:cap]
691
+
692
+
693
+
694
+ def select_queries(
695
+ explicit: list[str], discovered: list[str], cap: int
696
+ ) -> tuple[list[str], dict[str, str]]:
697
+ """Merge explicit seeds ahead of KWE-discovered keywords, dropping
698
+ token-set duplicates ('online whiteboard online' is the same query as
699
+ 'online whiteboard' — a KWE keyword-variant artifact)."""
700
+ queries: list[str] = []
701
+ query_sources: dict[str, str] = {}
702
+ seen_token_sets: set[frozenset[str]] = set()
703
+ for query, source in [(q, "explicit") for q in explicit] + [
704
+ (q, "keyword_economics") for q in discovered
705
+ ]:
706
+ if len(queries) >= cap:
707
+ break
708
+ token_set = frozenset(re.findall(r"[a-z0-9]+", query))
709
+ if not token_set or token_set in seen_token_sets:
710
+ continue
711
+ seen_token_sets.add(token_set)
712
+ queries.append(query)
713
+ query_sources[query] = source
714
+ return queries, query_sources
715
+
716
+
717
+ def queries_from_keyword_evidence(workspace: Path, cap: int) -> list[str]:
718
+ evidence = _load_script("research-evidence.py", "pmr_evidence")
719
+ return evidence.keyword_inputs(workspace, consumer=SECTION_ID, row_type="keyword", field="keyword", cap=cap)
720
+
721
+
722
+ # ── normalization: one raw item -> one candidate quote ─────────────────────
723
+
724
+ def normalize_hn_story(hit: dict[str, Any], query: str) -> dict[str, Any] | None:
725
+ object_id = str(_clean(hit.get("objectID")) or "")
726
+ title = _plain_text(hit.get("title"))
727
+ body = _plain_text(hit.get("story_text"))
728
+ text = " ".join(part for part in (title, body) if part)
729
+ if not object_id or not text:
730
+ return None
731
+ if _PROMO_TITLE_RE.search(title):
732
+ return None # the author is selling, not asking
733
+ return {
734
+ "id": f"hn_{object_id}",
735
+ "source": "hackernews",
736
+ "community": HN_COMMUNITY,
737
+ "kind": "story",
738
+ "query": query,
739
+ "text": text,
740
+ "url": f"https://news.ycombinator.com/item?id={object_id}",
741
+ "votes": _to_int(hit.get("points")) or 0,
742
+ "replies": _to_int(hit.get("num_comments")),
743
+ "created": str(_clean(hit.get("created_at")) or "")[:10] or None,
744
+ }
745
+
746
+
747
+ def normalize_hn_comment(hit: dict[str, Any], query: str) -> dict[str, Any] | None:
748
+ object_id = str(_clean(hit.get("objectID")) or "")
749
+ text = _plain_text(hit.get("comment_text"))
750
+ if not object_id or not text:
751
+ return None
752
+ if _JOBS_THREAD_RE.search(str(_clean(hit.get("story_title")) or "")):
753
+ return None # monthly jobs threads are classifieds, not demand
754
+ return {
755
+ "id": f"hn_{object_id}",
756
+ "source": "hackernews",
757
+ "community": HN_COMMUNITY,
758
+ "kind": "comment",
759
+ "query": query,
760
+ "text": text,
761
+ "url": f"https://news.ycombinator.com/item?id={object_id}",
762
+ "votes": _to_int(hit.get("points")) or 0,
763
+ "replies": None,
764
+ "created": str(_clean(hit.get("created_at")) or "")[:10] or None,
765
+ }
766
+
767
+
768
+ def normalize_reddit_row(row: dict[str, Any], query: str) -> dict[str, Any] | None:
769
+ data_type = str(_clean(row.get("dataType")) or "")
770
+ if data_type not in ("post", "comment"):
771
+ return None
772
+ row_id = str(_clean(row.get("id")) or "")
773
+ community = str(
774
+ _clean(row.get("parsedCommunityName"))
775
+ or str(_clean(row.get("communityName")) or "").removeprefix("r/")
776
+ or ""
777
+ ).strip()
778
+ username = str(_clean(row.get("username")) or "").strip().lower()
779
+ title = _plain_text(row.get("title")) if data_type == "post" else ""
780
+ body = _plain_text(row.get("body"))
781
+ text = " ".join(part for part in (title, body) if part)
782
+ if not row_id or not community or not text:
783
+ return None
784
+ if username in _BOT_USERNAMES or "i am a bot" in text.lower():
785
+ return None
786
+ if data_type == "post" and _SELF_PROMO_RE.search(text):
787
+ return None # the author is selling, not asking
788
+ url = str(_clean(row.get("url")) or _clean(row.get("link")) or "")
789
+ if not url:
790
+ permalink = str(_clean(row.get("permalink")) or "")
791
+ url = f"https://www.reddit.com{permalink}" if permalink.startswith("/") else ""
792
+ return {
793
+ "id": f"reddit_{row_id}",
794
+ "source": "reddit",
795
+ "community": f"r/{community}",
796
+ "kind": data_type,
797
+ "query": query,
798
+ "text": text,
799
+ "url": url or None,
800
+ "votes": _to_int(row.get("upVotes")) or 0,
801
+ "replies": _to_int(row.get("numberOfComments") if data_type == "post" else row.get("numberOfReplies")),
802
+ "created": str(_clean(row.get("createdAt")) or "")[:10] or None,
803
+ }
804
+
805
+
806
+ def normalize_youtube_comment(
807
+ row: dict[str, Any], query: str, video: dict[str, Any]
808
+ ) -> dict[str, Any] | None:
809
+ comment_id = str(_clean(row.get("comment_id")) or "")
810
+ video_id = str(_clean(row.get("video_id")) or _clean(video.get("video_id")) or "")
811
+ text = _plain_text(row.get("text"))
812
+ if not comment_id or not video_id or not text:
813
+ return None
814
+ return {
815
+ "id": f"yt_{comment_id}",
816
+ "source": "youtube",
817
+ "community": "youtube.com",
818
+ "kind": "comment",
819
+ "query": query,
820
+ "text": text,
821
+ "url": f"https://www.youtube.com/watch?v={video_id}&lc={comment_id}",
822
+ "votes": _to_int(row.get("like_count")) or 0,
823
+ "replies": _to_int(row.get("reply_count")),
824
+ "created": str(_clean(row.get("published_at")) or "")[:10] or None,
825
+ "video_title": _truncate(_plain_text(video.get("title")), 120) or None,
826
+ }
827
+
828
+
829
+ def to_quote_row(
830
+ candidate: dict[str, Any], hn_cutoff: str, context_tokens: list[str]
831
+ ) -> dict[str, Any] | None:
832
+ """Marker-match one normalized candidate into an evidence quote row."""
833
+ text = candidate["text"]
834
+ if len(text) < MIN_QUOTE_CHARS:
835
+ return None
836
+ if is_listing(text):
837
+ return None
838
+ if candidate["source"] == "hackernews":
839
+ created = candidate.get("created")
840
+ if created is not None and created < hn_cutoff:
841
+ return None
842
+ categories = matched_categories(text, candidate["query"])
843
+ if not categories:
844
+ return None
845
+ quote = {
846
+ "type": "community_quote",
847
+ "id": candidate["id"],
848
+ "source": candidate["source"],
849
+ "community": candidate["community"],
850
+ "kind": candidate["kind"],
851
+ "query": candidate["query"],
852
+ "categories": categories,
853
+ "has_dollar_amount": bool(_MONEY_RE.search(text)),
854
+ # Category demand vs THIS product's mechanism: a quote about
855
+ # human-to-human realtime collaboration is not evidence for an
856
+ # AI-agent whiteboard even when both say "whiteboard".
857
+ "product_context": mentions_product_context(text, context_tokens)
858
+ if context_tokens
859
+ else None,
860
+ "text": _truncate(text, TEXT_TRUNCATE),
861
+ "url": candidate.get("url"),
862
+ "votes": candidate.get("votes") or 0,
863
+ "replies": candidate.get("replies"),
864
+ "created": candidate.get("created"),
865
+ "tier": "A",
866
+ }
867
+ if candidate.get("video_title"):
868
+ quote["video_title"] = candidate["video_title"]
869
+ return quote
870
+
871
+
872
+ def _created_recency_key(quote: dict[str, Any]) -> int:
873
+ """Newest first, missing/unparsable dates last (0 sorts after every
874
+ negated real date)."""
875
+ digits = re.sub(r"[^0-9]", "", str(quote.get("created") or ""))[:8]
876
+ return -int(digits) if len(digits) == 8 else 0
877
+
878
+
879
+ def select_stored_quotes(quotes: list[dict[str, Any]]) -> list[dict[str, Any]]:
880
+ """Bounded observation sample: newest first, stable ID ties.
881
+ Per-community caps prevent one thread population consuming the sample;
882
+ payment markers and votes do not grant evidence priority."""
883
+ ranked = sorted(quotes, key=lambda q: (_created_recency_key(q), q["id"]))
884
+ stored: list[dict[str, Any]] = []
885
+ per_community: dict[str, int] = {}
886
+ for quote in ranked:
887
+ if len(stored) >= STORED_QUOTES_CAP:
888
+ break
889
+ count = per_community.get(quote["community"], 0)
890
+ if count >= STORED_PER_COMMUNITY_CAP:
891
+ continue
892
+ per_community[quote["community"]] = count + 1
893
+ stored.append(quote)
894
+ return stored
895
+
896
+
897
+ def summarize_communities(stored: list[dict[str, Any]]) -> list[dict[str, Any]]:
898
+ by_community: dict[str, list[dict[str, Any]]] = {}
899
+ for quote in stored:
900
+ by_community.setdefault(quote["community"], []).append(quote)
901
+ rows: list[dict[str, Any]] = []
902
+ for community, quotes in by_community.items():
903
+ wtp = sum(1 for q in quotes if "wtp" in q["categories"])
904
+ seeking = sum(1 for q in quotes if "seeking" in q["categories"])
905
+ pain = sum(1 for q in quotes if "pain" in q["categories"])
906
+ price_sens = sum(
907
+ 1 for q in quotes if "price_sensitivity" in q["categories"]
908
+ )
909
+ product_context = sum(1 for q in quotes if q.get("product_context"))
910
+ votes = sum(q["votes"] for q in quotes)
911
+ sample = min(quotes, key=lambda q: (_created_recency_key(q), q["id"]))
912
+ rows.append(
913
+ {
914
+ "type": "community",
915
+ "community": community,
916
+ "source": quotes[0]["source"],
917
+ "quotes": len(quotes),
918
+ "wtp_quotes": wtp,
919
+ "seeking_quotes": seeking,
920
+ "pain_quotes": pain,
921
+ "price_sensitivity_quotes": price_sens,
922
+ "product_context_quotes": product_context,
923
+ "votes_on_quotes": votes,
924
+ "latest_quote_date": max((q.get("created") or "" for q in quotes), default="") or None,
925
+ "sample_url": sample.get("url"),
926
+ "tier": "A",
927
+ }
928
+ )
929
+ rows.sort(key=lambda r: r["community"])
930
+ return rows
931
+
932
+
933
+ def excluded_source_rows(youtube_failure: str | None) -> list[dict[str, Any]]:
934
+ rows = [
935
+ {
936
+ "type": "excluded_source",
937
+ "source": "product_hunt",
938
+ "reason": "GraphQL API v2 cannot keyword-search launches (no "
939
+ "post search, coarse topics, url filter does not match "
940
+ "domains); excluded after live probes 2026-08-19",
941
+ "tier": "C",
942
+ },
943
+ {
944
+ "type": "excluded_source",
945
+ "source": "arctic_shift",
946
+ "reason": "search API cannot keyword-search across all of Reddit "
947
+ "(per-subreddit only); deferred to v2",
948
+ "tier": "C",
949
+ },
950
+ {
951
+ "type": "excluded_source",
952
+ "source": "stack_exchange",
953
+ "reason": "v2 scope",
954
+ "tier": "C",
955
+ },
956
+ ]
957
+ if youtube_failure:
958
+ # YouTube is best-effort: the fleet-shared Google quota or an
959
+ # undeployed gateway must not fail the whole section, but the gap
960
+ # is recorded, never silent.
961
+ rows.append(
962
+ {
963
+ "type": "excluded_source",
964
+ "source": "youtube",
965
+ "reason": youtube_failure,
966
+ "tier": "C",
967
+ }
968
+ )
969
+ return rows
970
+
971
+
972
+ # ── rendering ──────────────────────────────────────────────────────────────
973
+
974
+ def _quote_cell(quote: dict[str, Any], cap: int = 200) -> str:
975
+ text = _truncate(quote["text"], cap).replace("|", "/")
976
+ if quote.get("url"):
977
+ return f"[{text}]({quote['url']})"
978
+ return text
979
+
980
+
981
+ def render_section(
982
+ *,
983
+ queries: list[str],
984
+ query_sources: dict[str, str],
985
+ product_context: list[str],
986
+ search_summaries: list[dict[str, Any]],
987
+ communities: list[dict[str, Any]],
988
+ stored_quotes: list[dict[str, Any]],
989
+ excluded_sources: list[dict[str, Any]],
990
+ totals: dict[str, Any],
991
+ retrieved_on: str,
992
+ ) -> str:
993
+ lines: list[str] = []
994
+ add = lines.append
995
+
996
+ add(
997
+ f"Public community posts and comments retrieved {retrieved_on} from "
998
+ "Hacker News (keyless Algolia search; stories and comments, last "
999
+ f"{HN_RECENCY_DAYS // 365} years), Reddit (Apify actor search "
1000
+ "across all subreddits, posts and comments from the last 12 "
1001
+ "months), and YouTube (comments under the top videos for each "
1002
+ f"query, videos from the last {YT_RECENCY_DAYS // 365} years; "
1003
+ "best-effort). Quotes are selected by a fixed marker-regex list "
1004
+ f"(v{MARKER_LIST_VERSION}: payment language, tool-seeking language, "
1005
+ "pain language, price-sensitivity language) — deterministic keyword "
1006
+ "heuristics, not comprehension, so category counts measure marker "
1007
+ "density, not verified intent. All aggregates below are computed "
1008
+ "over exactly the quote rows persisted in the evidence file; "
1009
+ "hypothesis synthesis happens downstream."
1010
+ )
1011
+ add("")
1012
+ add(
1013
+ "Queries searched: "
1014
+ + ", ".join(f"`{q}` ({query_sources.get(q, 'explicit')})" for q in queries)
1015
+ + "."
1016
+ )
1017
+ add("")
1018
+ if product_context:
1019
+ add(
1020
+ "Product-context tokens: "
1021
+ + ", ".join(f"`{t}`" for t in product_context)
1022
+ + ". A quote marked product-context mentions this product's "
1023
+ "MECHANISM; all other quotes are bare category demand that may "
1024
+ "belong to different mechanics (for example human-to-human "
1025
+ "realtime collaboration versus working with an AI agent) — do "
1026
+ "not read them as segment evidence for this product without "
1027
+ "checking the quote itself."
1028
+ )
1029
+ add("")
1030
+
1031
+ add("### Search coverage")
1032
+ add("")
1033
+ add("| Query | Source | Items retrieved | Marker-matched quotes | Stored |")
1034
+ add("| --- | --- | ---: | ---: | ---: |")
1035
+ for summary in search_summaries:
1036
+ add(
1037
+ f"| {summary['query']} | {summary['source']} | "
1038
+ f"{summary['retrieved']} | {summary['matched']} | "
1039
+ f"{summary['stored']} |"
1040
+ )
1041
+ add("")
1042
+
1043
+ if communities:
1044
+ add("### Community observations")
1045
+ add("")
1046
+ context_col = " Product-context |" if product_context else ""
1047
+ add(
1048
+ "| Community | Quotes | Payment | Seeking | Pain | Price-sens |"
1049
+ f"{context_col} Votes on quotes | Latest |"
1050
+ )
1051
+ add(
1052
+ "| --- | ---: | ---: | ---: | ---: | ---: |"
1053
+ + (" ---: |" if product_context else "")
1054
+ + " ---: | --- |"
1055
+ )
1056
+ for row in communities:
1057
+ context_cell = (
1058
+ f" {row['product_context_quotes']} |" if product_context else ""
1059
+ )
1060
+ add(
1061
+ f"| {row['community']} | {row['quotes']} | {row['wtp_quotes']} | "
1062
+ f"{row['seeking_quotes']} | {row['pain_quotes']} | "
1063
+ f"{row['price_sensitivity_quotes']} |{context_cell} "
1064
+ f"{row['votes_on_quotes']} | {row['latest_quote_date'] or '—'} |"
1065
+ )
1066
+ add("")
1067
+ add(
1068
+ "Counts describe marker matches in the stored sample, not buyer intent, "
1069
+ "relative evidence strength or a channel recommendation. A "
1070
+ "community's absence here means the searched queries surfaced "
1071
+ "nothing marker-matched — weak evidence, not proof of no "
1072
+ "demand."
1073
+ )
1074
+ add("")
1075
+
1076
+ wtp_quotes = [q for q in stored_quotes if "wtp" in q["categories"]]
1077
+ if wtp_quotes:
1078
+ add("### Payment-language quotes")
1079
+ add("")
1080
+ context_col = " Product-ctx |" if product_context else ""
1081
+ add(f"| Community | Date | Votes |{context_col} Quote |")
1082
+ add(
1083
+ "| --- | --- | ---: |"
1084
+ + (" :-: |" if product_context else "")
1085
+ + " --- |"
1086
+ )
1087
+ for quote in wtp_quotes[:6]:
1088
+ context_cell = (
1089
+ (" ✓ |" if quote.get("product_context") else " — |")
1090
+ if product_context
1091
+ else ""
1092
+ )
1093
+ add(
1094
+ f"| {quote['community']} | {quote['created'] or '—'} | "
1095
+ f"{quote['votes']} |{context_cell} {_quote_cell(quote)} |"
1096
+ )
1097
+ add("")
1098
+
1099
+ other_quotes = [
1100
+ q for q in stored_quotes if "wtp" not in q["categories"]
1101
+ ]
1102
+ if other_quotes:
1103
+ add("### Seeking, pain, and price-sensitivity quotes")
1104
+ add("")
1105
+ context_col = " Product-ctx |" if product_context else ""
1106
+ add(f"| Community | Date | Votes | Markers |{context_col} Quote |")
1107
+ add(
1108
+ "| --- | --- | ---: | --- |"
1109
+ + (" :-: |" if product_context else "")
1110
+ + " --- |"
1111
+ )
1112
+ for quote in other_quotes[:10]:
1113
+ context_cell = (
1114
+ (" ✓ |" if quote.get("product_context") else " — |")
1115
+ if product_context
1116
+ else ""
1117
+ )
1118
+ add(
1119
+ f"| {quote['community']} | {quote['created'] or '—'} | "
1120
+ f"{quote['votes']} | {', '.join(quote['categories'])} |"
1121
+ f"{context_cell} {_quote_cell(quote)} |"
1122
+ )
1123
+ add("")
1124
+
1125
+ add("### Method and coverage")
1126
+ add("")
1127
+ add(
1128
+ f"- Spend recorded: {totals['actor_runs']} Reddit actor run(s), "
1129
+ f"{totals['hn_calls']} free HN API call(s), and "
1130
+ f"{totals['youtube_calls']} metered YouTube API call(s), "
1131
+ "accumulated across attempts; cached replays add no new spend."
1132
+ )
1133
+ add(
1134
+ f"- Caps: {MAX_QUERIES} queries, {REDDIT_ITEMS_PER_QUERY} Reddit "
1135
+ f"items per query, {HN_STORY_HITS}+{HN_COMMENT_HITS} HN hits per "
1136
+ f"query, {YT_VIDEOS_PER_QUERY}×{YT_COMMENTS_PER_VIDEO} YouTube "
1137
+ f"comments per query, {STORED_QUOTES_CAP} stored quotes "
1138
+ f"({STORED_PER_COMMUNITY_CAP} per community)."
1139
+ )
1140
+ add(
1141
+ "- Excluded deterministically: Show HN/Launch HN stories and "
1142
+ "self-promo Reddit posts (sellers, not askers), jobs-thread "
1143
+ "comments and separator-formatted classifieds, known bots. A "
1144
+ f"marker counts only within {_ANCHOR_PROXIMITY_WINDOW} characters "
1145
+ "of a non-generic query token, and money mentions count as payment "
1146
+ "language only near a payment word ($100M scale-talk does not)."
1147
+ )
1148
+ add(
1149
+ "- Sources excluded from this run (recorded as excluded_source "
1150
+ "rows in the evidence file): "
1151
+ + "; ".join(f"{row['source']} — {row['reason']}" for row in excluded_sources)
1152
+ + ". Communities over-represent complainers and early adopters; "
1153
+ "marker matching misses paraphrased demand and admits some noise."
1154
+ )
1155
+ add("")
1156
+ add("### Sources")
1157
+ add("")
1158
+ add(
1159
+ f"- Hacker News via the keyless Algolia API; Reddit via Apify "
1160
+ f"`{REDDIT_ACTOR}`; YouTube via the Data API v3 (search + comment "
1161
+ f"threads) — retrieved {retrieved_on}; the stored quote rows in "
1162
+ f"`{EVIDENCE_REL.as_posix()}` are exactly the set every table "
1163
+ "above is computed from (texts truncated for storage, markers "
1164
+ "matched on full text before truncation)."
1165
+ )
1166
+ return "\n".join(lines).strip() + "\n"
1167
+
1168
+
1169
+ # ── orchestration ──────────────────────────────────────────────────────────
1170
+
1171
+ @dataclass
1172
+ class RunInputs:
1173
+ queries: list[str]
1174
+ query_sources: dict[str, str]
1175
+ product_context: list[str]
1176
+
1177
+
1178
+ def collect_youtube(client: OsClient, query: str, published_after: str) -> tuple[list[dict[str, Any]], int]:
1179
+ """One search plus comment pulls for the top videos. Raises on provider
1180
+ failure; the caller decides best-effort semantics."""
1181
+ search, _ = client.yt_call(
1182
+ "search",
1183
+ {
1184
+ "query": query,
1185
+ "max_results": YT_SEARCH_RESULTS,
1186
+ "order": "relevance",
1187
+ "published_after": published_after,
1188
+ },
1189
+ )
1190
+ videos = [v for v in (search.get("rows") or []) if isinstance(v, dict)]
1191
+ candidates: list[dict[str, Any]] = []
1192
+ retrieved = 0
1193
+ for video in videos[:YT_VIDEOS_PER_QUERY]:
1194
+ threads, _ = client.yt_call(
1195
+ "comment-threads",
1196
+ {
1197
+ "video_id": video.get("video_id"),
1198
+ "max_results": YT_COMMENTS_PER_VIDEO,
1199
+ "order": "relevance",
1200
+ },
1201
+ )
1202
+ rows = [r for r in (threads.get("rows") or []) if isinstance(r, dict)]
1203
+ retrieved += len(rows)
1204
+ candidates.extend(
1205
+ c
1206
+ for c in (normalize_youtube_comment(r, query, video) for r in rows)
1207
+ if c
1208
+ )
1209
+ return candidates, retrieved
1210
+
1211
+
1212
+ def _call_with_context(client: OsClient, query: str, label: str, fn: Callable[[], Any]) -> Any:
1213
+ """Re-raise a provider-call failure with WHICH query/call failed and how
1214
+ much of the attempt's paid budget is already spent — the raw gateway
1215
+ envelope alone doesn't say either, so a human can't tell whether a retry
1216
+ is even possible without this."""
1217
+ try:
1218
+ return fn()
1219
+ except CommunityDemandError as exc:
1220
+ raise CommunityDemandError(
1221
+ f"{label} for `{query}` failed: {exc} — this attempt had "
1222
+ f"already started {client.actor_runs}/{client.max_actor_runs} "
1223
+ "actor run(s); see GET /me/limits?resource=apify:runs for "
1224
+ "remaining daily quota."
1225
+ ) from exc
1226
+
1227
+
1228
+ def run_pipeline(client: OsClient, inputs: RunInputs) -> dict[str, Any]:
1229
+ if not inputs.queries:
1230
+ raise CommunityDemandError(
1231
+ "no seed queries available: pass --seed-keywords with 1-5 "
1232
+ "category or problem phrases derived from the research assignment (see the "
1233
+ "workflow doc), or complete the Keyword Economics section first"
1234
+ )
1235
+
1236
+ hn_cutoff_date = date.today() - timedelta(days=HN_RECENCY_DAYS)
1237
+ hn_cutoff = hn_cutoff_date.isoformat()
1238
+ # Midnight-UTC epoch: stable within a day, so same-day replays hit the
1239
+ # cache while a fresh attempt on a later day refetches.
1240
+ hn_cutoff_epoch = int(
1241
+ datetime.combine(hn_cutoff_date, datetime.min.time(), tzinfo=timezone.utc).timestamp()
1242
+ )
1243
+ yt_published_after = (
1244
+ date.today() - timedelta(days=YT_RECENCY_DAYS)
1245
+ ).isoformat() + "T00:00:00Z"
1246
+ candidates: list[dict[str, Any]] = []
1247
+ retrieval_counts: list[dict[str, Any]] = []
1248
+ youtube_failure: str | None = None
1249
+
1250
+ for query in inputs.queries:
1251
+ story_hits, _ = client.hn_search(query, "story", HN_STORY_HITS, hn_cutoff_epoch)
1252
+ comment_hits, _ = client.hn_search(query, "comment", HN_COMMENT_HITS, hn_cutoff_epoch)
1253
+ reddit_rows, _ = _call_with_context(
1254
+ client,
1255
+ query,
1256
+ "Reddit",
1257
+ lambda: client.run_actor(
1258
+ REDDIT_ACTOR,
1259
+ {
1260
+ "searches": [query],
1261
+ "searchPosts": True,
1262
+ "searchComments": True,
1263
+ "searchCommunities": False,
1264
+ "searchUsers": False,
1265
+ "skipUserPosts": True,
1266
+ "includeMediaLinks": True,
1267
+ "sort": "relevance",
1268
+ "time": "year",
1269
+ "maxItems": REDDIT_ITEMS_PER_QUERY,
1270
+ "proxy": {"useApifyProxy": True},
1271
+ },
1272
+ limit=REDDIT_ITEMS_PER_QUERY,
1273
+ ),
1274
+ )
1275
+
1276
+ hn_candidates = [
1277
+ c
1278
+ for c in (
1279
+ [normalize_hn_story(h, query) for h in story_hits]
1280
+ + [normalize_hn_comment(h, query) for h in comment_hits]
1281
+ )
1282
+ if c
1283
+ ]
1284
+ reddit_candidates = [
1285
+ c for c in (normalize_reddit_row(r, query) for r in reddit_rows) if c
1286
+ ]
1287
+ candidates.extend(hn_candidates)
1288
+ candidates.extend(reddit_candidates)
1289
+ retrieval_counts.append(
1290
+ {
1291
+ "query": query,
1292
+ "source": "hackernews",
1293
+ "retrieved": len(story_hits) + len(comment_hits),
1294
+ "normalized": len(hn_candidates),
1295
+ }
1296
+ )
1297
+ retrieval_counts.append(
1298
+ {
1299
+ "query": query,
1300
+ "source": "reddit",
1301
+ "retrieved": len(reddit_rows),
1302
+ "normalized": len(reddit_candidates),
1303
+ }
1304
+ )
1305
+ if youtube_failure is None:
1306
+ try:
1307
+ yt_candidates, yt_retrieved = collect_youtube(
1308
+ client, query, yt_published_after
1309
+ )
1310
+ except CommunityDemandError as exc:
1311
+ # Best-effort: an undeployed provider or exhausted
1312
+ # fleet-shared quota degrades to a recorded exclusion, never
1313
+ # a silent gap and never a failed section.
1314
+ youtube_failure = f"unavailable during this run: {str(exc)[:200]}"
1315
+ else:
1316
+ candidates.extend(yt_candidates)
1317
+ retrieval_counts.append(
1318
+ {
1319
+ "query": query,
1320
+ "source": "youtube",
1321
+ "retrieved": yt_retrieved,
1322
+ "normalized": len(yt_candidates),
1323
+ }
1324
+ )
1325
+
1326
+ # The same item can match several queries: first query wins, once.
1327
+ seen_ids: set[str] = set()
1328
+ quotes: list[dict[str, Any]] = []
1329
+ for candidate in candidates:
1330
+ if candidate["id"] in seen_ids:
1331
+ continue
1332
+ seen_ids.add(candidate["id"])
1333
+ quote = to_quote_row(candidate, hn_cutoff, inputs.product_context)
1334
+ if quote:
1335
+ quotes.append(quote)
1336
+
1337
+ stored_quotes = select_stored_quotes(quotes)
1338
+ communities = summarize_communities(stored_quotes)
1339
+
1340
+ search_summaries = [
1341
+ {
1342
+ "type": "search_summary",
1343
+ "query": count["query"],
1344
+ "source": count["source"],
1345
+ "retrieved": count["retrieved"],
1346
+ "matched": sum(
1347
+ 1
1348
+ for q in quotes
1349
+ if q["query"] == count["query"] and q["source"] == count["source"]
1350
+ ),
1351
+ "stored": sum(
1352
+ 1
1353
+ for q in stored_quotes
1354
+ if q["query"] == count["query"] and q["source"] == count["source"]
1355
+ ),
1356
+ "tier": "A",
1357
+ }
1358
+ for count in retrieval_counts
1359
+ ]
1360
+
1361
+ totals = client.ledger_totals()
1362
+ retrieved_on = (
1363
+ totals["first_retrieved_on"]
1364
+ if totals["first_retrieved_on"] == totals["last_retrieved_on"]
1365
+ else f"{totals['first_retrieved_on']}..{totals['last_retrieved_on']}"
1366
+ )
1367
+
1368
+ excluded_sources = excluded_source_rows(youtube_failure)
1369
+ evidence = {
1370
+ "schema_version": SCHEMA_VERSION,
1371
+ "section_id": SECTION_ID,
1372
+ "generated_by": Path(__file__).name,
1373
+ "retrieved_on": retrieved_on,
1374
+ "marker_list_version": MARKER_LIST_VERSION,
1375
+ "inputs": {
1376
+ "queries": inputs.queries,
1377
+ "query_sources": inputs.query_sources,
1378
+ "product_context": inputs.product_context,
1379
+ },
1380
+ "rows": (
1381
+ search_summaries
1382
+ + communities
1383
+ + stored_quotes
1384
+ + excluded_sources
1385
+ ),
1386
+ "totals": {
1387
+ "actor_runs": totals["actor_runs"],
1388
+ "provider_calls": totals["provider_calls"],
1389
+ "hn_calls": totals["hn_calls"],
1390
+ "youtube_calls": totals["youtube_calls"],
1391
+ },
1392
+ }
1393
+
1394
+ section_markdown = render_section(
1395
+ queries=inputs.queries,
1396
+ query_sources=inputs.query_sources,
1397
+ product_context=inputs.product_context,
1398
+ search_summaries=search_summaries,
1399
+ communities=communities,
1400
+ stored_quotes=stored_quotes,
1401
+ excluded_sources=excluded_sources,
1402
+ totals=totals,
1403
+ retrieved_on=retrieved_on,
1404
+ )
1405
+ return {
1406
+ "evidence": evidence,
1407
+ "section_markdown": section_markdown,
1408
+ "totals": totals,
1409
+ "attempt": {
1410
+ "actor_runs": client.actor_runs,
1411
+ "hn_calls": client.hn_calls,
1412
+ "yt_calls": client.yt_calls,
1413
+ "cache_hits": client.cache_hits,
1414
+ },
1415
+ }
1416
+
1417
+
1418
+ def _write_atomic(path: Path, content: str) -> None:
1419
+ path.parent.mkdir(parents=True, exist_ok=True)
1420
+ tmp = path.with_suffix(path.suffix + ".tmp")
1421
+ tmp.write_text(content, encoding="utf-8")
1422
+ tmp.replace(path)
1423
+
1424
+
1425
+ def _canonicalize_legacy_headings(upsert: ModuleType, report_path: Path) -> None:
1426
+ """Fold pre-rename legacy H2s of this section into the canonical heading.
1427
+
1428
+ A report materialized by an older bundle keeps `## Community demand`; the
1429
+ PRD extractor and this runner's upsert address the canonical H2 only, so a
1430
+ stale legacy heading would either fail extraction or make the section
1431
+ coexist under two names after a rerun. Rename the legacy H2 in place, or
1432
+ drop the stale legacy section when a canonical one already exists.
1433
+ """
1434
+ if not report_path.is_file():
1435
+ return
1436
+ text = report_path.read_text(encoding="utf-8")
1437
+ changed = False
1438
+ for legacy in LEGACY_SECTION_HEADINGS:
1439
+ while True:
1440
+ offsets = upsert._h2_offsets(text)
1441
+ match = next(
1442
+ (
1443
+ (index, offset)
1444
+ for index, (name, offset) in enumerate(offsets)
1445
+ if name == legacy
1446
+ ),
1447
+ None,
1448
+ )
1449
+ if match is None:
1450
+ break
1451
+ canonical_present = any(
1452
+ name == SECTION_HEADING for name, _ in offsets
1453
+ )
1454
+ index, start = match
1455
+ if canonical_present:
1456
+ end = (
1457
+ offsets[index + 1][1]
1458
+ if index + 1 < len(offsets)
1459
+ else len(text)
1460
+ )
1461
+ text = text[:start] + text[end:]
1462
+ else:
1463
+ line_end = text.find("\n", start)
1464
+ if line_end == -1:
1465
+ line_end = len(text)
1466
+ text = text[:start] + f"## {SECTION_HEADING}" + text[line_end:]
1467
+ changed = True
1468
+ if changed:
1469
+ _write_atomic(report_path, text)
1470
+
1471
+
1472
+ def run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
1473
+ workspace = workspace.resolve()
1474
+ io = _load_script("research-artifact-io.py", "pmr_artifact_io")
1475
+ with io.section_attempt(workspace, SECTION_ID, context_file=getattr(args, "context_file", None)):
1476
+ return _run(workspace, args)
1477
+
1478
+
1479
+ def _run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
1480
+ workspace = workspace.resolve()
1481
+ runtime = workspace / RUNTIME_REL
1482
+ cache_dir = runtime / "cache"
1483
+ runtime.mkdir(parents=True, exist_ok=True)
1484
+
1485
+ api_key = (os.environ.get("OS_API_KEY") or "").strip()
1486
+ if not api_key:
1487
+ raise CommunityDemandError("OS_API_KEY is required")
1488
+ os_url = (os.environ.get("OS_URL") or DEFAULT_OS_URL).strip() or DEFAULT_OS_URL
1489
+
1490
+ explicit = parse_terms(args.seed_keywords or "", MAX_QUERIES)
1491
+ discovered = queries_from_keyword_evidence(workspace, MAX_QUERIES)
1492
+ queries, query_sources = select_queries(explicit, discovered, args.max_queries)
1493
+ # One actor run and two HN calls per query: never admit more queries
1494
+ # than the required budgets can finish in one attempt (YouTube is
1495
+ # best-effort and merely stops when its own budget runs out).
1496
+ queries = queries[: max(1, min(args.max_actor_runs, args.max_hn_calls // 2))]
1497
+ query_sources = {q: query_sources[q] for q in queries}
1498
+
1499
+ inputs = RunInputs(
1500
+ queries=queries,
1501
+ query_sources=query_sources,
1502
+ product_context=parse_context_tokens(args.product_context or ""),
1503
+ )
1504
+ client = OsClient(
1505
+ os_url=os_url,
1506
+ api_key=api_key,
1507
+ cache_dir=cache_dir,
1508
+ max_actor_runs=args.max_actor_runs,
1509
+ max_hn_calls=args.max_hn_calls,
1510
+ max_yt_calls=args.max_yt_calls,
1511
+ )
1512
+
1513
+ outcome = run_pipeline(client, inputs)
1514
+
1515
+ evidence_path = workspace / EVIDENCE_REL
1516
+ evidence_json = json.dumps(outcome["evidence"], indent=1, ensure_ascii=False)
1517
+ _write_atomic(evidence_path, evidence_json)
1518
+
1519
+ section_file = runtime / "result.md"
1520
+ _write_atomic(section_file, outcome["section_markdown"])
1521
+
1522
+ upsert = _load_script("upsert-markdown-section.py", "pmr_upsert_markdown_section")
1523
+ report_path = workspace / REPORT_REL
1524
+ report_path.parent.mkdir(parents=True, exist_ok=True)
1525
+ _canonicalize_legacy_headings(upsert, report_path)
1526
+ upsert.upsert_markdown_section(report_path, SECTION_HEADING, outcome["section_markdown"])
1527
+
1528
+ persist = _load_script("persist-research-report.py", "pmr_persist_research_report")
1529
+ persistence = persist.persist_research_report(
1530
+ workspace,
1531
+ commit_message=f"research: update {SECTION_HEADING.lower()} section",
1532
+ extra_paths=(Path(_artifact_path("egbe-docs/research/market/evidence/community-demand.json")),),
1533
+ )
1534
+
1535
+ state = {
1536
+ "schema_version": SCHEMA_VERSION,
1537
+ "section_id": SECTION_ID,
1538
+ "status": "completed",
1539
+ "queries": inputs.queries,
1540
+ "query_sources": inputs.query_sources,
1541
+ "product_context": inputs.product_context,
1542
+ "evidence_path": EVIDENCE_REL.as_posix(),
1543
+ "evidence_sha256": hashlib.sha256(evidence_json.encode("utf-8")).hexdigest(),
1544
+ "result_path": (RUNTIME_REL / "result.md").as_posix(),
1545
+ "result_sha256": hashlib.sha256(outcome["section_markdown"].encode("utf-8")).hexdigest(),
1546
+ "provider_calls": outcome["totals"]["provider_calls"],
1547
+ "actor_runs": outcome["totals"]["actor_runs"],
1548
+ "hn_calls": outcome["totals"]["hn_calls"],
1549
+ "youtube_calls": outcome["totals"]["youtube_calls"],
1550
+ "actor_starts_recorded": outcome["totals"].get("actor_starts_recorded", 0),
1551
+ "attempt_actor_runs": outcome["attempt"]["actor_runs"],
1552
+ "attempt_hn_calls": outcome["attempt"]["hn_calls"],
1553
+ "attempt_yt_calls": outcome["attempt"]["yt_calls"],
1554
+ "attempt_cache_hits": outcome["attempt"]["cache_hits"],
1555
+ "report_commit_sha": persistence.commit_sha,
1556
+ "publication_mode": persistence.publication_mode,
1557
+ "research_context": _load_script("research-artifact-io.py", "pmr_input_io").context_binding(workspace, SECTION_ID),
1558
+ "error": None,
1559
+ }
1560
+ _write_atomic(runtime / "request.json", json.dumps(state, indent=1))
1561
+ return state
1562
+
1563
+
1564
+ def main() -> int:
1565
+ parser = argparse.ArgumentParser(description=__doc__)
1566
+ parser.add_argument("--context-file", type=Path, help="Explicit context for this study; no project files are discovered implicitly.")
1567
+ parser.add_argument("--workspace", type=Path, required=True)
1568
+ parser.add_argument(
1569
+ "--seed-keywords",
1570
+ default="",
1571
+ help="Comma-separated category AND problem phrases derived from "
1572
+ "the research assignment; merged ahead of the top keywords discovered by Keyword "
1573
+ "Economics.",
1574
+ )
1575
+ parser.add_argument(
1576
+ "--product-context",
1577
+ default="",
1578
+ help="Comma-separated mechanism tokens from the research assignment (e.g. 'ai, "
1579
+ "agent'); quotes mentioning them are flagged as speaking to THIS "
1580
+ "product's mechanism rather than the bare category.",
1581
+ )
1582
+ parser.add_argument("--max-queries", type=int, default=MAX_QUERIES)
1583
+ parser.add_argument("--max-actor-runs", type=int, default=MAX_ACTOR_RUNS_PER_ATTEMPT)
1584
+ parser.add_argument("--max-hn-calls", type=int, default=MAX_HN_CALLS_PER_ATTEMPT)
1585
+ parser.add_argument("--max-yt-calls", type=int, default=MAX_YT_CALLS_PER_ATTEMPT)
1586
+ args = parser.parse_args()
1587
+
1588
+ try:
1589
+ state = run(args.workspace, args)
1590
+ except (CommunityDemandError, ValueError) as exc:
1591
+ print(f"{DISPLAY_NAME}: {exc}", file=sys.stderr)
1592
+ return 1
1593
+ print(
1594
+ f"{DISPLAY_NAME}: completed — {state['actor_runs']} actor run(s), "
1595
+ f"{state['hn_calls']} HN call(s), {state['youtube_calls']} YouTube "
1596
+ f"call(s), evidence at {state['evidence_path']}, report commit "
1597
+ f"{state['report_commit_sha'][:12]}"
1598
+ )
1599
+ return 0
1600
+
1601
+
1602
+ if __name__ == "__main__":
1603
+ raise SystemExit(main())