@egbe-ai/opencode 0.0.0-stage → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (479) hide show
  1. package/LICENSE +4 -0
  2. package/README.md +124 -2
  3. package/index.js +5 -0
  4. package/package.json +55 -4
  5. package/plugin.js +230 -0
  6. package/session-log.js +315 -0
  7. package/skills/.egbe-catalog/manifest.json +1 -0
  8. package/skills/egbe-os-analytics/SKILL.md +499 -0
  9. package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
  10. package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
  11. package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
  12. package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
  13. package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
  14. package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
  15. package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
  16. package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
  17. package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
  18. package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
  19. package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
  20. package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
  21. package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
  22. package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
  23. package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
  24. package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
  25. package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
  26. package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
  27. package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
  28. package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
  29. package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
  30. package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
  31. package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
  32. package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
  33. package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
  34. package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
  35. package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
  36. package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
  37. package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
  38. package/skills/egbe-os-db/SKILL.md +174 -0
  39. package/skills/egbe-os-deploy/SKILL.md +412 -0
  40. package/skills/egbe-os-design/SKILL.md +182 -0
  41. package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
  42. package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
  43. package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
  44. package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
  45. package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
  46. package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
  47. package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
  48. package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
  49. package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
  50. package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
  51. package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
  52. package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
  53. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
  54. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
  55. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
  56. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
  57. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
  58. package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
  59. package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
  60. package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
  61. package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
  62. package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
  63. package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
  64. package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
  65. package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
  66. package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
  67. package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
  68. package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
  69. package/skills/egbe-os-destination-readiness/references/report.md +116 -0
  70. package/skills/egbe-os-domains/SKILL.md +189 -0
  71. package/skills/egbe-os-errors/SKILL.md +259 -0
  72. package/skills/egbe-os-git/SKILL.md +275 -0
  73. package/skills/egbe-os-git/references/issues.md +33 -0
  74. package/skills/egbe-os-google-ads/SKILL.md +163 -0
  75. package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
  76. package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
  77. package/skills/egbe-os-google-ads/references/conversions.md +153 -0
  78. package/skills/egbe-os-google-ads/references/delivery.md +128 -0
  79. package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
  80. package/skills/egbe-os-google-ads/references/reporting.md +105 -0
  81. package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
  82. package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
  83. package/skills/egbe-os-google-auth/SKILL.md +285 -0
  84. package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
  85. package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
  86. package/skills/egbe-os-images/SKILL.md +163 -0
  87. package/skills/egbe-os-images/references/stock-images.md +21 -0
  88. package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
  89. package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
  90. package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
  91. package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
  92. package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
  93. package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
  94. package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
  95. package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
  96. package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
  97. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
  98. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
  99. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
  100. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
  101. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
  102. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
  103. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
  104. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
  105. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
  106. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
  107. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
  108. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
  109. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
  110. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
  111. package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
  112. package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
  113. package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
  114. package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
  115. package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
  116. package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
  117. package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
  118. package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
  119. package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
  120. package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
  121. package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
  122. package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
  123. package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
  124. package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
  125. package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
  126. package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
  127. package/skills/egbe-os-llm/SKILL.md +160 -0
  128. package/skills/egbe-os-mail/SKILL.md +180 -0
  129. package/skills/egbe-os-me/SKILL.md +119 -0
  130. package/skills/egbe-os-meta-ads/SKILL.md +114 -0
  131. package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
  132. package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
  133. package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
  134. package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
  135. package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
  136. package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
  137. package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
  138. package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
  139. package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
  140. package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
  141. package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
  142. package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
  143. package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
  144. package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
  145. package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
  146. package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
  147. package/skills/egbe-os-notifications/SKILL.md +78 -0
  148. package/skills/egbe-os-openai-ads/SKILL.md +149 -0
  149. package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
  150. package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
  151. package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
  152. package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
  153. package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
  154. package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
  155. package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
  156. package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
  157. package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
  158. package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
  159. package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
  160. package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
  161. package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
  162. package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
  163. package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
  164. package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
  165. package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
  166. package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
  167. package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
  168. package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
  169. package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
  170. package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
  171. package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
  172. package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
  173. package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
  174. package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
  175. package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
  176. package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
  177. package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
  178. package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
  179. package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
  180. package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
  181. package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
  182. package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
  183. package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
  184. package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
  185. package/skills/egbe-os-payments/SKILL.md +215 -0
  186. package/skills/egbe-os-payments/agents/openai.yaml +4 -0
  187. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
  188. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
  189. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
  190. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
  191. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
  192. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
  193. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
  194. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
  195. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
  196. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
  197. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
  198. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
  199. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
  200. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
  201. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
  202. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
  203. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
  204. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
  205. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
  206. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
  207. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
  208. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
  209. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
  210. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
  211. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
  212. package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
  213. package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
  214. package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
  215. package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
  216. package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
  217. package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
  218. package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
  219. package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
  220. package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
  221. package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
  222. package/skills/egbe-os-playbook/SKILL.md +79 -0
  223. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
  224. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
  225. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
  226. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
  227. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
  228. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
  229. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
  230. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
  231. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
  232. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
  233. package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
  234. package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
  235. package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
  236. package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
  237. package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
  238. package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
  239. package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
  240. package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
  241. package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
  242. package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
  243. package/skills/egbe-os-playbook/playbooks/product.md +67 -0
  244. package/skills/egbe-os-playbook/references/catalog.md +127 -0
  245. package/skills/egbe-os-playbook/references/changes.md +167 -0
  246. package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
  247. package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
  248. package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
  249. package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
  250. package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
  251. package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
  252. package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
  253. package/skills/egbe-os-product-auth/SKILL.md +151 -0
  254. package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
  255. package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
  256. package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
  257. package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
  258. package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
  259. package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
  260. package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
  261. package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
  262. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
  263. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
  264. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
  265. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
  266. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
  267. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
  268. package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
  269. package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
  270. package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
  271. package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
  272. package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
  273. package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
  274. package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
  275. package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
  276. package/skills/egbe-os-product-build/SKILL.md +111 -0
  277. package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
  278. package/skills/egbe-os-product-build/references/capabilities.md +25 -0
  279. package/skills/egbe-os-product-build/references/engineering.md +72 -0
  280. package/skills/egbe-os-product-build/references/execution.md +55 -0
  281. package/skills/egbe-os-product-build/references/foundations.md +93 -0
  282. package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
  283. package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
  284. package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
  285. package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
  286. package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
  287. package/skills/egbe-os-product-market-research/SKILL.md +159 -0
  288. package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
  289. package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
  290. package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
  291. package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
  292. package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
  293. package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
  294. package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
  295. package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
  296. package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
  297. package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
  298. package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
  299. package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
  300. package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
  301. package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
  302. package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
  303. package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
  304. package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
  305. package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
  306. package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
  307. package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
  308. package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
  309. package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
  310. package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
  311. package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
  312. package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
  313. package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
  314. package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
  315. package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
  316. package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
  317. package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
  318. package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
  319. package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
  320. package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
  321. package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
  322. package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
  323. package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
  324. package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
  325. package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
  326. package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
  327. package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
  328. package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
  329. package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
  330. package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
  331. package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
  332. package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
  333. package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
  334. package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
  335. package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
  336. package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
  337. package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
  338. package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
  339. package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
  340. package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
  341. package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
  342. package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
  343. package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
  344. package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
  345. package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
  346. package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
  347. package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
  348. package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
  349. package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
  350. package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
  351. package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
  352. package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
  353. package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
  354. package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
  355. package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
  356. package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
  357. package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
  358. package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
  359. package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
  360. package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
  361. package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
  362. package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
  363. package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
  364. package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
  365. package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
  366. package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
  367. package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
  368. package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
  369. package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
  370. package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
  371. package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
  372. package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
  373. package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
  374. package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
  375. package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
  376. package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
  377. package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
  378. package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
  379. package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
  380. package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
  381. package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
  382. package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
  383. package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
  384. package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
  385. package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
  386. package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
  387. package/skills/egbe-os-product-report/SKILL.md +150 -0
  388. package/skills/egbe-os-product-report/references/artifacts.md +84 -0
  389. package/skills/egbe-os-product-report/references/brief.md +34 -0
  390. package/skills/egbe-os-product-report/references/daily.md +76 -0
  391. package/skills/egbe-os-product-report/references/sources.md +68 -0
  392. package/skills/egbe-os-product-routines/SKILL.md +166 -0
  393. package/skills/egbe-os-product-routines/references/catalog.md +22 -0
  394. package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
  395. package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
  396. package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
  397. package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
  398. package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
  399. package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
  400. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
  401. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
  402. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
  403. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
  404. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
  405. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
  406. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
  407. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
  408. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
  409. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
  410. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
  411. package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
  412. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
  413. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
  414. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
  415. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
  416. package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
  417. package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
  418. package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
  419. package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
  420. package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
  421. package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
  422. package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
  423. package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
  424. package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
  425. package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
  426. package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
  427. package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
  428. package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
  429. package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
  430. package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
  431. package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
  432. package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
  433. package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
  434. package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
  435. package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
  436. package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
  437. package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
  438. package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
  439. package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
  440. package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
  441. package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
  442. package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
  443. package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
  444. package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
  445. package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
  446. package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
  447. package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
  448. package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
  449. package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
  450. package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
  451. package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
  452. package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
  453. package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
  454. package/skills/egbe-os-report-ui/SKILL.md +133 -0
  455. package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
  456. package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
  457. package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
  458. package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
  459. package/skills/egbe-os-report-ui/references/components.md +209 -0
  460. package/skills/egbe-os-research/SKILL.md +125 -0
  461. package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
  462. package/skills/egbe-os-scrape/SKILL.md +160 -0
  463. package/skills/egbe-os-search/SKILL.md +182 -0
  464. package/skills/egbe-os-search/references/google-places.md +81 -0
  465. package/skills/egbe-os-support/SKILL.md +149 -0
  466. package/skills/egbe-os-telemetry/SKILL.md +344 -0
  467. package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
  468. package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
  469. package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
  470. package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
  471. package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
  472. package/skills/egbe-os-tools/SKILL.md +180 -0
  473. package/skills/egbe-os-tools/references/theona.md +26 -0
  474. package/skills/egbe-os-wallet/SKILL.md +181 -0
  475. package/skills/egbe-setup/EGBE.md +484 -0
  476. package/skills/egbe-setup/SKILL.md +116 -0
  477. package/skills/egbe-setup/references/project.md +80 -0
  478. package/state.js +37 -0
  479. package/update.js +361 -0
@@ -0,0 +1,1443 @@
1
+ #!/usr/bin/env python3
2
+ """Run the deterministic Keyword Economics section against OS DataForSEO tools.
3
+
4
+ Unlike the OS Research sections, this runner performs NO free-form web
5
+ research: every number comes from a metered DataForSEO call through the OS
6
+ tools passthrough (`POST {OS_URL}/tools/dataforseo/<op>`), and the section
7
+ markdown plus a machine-readable evidence file are rendered deterministically
8
+ from those responses. The executing agent supplies only candidate seed
9
+ keywords (validated by auction data — a hallucinated seed returns zero volume
10
+ and is dropped); it never writes section prose.
11
+
12
+ Pipeline: seeds -> expansion (keywords_for_keywords + search_volume) ->
13
+ commercial filter -> SERP on top heads (who ranks, who pays) -> competitor
14
+ confirmation -> ranked-keywords enrichment per competitor -> test-budget
15
+ simulation (ad_traffic_by_keywords) -> evidence.json + report section.
16
+
17
+ Every upstream response is cached under the runtime directory keyed by request
18
+ hash, so re-runs replay from cache instead of re-spending the shared deposit.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import sys as _artifact_sys
24
+ from pathlib import Path as _ArtifactPath
25
+ _artifact_sys.path.insert(0, str(_ArtifactPath(__file__).resolve().parent))
26
+ from egbe_artifacts import artifact_path as _artifact_path
27
+
28
+ import runpy as _egbe_runpy
29
+ from pathlib import Path as _EgbePath
30
+ _egbe_adapter = _EgbePath(__file__).with_name("connection_identity.py")
31
+ _connection_headers = _egbe_runpy.run_path(str(_egbe_adapter))["connection_headers"] if _egbe_adapter.is_file() else lambda _url: {}
32
+
33
+ import argparse
34
+ import hashlib
35
+ import importlib.util
36
+ import json
37
+ import math
38
+ import os
39
+ import re
40
+ import sys
41
+ import time
42
+ import urllib.error
43
+ import urllib.request
44
+ from dataclasses import dataclass, field
45
+ from datetime import date
46
+ from pathlib import Path
47
+ from types import ModuleType
48
+ from typing import Any, Callable
49
+ from urllib.parse import urlparse
50
+
51
+ SCHEMA_VERSION = 1
52
+ SECTION_ID = "keyword_economics"
53
+ SECTION_HEADING = "Keyword economics"
54
+ DISPLAY_NAME = "Keyword Economics"
55
+
56
+ DEFAULT_OS_URL = "https://os.hz.egbe.dev"
57
+ RUNTIME_REL = Path(_artifact_path("egbe-docs/research/market/work")) / SECTION_ID
58
+ EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/keyword-economics.json"))
59
+ REPORT_REL = Path(_artifact_path("egbe-docs/research/market/full-report.md"))
60
+
61
+ # Hard ceiling on metered upstream tasks per run (cache hits are free). The
62
+ # most expensive task is $0.09, so this bounds one run at ~$2.25 worst case;
63
+ # a normal run uses 8-16 tasks (~$0.50).
64
+ MAX_TASKS_PER_RUN = 25
65
+
66
+ # DataForSEO rate-limits google_ads live endpoints at 12 requests/minute and
67
+ # answers bursts with task status 50301 "Too many requests". Failed tasks are
68
+ # not billed, so a bounded backoff retry is safe; the window clears within a
69
+ # minute.
70
+ # Also used for structured internal errors explicitly confirmed as unpaid.
71
+ PROVIDER_MAX_RETRIES = 3
72
+ PROVIDER_BACKOFF_SECONDS = (20.0, 40.0, 60.0)
73
+ MAX_ERROR_RESPONSE_BYTES = 64 * 1024
74
+
75
+ MAX_SEED_KEYWORDS = 20
76
+ MAX_SEED_DOMAINS = 10
77
+ SERP_HEADS = 8
78
+ SERP_DEPTH = 20
79
+ CONFIRM_COMPETITORS = 4
80
+ RANKED_KEYWORDS_LIMIT = 200
81
+ BUDGET_SIM_KEYWORDS = 1000
82
+ TOP_TABLE_ROWS = 20
83
+
84
+ # Keywords containing one of these tokens carry commercial/search-to-buy
85
+ # intent and get priority in head selection and the report tables.
86
+ INTENT_TOKENS = {
87
+ "software", "tool", "tools", "app", "apps", "platform", "service",
88
+ "services", "pricing", "price", "cost", "best", "top", "alternative",
89
+ "alternatives", "vs", "generator", "builder", "maker", "for", "buy",
90
+ "online", "ai",
91
+ }
92
+
93
+ # Queries carrying these tokens signal free/education intent: real demand,
94
+ # but poor targets for PAID acquisition simulation, and their ad-free SERPs
95
+ # are partially self-explanatory.
96
+ FREE_EDU_TOKENS = {
97
+ "free", "teacher", "teachers", "teaching", "classroom", "student",
98
+ "students", "school", "kids",
99
+ }
100
+
101
+ # Lexical evidence, not a verdict that the query fits a particular offer.
102
+ # Google Ads compares these signals with the actual product and campaign goal.
103
+ VENDOR_TERMS = {"agency", "agencies", "consultant", "consultants", "consulting", "company", "companies", "services", "service", "hire"}
104
+ TOOL_TERMS = {"tool", "tools", "checker", "tracker", "software", "platform", "app", "apps"}
105
+ INFORMATION_TERMS = {"what", "how", "why", "meaning", "definition", "guide", "tutorial"}
106
+ JOB_TERMS = {"job", "jobs", "career", "careers", "salary", "salaries", "hiring"}
107
+
108
+ # SERP domains that are platforms/media, not product competitors.
109
+ NOISE_DOMAINS = {
110
+ "google.com", "youtube.com", "reddit.com", "facebook.com",
111
+ "instagram.com", "wikipedia.org", "amazon.com", "pinterest.com",
112
+ "linkedin.com", "twitter.com", "x.com", "tiktok.com", "quora.com",
113
+ "medium.com", "etsy.com", "ebay.com", "apple.com", "microsoft.com",
114
+ "yelp.com", "github.com", "canva.com", "adobe.com",
115
+ }
116
+
117
+ # Function words never anchor topicality: a competitor keyword sharing ONLY
118
+ # one of these with the seed market (e.g. "from", "list", "and") must not
119
+ # merge into the demand pool. Found live: "best flavor wings from wingstop"
120
+ # merged into a meal-planning market via "from".
121
+ FUNCTION_TOKENS = {
122
+ "the", "and", "from", "with", "your", "our", "you", "what", "when",
123
+ "where", "how", "why", "this", "that", "near", "name", "list", "get",
124
+ "make", "made", "does", "can", "will", "are", "was", "has", "have",
125
+ "free", "new", "without", "using", "use",
126
+ }
127
+
128
+ # Review/listing directories: strong demand evidence, but not competitors.
129
+ DIRECTORY_DOMAINS = {
130
+ "g2.com", "capterra.com", "getapp.com", "softwareadvice.com",
131
+ "trustpilot.com", "producthunt.com", "sourceforge.net", "slashdot.org",
132
+ "alternativeto.net", "trustradius.com",
133
+ }
134
+
135
+
136
+ class KeywordEconomicsError(RuntimeError):
137
+ """Terminal runner failure; runtime state is preserved for resume."""
138
+
139
+
140
+ class DataForSeoError(KeywordEconomicsError):
141
+ """Structured provider failure preserved through the OS tools envelope."""
142
+
143
+ def __init__(self, operation: str, http_status: int, detail: dict[str, Any]):
144
+ self.http_status = http_status
145
+ self.detail = detail
146
+ super().__init__(
147
+ f"OS tools/dataforseo/{operation} failed: HTTP {http_status} "
148
+ f"{detail.get('error')} status={detail.get('status_code')} "
149
+ f"task={detail.get('task_id') or 'unknown'}: "
150
+ f"{str(detail.get('message') or '')[:500]}"
151
+ )
152
+
153
+
154
+ def _is_rate_limited(exc: Exception) -> bool:
155
+ # 50301 also wraps invalid Google Ads input; retrying that cannot recover.
156
+ message = str(exc.detail.get("message") or "") if isinstance(exc, DataForSeoError) else str(exc)
157
+ if "invalid argument" in message.lower():
158
+ return False
159
+ if isinstance(exc, DataForSeoError):
160
+ return exc.detail.get("status_code") == 50301
161
+ text = str(exc)
162
+ return "50301" in text or "Too many requests" in text
163
+
164
+
165
+ def _is_unpaid_internal_error(exc: Exception) -> bool:
166
+ if not isinstance(exc, DataForSeoError):
167
+ return False
168
+ detail = exc.detail
169
+ return (
170
+ exc.http_status == 503
171
+ and detail.get("error") == "dataforseo_upstream_internal_error"
172
+ and detail.get("status_code") == 50000
173
+ and detail.get("retryable") is True
174
+ and all(type(detail.get(key)) in (int, float) and detail[key] == 0
175
+ for key in ("cost", "task_cost"))
176
+ )
177
+
178
+
179
+ # ── bundled-script loading (same pattern as the other section runners) ─────
180
+
181
+ def _load_script(filename: str, module_name: str) -> ModuleType:
182
+ path = Path(__file__).resolve().parent / filename
183
+ spec = importlib.util.spec_from_file_location(module_name, path)
184
+ if spec is None or spec.loader is None:
185
+ raise KeywordEconomicsError(f"cannot load bundled script {filename}")
186
+ module = importlib.util.module_from_spec(spec)
187
+ sys.modules[spec.name] = module
188
+ previous = sys.dont_write_bytecode
189
+ sys.dont_write_bytecode = True
190
+ try:
191
+ spec.loader.exec_module(module)
192
+ finally:
193
+ sys.dont_write_bytecode = previous
194
+ return module
195
+
196
+
197
+ # ── OS tools client with response cache and task budget ───────────────────
198
+
199
+ @dataclass
200
+ class ToolsClient:
201
+ os_url: str
202
+ api_key: str
203
+ cache_dir: Path
204
+ max_tasks: int = MAX_TASKS_PER_RUN
205
+ tasks_spent: int = 0
206
+ estimated_cost_usd: float = 0.0
207
+ cache_hits: int = 0
208
+ request: Callable[[str, dict[str, Any]], dict[str, Any]] | None = None
209
+ sleep: Callable[[float], None] | None = None
210
+ _ledger_entries: set[Path] = field(default_factory=set, init=False, repr=False)
211
+
212
+ def _http(self, operation: str, payload: dict[str, Any]) -> dict[str, Any]:
213
+ url = f"{self.os_url.rstrip('/')}/tools/dataforseo/{operation}"
214
+ body = json.dumps(payload).encode("utf-8")
215
+ req = urllib.request.Request(
216
+ url,
217
+ data=body,
218
+ headers={
219
+ "Authorization": f"Bearer {self.api_key}",
220
+ "Content-Type": "application/json",
221
+ "Accept": "application/json",
222
+ },
223
+ method="POST",
224
+ )
225
+ try:
226
+ for _name, _value in _connection_headers(req.full_url).items():
227
+ req.add_unredirected_header(_name, _value)
228
+ with urllib.request.urlopen(req, timeout=180) as resp:
229
+ envelope = json.loads(resp.read().decode("utf-8"))
230
+ except urllib.error.HTTPError as exc:
231
+ raw = exc.read(MAX_ERROR_RESPONSE_BYTES + 1)
232
+ detail = raw.decode("utf-8", errors="replace")
233
+ if len(raw) <= MAX_ERROR_RESPONSE_BYTES:
234
+ try:
235
+ envelope = json.loads(detail)
236
+ except ValueError:
237
+ envelope = None
238
+ error = envelope.get("data") if isinstance(envelope, dict) else None
239
+ if isinstance(error, dict) and str(error.get("error", "")).startswith("dataforseo_"):
240
+ raise DataForSeoError(operation, exc.code, error) from exc
241
+ raise KeywordEconomicsError(
242
+ f"OS tools/dataforseo/{operation} failed: HTTP {exc.code} {detail[:500]}"
243
+ ) from exc
244
+ except urllib.error.URLError as exc:
245
+ raise KeywordEconomicsError(
246
+ f"OS tools/dataforseo/{operation} unreachable: {exc.reason}"
247
+ ) from exc
248
+ data = envelope.get("data")
249
+ if envelope.get("code") not in (None, 200) or not isinstance(data, dict):
250
+ raise KeywordEconomicsError(
251
+ f"OS tools/dataforseo/{operation} returned no data: "
252
+ f"{str(envelope)[:300]}"
253
+ )
254
+ return data
255
+
256
+ def call(self, operation: str, payload: dict[str, Any]) -> dict[str, Any]:
257
+ key_raw = json.dumps({"op": operation, "payload": payload}, sort_keys=True)
258
+ digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
259
+ cache_file = self.cache_dir / f"{digest}.json"
260
+ if cache_file.is_file():
261
+ cached = json.loads(cache_file.read_text(encoding="utf-8"))
262
+ self.cache_hits += 1
263
+ self._ledger_entries.add(cache_file)
264
+ return cached["response"]
265
+
266
+ if self.tasks_spent >= self.max_tasks:
267
+ raise KeywordEconomicsError(
268
+ f"task budget exhausted ({self.max_tasks} metered calls); "
269
+ "narrow the seeds or raise --max-tasks"
270
+ )
271
+ transport = self.request or self._http
272
+ attempt = 0
273
+ while True:
274
+ try:
275
+ response = transport(operation, payload)
276
+ break
277
+ except KeywordEconomicsError as exc:
278
+ if not (_is_rate_limited(exc) or _is_unpaid_internal_error(exc)):
279
+ raise
280
+ if attempt >= PROVIDER_MAX_RETRIES:
281
+ raise KeywordEconomicsError(
282
+ f"{exc}; automatic retries exhausted ({attempt}). "
283
+ "Research is incomplete. Successful responses remain cached; "
284
+ "preserve the workspace and report the blocked section. "
285
+ "A later recovery attempt can rerun the same command from cache."
286
+ ) from exc
287
+ delay = PROVIDER_BACKOFF_SECONDS[attempt]
288
+ print(
289
+ f"{DISPLAY_NAME}: {exc}; retry {attempt + 1}/{PROVIDER_MAX_RETRIES} "
290
+ f"in {delay:g}s with unchanged parameters.", file=sys.stderr,
291
+ )
292
+ (self.sleep or time.sleep)(delay)
293
+ attempt += 1
294
+ self.tasks_spent += 1
295
+ cost = response.get("cost")
296
+ if isinstance(cost, (int, float)):
297
+ self.estimated_cost_usd += float(cost)
298
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
299
+ _write_atomic(cache_file,
300
+ json.dumps(
301
+ {
302
+ "operation": operation,
303
+ "payload": payload,
304
+ "response": response,
305
+ "retrieved_on": date.today().isoformat(),
306
+ }
307
+ ),
308
+ )
309
+ self._ledger_entries.add(cache_file)
310
+ return response
311
+
312
+ def ledger_totals(self) -> dict[str, Any]:
313
+ """Durable spend provenance for calls used by this pipeline execution.
314
+
315
+ Every used cache entry is one metered upstream call that actually
316
+ happened, regardless of which attempt made it. A cache-only replay
317
+ therefore reports the ORIGINAL spend without pulling unrelated entries
318
+ from older runs with different inputs into the current section totals.
319
+ """
320
+ tasks = 0
321
+ cost = 0.0
322
+ dates: list[str] = []
323
+ if self.cache_dir.is_dir():
324
+ for entry_path in sorted(self._ledger_entries):
325
+ try:
326
+ entry = json.loads(entry_path.read_text(encoding="utf-8"))
327
+ except (OSError, json.JSONDecodeError):
328
+ continue
329
+ tasks += 1
330
+ entry_cost = (entry.get("response") or {}).get("cost")
331
+ if isinstance(entry_cost, (int, float)):
332
+ cost += float(entry_cost)
333
+ if isinstance(entry.get("retrieved_on"), str):
334
+ dates.append(entry["retrieved_on"])
335
+ return {
336
+ "tasks_spent": tasks,
337
+ "estimated_cost_usd": round(cost, 4),
338
+ "first_retrieved_on": min(dates) if dates else date.today().isoformat(),
339
+ "last_retrieved_on": max(dates) if dates else date.today().isoformat(),
340
+ }
341
+
342
+
343
+ # ── deterministic pipeline steps ───────────────────────────────────────────
344
+
345
+ def parse_terms(raw: str, cap: int) -> list[str]:
346
+ """Split a comma/semicolon/newline separated list, dedupe, cap."""
347
+ seen: dict[str, None] = {}
348
+ for part in re.split(r"[,;\n]", raw or ""):
349
+ term = " ".join(part.split()).strip().lower()
350
+ if term and term not in seen:
351
+ seen[term] = None
352
+ return list(seen)[:cap]
353
+
354
+
355
+ # Second-level labels that make a public suffix under a country code
356
+ # (example.co.uk, example.com.au, example.edu.in, example.or.kr): the
357
+ # registrable name is the label before them.
358
+ SECOND_LEVEL_SUFFIX_LABELS = frozenset({
359
+ "co", "com", "org", "net", "gov", "ac", "edu", "or", "ne", "go", "gob", "nic", "ltd", "plc", "sch", "res", "gen",
360
+ "ind", "firm", "nom", "mil", "int", "biz", "info",
361
+ })
362
+ # Course and creator hosts that give every seller a subdomain: the subdomain
363
+ # is the seller (dair-ai.thinkific.com is DAIR.AI's school, not Thinkific's),
364
+ # so it is a domain of its own. The host's service subdomains still mean the host.
365
+ HOSTED_SELLER_PLATFORMS = frozenset({
366
+ "thinkific.com", "teachable.com", "kajabi.com", "mykajabi.com", "podia.com", "learnworlds.com", "gumroad.com",
367
+ "substack.com", "systeme.io", "circle.so", "mn.co", "github.io", "notion.site", "webflow.io", "framer.website",
368
+ "carrd.co", "wixsite.com", "wordpress.com", "blogspot.com", "hashnode.dev",
369
+ })
370
+ HOST_SERVICE_LABELS = frozenset({"www", "app", "help", "support", "docs", "status", "api", "community", "blog"})
371
+
372
+
373
+ def registrable_domain(value: str) -> str:
374
+ """Normalize a URL or hostname to its registrable domain (best effort);
375
+ a seller's subdomain on a course or creator host is kept whole."""
376
+ host = (value or "").strip().lower()
377
+ if "//" in host:
378
+ host = urlparse(host).hostname or ""
379
+ host = host.split("/", 1)[0].removeprefix("www.")
380
+ parts = [p for p in host.split(".") if p]
381
+ if len(parts) <= 2:
382
+ return ".".join(parts)
383
+ # Collapse deep subdomains; keep two-level public suffixes intact.
384
+ keep = 3 if parts[-2] in SECOND_LEVEL_SUFFIX_LABELS and len(parts[-1]) == 2 else 2
385
+ if ".".join(parts[-keep:]) in HOSTED_SELLER_PLATFORMS and len(parts) > keep and parts[-keep - 1] not in HOST_SERVICE_LABELS:
386
+ keep += 1
387
+ return ".".join(parts[-keep:])
388
+
389
+
390
+ def hosted_seller(domain: str) -> bool:
391
+ """A seller's own subdomain on a course or creator host."""
392
+ return domain.count(".") >= 2 and domain.split(".", 1)[1] in HOSTED_SELLER_PLATFORMS
393
+
394
+
395
+ def has_intent(keyword: str) -> bool:
396
+ return any(token in keyword.split() for token in INTENT_TOKENS)
397
+
398
+
399
+ def content_tokens(keywords: list[str]) -> set[str]:
400
+ """Category-bearing tokens of the seed market: everything except generic
401
+ intent modifiers and short function words. These anchor the topical filter
402
+ that keeps competitor footprints from contaminating the demand model."""
403
+ tokens: set[str] = set()
404
+ for keyword in keywords:
405
+ for token in keyword.split():
406
+ if len(token) >= 3 and token not in INTENT_TOKENS and token not in FUNCTION_TOKENS:
407
+ tokens.add(token)
408
+ return tokens
409
+
410
+
411
+ def brand_token(domain: str) -> str:
412
+ """clickup.com -> clickup — the brand token used to keep competitor-brand
413
+ queries out of the budget simulation."""
414
+ stem = registrable_domain(domain).split(".")[0]
415
+ return re.sub(r"[^a-z0-9]", "", stem)
416
+
417
+
418
+ def has_free_edu_intent(keyword: str) -> bool:
419
+ """Note/flag marker: free OR education audience terms."""
420
+ return any(token in FREE_EDU_TOKENS for token in keyword.split())
421
+
422
+
423
+ def has_free_intent(keyword: str) -> bool:
424
+ """Simulation gate: ONLY explicit 'free' intent — education/audience
425
+ terms mark verticals ('school management software' is commercial) and
426
+ must not exclude a query from paid simulation."""
427
+ return "free" in keyword.split()
428
+
429
+
430
+ def query_intent_signal(keyword: str) -> str:
431
+ """Return only a defensible lexical signal; unknown is a valid outcome."""
432
+ tokens = set(keyword.lower().split())
433
+ if tokens & JOB_TERMS:
434
+ return "job_seeking"
435
+ if "free" in tokens:
436
+ return "free_intent"
437
+ if tokens & INFORMATION_TERMS:
438
+ return "informational"
439
+ if tokens & VENDOR_TERMS:
440
+ return "vendor_seeking"
441
+ if tokens & TOOL_TERMS:
442
+ return "tool_seeking"
443
+ return "unknown"
444
+
445
+
446
+ def _tokens_match(a: str, b: str) -> bool:
447
+ """Exact match, or simple plural-suffix equality (+s/+es) for stems of
448
+ 6+ characters: 'whiteboards' matches 'whiteboard', but 'canvas' does NOT
449
+ match 'canva' (short stems collide — the live canva/canvas adversarial
450
+ pair). Deliberately NOT a stemmer: irregular morphology ('categories')
451
+ is out of scope and documented."""
452
+ if a == b:
453
+ return True
454
+ shorter, longer = (a, b) if len(a) <= len(b) else (b, a)
455
+ if len(shorter) < 6:
456
+ return False
457
+ return longer == shorter + "s" or longer == shorter + "es"
458
+
459
+
460
+ def is_topical(keyword: str, market_tokens: set[str]) -> bool:
461
+ """A keyword joins the market pool only when it shares at least one
462
+ category-bearing token (plural-tolerant) with the seed anchors. 'ai
463
+ furniture staging' shares 'staging'; a broad competitor's 'best crm app'
464
+ shares nothing and stays footprint-only evidence."""
465
+ for token in keyword.split():
466
+ if len(token) < 3 or token in INTENT_TOKENS or token in FUNCTION_TOKENS:
467
+ continue
468
+ if any(_tokens_match(token, anchor) for anchor in market_tokens):
469
+ return True
470
+ return False
471
+
472
+
473
+ def build_foreign_brand_veto(
474
+ rejected_keywords: list[str], seed_tokens: set[str]
475
+ ) -> set[str]:
476
+ """Bounded foreign-brand detector using POSITION as brand evidence, not
477
+ bare frequency: brands lead their phrases ('canva ai', 'canva ai
478
+ generator'), while recurring category nouns trail ('brainstorm
479
+ template', 'meeting template' must not veto 'template'). A non-seed
480
+ content token LEADING 2+ drift-rejected phrases is treated as a foreign
481
+ brand; keywords that slipped past the first pass on a weak shared token
482
+ ('canva ai text' via seed token 'text') are removed only when the veto
483
+ token leads them too."""
484
+ counts: dict[str, int] = {}
485
+ for keyword in rejected_keywords:
486
+ tokens = keyword.split()
487
+ if not tokens:
488
+ continue
489
+ first = tokens[0]
490
+ if len(first) < 3 or first in INTENT_TOKENS or first in FUNCTION_TOKENS:
491
+ continue
492
+ if any(_tokens_match(first, anchor) for anchor in seed_tokens):
493
+ continue
494
+ counts[first] = counts.get(first, 0) + 1
495
+ return {token for token, count in counts.items() if count >= 2}
496
+
497
+
498
+ def normalize_keyword_row(raw: dict[str, Any], source: str) -> dict[str, Any] | None:
499
+ keyword = " ".join(str(raw.get("keyword") or "").split()).lower()
500
+ if not keyword:
501
+ return None
502
+ return {
503
+ "type": "keyword",
504
+ "keyword": keyword,
505
+ "intent_signal": query_intent_signal(keyword),
506
+ "free_edu_intent": has_free_edu_intent(keyword),
507
+ "search_volume": raw.get("search_volume"),
508
+ "cpc": raw.get("cpc"),
509
+ "cpc_low": raw.get("low_top_of_page_bid"),
510
+ "cpc_high": raw.get("high_top_of_page_bid"),
511
+ "competition": raw.get("competition"),
512
+ "sources": [source],
513
+ "tier": "A",
514
+ }
515
+
516
+
517
+ def normalize_ranked_row(item: dict[str, Any], domain: str) -> dict[str, Any] | None:
518
+ keyword_data = item.get("keyword_data") or {}
519
+ info = keyword_data.get("keyword_info") or {}
520
+ serp_item = (item.get("ranked_serp_element") or {}).get("serp_item") or {}
521
+ row = normalize_keyword_row(
522
+ {
523
+ "keyword": keyword_data.get("keyword"),
524
+ "search_volume": info.get("search_volume"),
525
+ "cpc": info.get("cpc"),
526
+ "low_top_of_page_bid": info.get("low_top_of_page_bid"),
527
+ "high_top_of_page_bid": info.get("high_top_of_page_bid"),
528
+ "competition": info.get("competition_level") or info.get("competition"),
529
+ },
530
+ source=f"ranked_keywords:{domain}",
531
+ )
532
+ if row is None:
533
+ return None
534
+ if serp_item.get("type") == "paid":
535
+ # The competitor BUYS this query — the strongest channel evidence here.
536
+ row["paid_by"] = [domain]
537
+ return row
538
+
539
+
540
+ def merge_keyword_rows(*row_lists: list[dict[str, Any]]) -> list[dict[str, Any]]:
541
+ merged: dict[str, dict[str, Any]] = {}
542
+ for rows in row_lists:
543
+ for row in rows:
544
+ if row is None:
545
+ continue
546
+ existing = merged.get(row["keyword"])
547
+ if existing is None:
548
+ merged[row["keyword"]] = {**row, "sources": list(row["sources"])}
549
+ continue
550
+ for source in row["sources"]:
551
+ if source not in existing["sources"]:
552
+ existing["sources"].append(source)
553
+ for paid_domain in row.get("paid_by", []):
554
+ existing.setdefault("paid_by", [])
555
+ if paid_domain not in existing["paid_by"]:
556
+ existing["paid_by"].append(paid_domain)
557
+ for key in ("search_volume", "cpc", "cpc_low", "cpc_high", "competition"):
558
+ if existing.get(key) is None and row.get(key) is not None:
559
+ existing[key] = row[key]
560
+ return list(merged.values())
561
+
562
+
563
+ def rank_keywords(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
564
+ def sort_key(row: dict[str, Any]):
565
+ volume = row.get("search_volume") or 0
566
+ return (1 if has_intent(row["keyword"]) else 0, volume)
567
+
568
+ return sorted(
569
+ (r for r in rows if (r.get("search_volume") or 0) > 0),
570
+ key=sort_key,
571
+ reverse=True,
572
+ )
573
+
574
+
575
+ def pick_serp_heads(ranked: list[dict[str, Any]], count: int = SERP_HEADS) -> list[str]:
576
+ heads: list[str] = []
577
+ seen_first_tokens: set[str] = set()
578
+ # First pass prefers lexical diversity so eight heads don't all probe the
579
+ # same phrase family; second pass fills remaining slots by rank.
580
+ for row in ranked:
581
+ first = row["keyword"].split()[0]
582
+ if first not in seen_first_tokens:
583
+ heads.append(row["keyword"])
584
+ seen_first_tokens.add(first)
585
+ if len(heads) >= count:
586
+ return heads
587
+ for row in ranked:
588
+ if row["keyword"] not in heads:
589
+ heads.append(row["keyword"])
590
+ if len(heads) >= count:
591
+ break
592
+ return heads
593
+
594
+
595
+ def extract_serp_observation(keyword: str, serp_result: list[dict[str, Any]]) -> dict[str, Any]:
596
+ entry = (serp_result or [{}])[0]
597
+ available = bool(serp_result) and isinstance(entry.get("items"), list)
598
+ items = entry.get("items") or []
599
+ paid: list[str] = []
600
+ organic: list[str] = []
601
+ for item in items:
602
+ domain = registrable_domain(item.get("domain") or item.get("url") or "")
603
+ if not domain:
604
+ continue
605
+ if item.get("type") == "paid" and domain not in paid:
606
+ paid.append(domain)
607
+ elif item.get("type") == "organic" and len(organic) < 10 and domain not in organic:
608
+ organic.append(domain)
609
+ return {
610
+ "type": "serp_observation",
611
+ "keyword": keyword,
612
+ "intent_signal": query_intent_signal(keyword),
613
+ "paid_advertisers": paid,
614
+ "ads_count": len(paid) if available else None,
615
+ "response_status": "available" if available else "unavailable",
616
+ "organic_top": organic,
617
+ "se_results_count": entry.get("se_results_count"),
618
+ "tier": "A",
619
+ }
620
+
621
+
622
+ def classify_domain(domain: str) -> str:
623
+ if domain in NOISE_DOMAINS:
624
+ return "noise"
625
+ if domain in DIRECTORY_DOMAINS:
626
+ return "directory"
627
+ return "candidate"
628
+
629
+
630
+ def tally_competitors(
631
+ observations: list[dict[str, Any]],
632
+ seed_domains: list[str],
633
+ keyword_volumes: dict[str, int] | None = None,
634
+ ) -> list[dict[str, Any]]:
635
+ """Score competitors by VOLUME-WEIGHTED SERP presence: one appearance on
636
+ a 165k/mo head outweighs several appearances on tiny queries (observed
637
+ live: education micro-sites outranked figma.com on raw appearance
638
+ counts). Weight = log10(head volume), ads-block presence x3."""
639
+ volumes = keyword_volumes or {}
640
+ tally: dict[str, dict[str, Any]] = {}
641
+
642
+ def bump(domain: str, via: str, weight: float) -> None:
643
+ kind = classify_domain(domain)
644
+ if kind == "noise":
645
+ return
646
+ row = tally.setdefault(
647
+ domain,
648
+ {
649
+ "type": "competitor_domain",
650
+ "domain": domain,
651
+ "kind": kind,
652
+ "found_via": [],
653
+ "serp_paid_appearances": 0,
654
+ "serp_organic_appearances": 0,
655
+ "serp_queries": [],
656
+ "score": 0,
657
+ "tier": "A",
658
+ },
659
+ )
660
+ if via not in row["found_via"]:
661
+ row["found_via"].append(via)
662
+ if via == "serp_ads":
663
+ row["serp_paid_appearances"] += 1
664
+ elif via == "serp_organic":
665
+ row["serp_organic_appearances"] += 1
666
+ row["score"] = round(row["score"] + weight, 2)
667
+
668
+ def head_weight(keyword: str) -> float:
669
+ return math.log10(max(volumes.get(keyword, 0), 10))
670
+
671
+ for domain in seed_domains:
672
+ normalized = registrable_domain(domain)
673
+ if normalized:
674
+ bump(normalized, "seed", 2.0)
675
+ for obs in observations:
676
+ weight = head_weight(obs["keyword"])
677
+ for domain in obs["paid_advertisers"]:
678
+ # A paid appearance is evidence, not yet category confirmation.
679
+ bump(domain, "serp_ads", 3.0 * weight)
680
+ row = tally.get(domain)
681
+ if row is not None:
682
+ row["serp_queries"].append({"keyword": obs["keyword"], "via": "serp_ads"})
683
+ for domain in obs["organic_top"]:
684
+ bump(domain, "serp_organic", weight)
685
+ row = tally.get(domain)
686
+ if row is not None:
687
+ row["serp_queries"].append({"keyword": obs["keyword"], "via": "serp_organic"})
688
+
689
+ return sorted(tally.values(), key=lambda r: r["score"], reverse=True)
690
+
691
+
692
+ def confirmed_competitors(
693
+ candidates: list[dict[str, Any]],
694
+ seed_domains: list[str] | None = None,
695
+ count: int = CONFIRM_COMPETITORS,
696
+ seed_keywords: list[str] | None = None,
697
+ ) -> list[str]:
698
+ """Founder-named seed domains are reserved FIRST — volume-weighted
699
+ discovery must not evict an explicitly supplied competitor from the
700
+ footprint pulls."""
701
+ ordered: list[str] = []
702
+ for domain in seed_domains or []:
703
+ normalized = registrable_domain(domain)
704
+ if normalized and normalized not in ordered and classify_domain(normalized) == "candidate":
705
+ ordered.append(normalized)
706
+ for row in candidates:
707
+ if len(ordered) >= count:
708
+ break
709
+ if (
710
+ row["kind"] == "candidate"
711
+ and row["domain"] not in ordered
712
+ and (seed_keywords is None or competitor_category_evidence(row, seed_keywords))
713
+ ):
714
+ ordered.append(row["domain"])
715
+ return ordered[:count]
716
+
717
+
718
+ def competitor_category_evidence(row: dict[str, Any], seeds: list[str]) -> bool:
719
+ """Require evidence beyond an ambiguous one-word category SERP.
720
+
721
+ One paid appearance on a category-qualified query, or two independent
722
+ organic appearances on such queries, is enough. Generic query modifiers
723
+ such as 'agency' are not category context. 'AI' can complete a two-token
724
+ context only alongside a real seed anchor (e.g. 'AI visibility').
725
+ """
726
+ anchors = content_tokens(seeds) - VENDOR_TERMS
727
+ relevant: dict[str, set[str]] = {}
728
+ for match in row.get("serp_queries") or []:
729
+ keyword = str(match.get("keyword") or "")
730
+ tokens = set(keyword.split())
731
+ category = tokens & anchors
732
+ contextual = len(category) >= 2 or ("ai" in tokens and bool(category))
733
+ if contextual:
734
+ relevant.setdefault(keyword, set()).add(str(match.get("via") or ""))
735
+ return any("serp_ads" in via for via in relevant.values()) or len(relevant) >= 2
736
+
737
+
738
+ def median(values: list[float]) -> float | None:
739
+ cleaned = sorted(v for v in values if isinstance(v, (int, float)) and v > 0)
740
+ if not cleaned:
741
+ return None
742
+ middle = len(cleaned) // 2
743
+ if len(cleaned) % 2:
744
+ return float(cleaned[middle])
745
+ return (cleaned[middle - 1] + cleaned[middle]) / 2.0
746
+
747
+
748
+ def build_budget_rows(
749
+ sim_keywords: list[str],
750
+ bid: float,
751
+ ad_traffic_result: list[dict[str, Any]],
752
+ test_budget_usd: float,
753
+ ) -> list[dict[str, Any]]:
754
+ rows: list[dict[str, Any]] = []
755
+ for raw in ad_traffic_result or []:
756
+ clicks = raw.get("clicks")
757
+ cost = raw.get("cost")
758
+ keyword = raw.get("keyword")
759
+ scale = None
760
+ clicks_for_budget = None
761
+ if test_budget_usd is not None and isinstance(clicks, (int, float)) and isinstance(cost, (int, float)) and cost > 0:
762
+ scale = test_budget_usd / cost
763
+ clicks_for_budget = round(clicks * min(scale, 1.0), 1) if scale < 1 else round(clicks, 1)
764
+ elif test_budget_usd is not None and clicks == 0 and cost == 0:
765
+ scale = 1.0
766
+ clicks_for_budget = 0
767
+ rows.append(
768
+ {
769
+ "type": "budget_simulation",
770
+ "keyword": keyword or "(all simulated keywords)",
771
+ "simulated_keywords": sim_keywords if keyword is None else None,
772
+ "bid": bid,
773
+ "match": raw.get("match"),
774
+ "average_cpc": raw.get("average_cpc"),
775
+ "monthly_clicks_at_bid": clicks,
776
+ "monthly_cost_at_bid": cost,
777
+ "test_budget_usd": test_budget_usd,
778
+ "est_clicks_within_test_budget": clicks_for_budget,
779
+ "budget_covers_full_month": scale >= 1 if scale is not None else None,
780
+ "tier": "A",
781
+ }
782
+ )
783
+ return rows
784
+
785
+
786
+ # ── rendering ──────────────────────────────────────────────────────────────
787
+
788
+ def _fmt(value: Any, money: bool = False) -> str:
789
+ if value is None:
790
+ return "—"
791
+ if isinstance(value, float):
792
+ return f"${value:,.2f}" if money else f"{value:,.1f}".rstrip("0").rstrip(".")
793
+ if isinstance(value, int):
794
+ return f"${value:,}" if money else f"{value:,}"
795
+ return str(value)
796
+
797
+
798
+ def render_section(
799
+ *,
800
+ locale: dict[str, Any],
801
+ seeds: list[str],
802
+ seed_rows_alive: int,
803
+ keyword_rows: list[dict[str, Any]],
804
+ footprint_rows: list[dict[str, Any]],
805
+ observations: list[dict[str, Any]],
806
+ competitor_rows: list[dict[str, Any]],
807
+ confirmed: list[str],
808
+ budget_rows: list[dict[str, Any]],
809
+ overlap_domains: list[str],
810
+ totals: dict[str, Any],
811
+ test_budget_usd: float,
812
+ retrieved_on: str,
813
+ expansion_rejected_count: int = 0,
814
+ expansion_veto_rejected_count: int = 0,
815
+ simulation_omitted_reason: str | None = None,
816
+ simulation_coverage: dict | None = None,
817
+ ) -> str:
818
+ ranked = rank_keywords(keyword_rows)
819
+ top_rows = ranked[:TOP_TABLE_ROWS]
820
+ lines: list[str] = []
821
+ add = lines.append
822
+
823
+ add(
824
+ f"Primary auction data retrieved {retrieved_on} via DataForSEO "
825
+ f"(Google Ads / Labs / SERP endpoints) for location code "
826
+ f"{locale['location_code']}, language `{locale['language_code']}`. "
827
+ f"Seeds supplied by the workflow: {len(seeds)}; seeds surviving "
828
+ f"volume validation: {seed_rows_alive}. All numbers below are "
829
+ "measured or engine-estimated market data, not model knowledge."
830
+ )
831
+ add("")
832
+ add("### Demand and click cost by keyword")
833
+ add("")
834
+ add("| Keyword | Monthly searches | CPC (avg) | Top-of-page bid low–high | Competition | Paid by competitor | Sources |")
835
+ add("| --- | ---: | ---: | ---: | --- | --- | --- |")
836
+ for row in top_rows:
837
+ bid_range = (
838
+ f"{_fmt(row.get('cpc_low'), money=True)}–{_fmt(row.get('cpc_high'), money=True)}"
839
+ if row.get("cpc_low") is not None or row.get("cpc_high") is not None
840
+ else "—"
841
+ )
842
+ add(
843
+ f"| {row['keyword']} | {_fmt(row.get('search_volume'))} | "
844
+ f"{_fmt(row.get('cpc'), money=True)} | {bid_range} | "
845
+ f"{_fmt(row.get('competition'))} | "
846
+ f"{', '.join(row.get('paid_by', [])) or '—'} | "
847
+ f"{', '.join(sorted({s.split(':', 1)[0] for s in row['sources']}))} |"
848
+ )
849
+ add("")
850
+ add(f"{len(ranked)} keywords with non-zero volume collected in total; the table shows the top {len(top_rows)} by commercial intent and volume. The full set is in the machine-readable evidence file.")
851
+ add("")
852
+
853
+ add("### Who occupies the results page")
854
+ add("")
855
+ add("| Head keyword | Advertisers in ads block | Organic top domains |")
856
+ add("| --- | --- | --- |")
857
+ for obs in observations:
858
+ add(
859
+ f"| {obs['keyword']} | "
860
+ f"{', '.join(obs['paid_advertisers']) or ('unavailable' if obs.get('response_status') == 'unavailable' else 'none observed')} | "
861
+ f"{', '.join(obs['organic_top'][:5])} |"
862
+ )
863
+ add("")
864
+ zero_ads = [o["keyword"] for o in observations if o["ads_count"] == 0]
865
+ if zero_ads:
866
+ add(
867
+ "No ads block was observed on: "
868
+ + ", ".join(f"`{k}`" for k in zero_ads)
869
+ + ". This sampled observation does not establish that nobody advertises "
870
+ "or pays for these queries; competitor spending remains unproven here."
871
+ )
872
+ free_heads = [k for k in zero_ads if has_free_edu_intent(k)]
873
+ if free_heads:
874
+ add(
875
+ "Note: "
876
+ + ", ".join(f"`{k}`" for k in free_heads)
877
+ + " carry free/education intent markers — an empty ads block "
878
+ "there is partially self-explanatory."
879
+ )
880
+ add("")
881
+
882
+ add("### Competitors on the demand surface")
883
+ add("")
884
+ add("| Domain | Kind | Found via | SERP paid | SERP organic |")
885
+ add("| --- | --- | --- | ---: | ---: |")
886
+ for row in competitor_rows[:12]:
887
+ add(
888
+ f"| {row['domain']} | {row['kind']} | {', '.join(row['found_via'])} | "
889
+ f"{row['serp_paid_appearances']} | {row['serp_organic_appearances']} |"
890
+ )
891
+ add("")
892
+ if confirmed:
893
+ add(
894
+ "Confirmed product competitors (keyword footprints pulled): "
895
+ + ", ".join(f"`{d}`" for d in confirmed)
896
+ + "."
897
+ )
898
+ if overlap_domains:
899
+ add(
900
+ "Additional keyword-overlap neighbors (Labs `competitors_domain`, "
901
+ "not re-crawled): " + ", ".join(f"`{d}`" for d in overlap_domains[:10]) + "."
902
+ )
903
+ add("")
904
+
905
+ if footprint_rows:
906
+ add("### Competitor keyword footprints (separate evidence)")
907
+ add("")
908
+ add(
909
+ "Footprints are competitor-scoped evidence: only rows sharing a "
910
+ "category token with the seed market are merged into the demand "
911
+ "table above; everything else stays here and in the evidence "
912
+ "file, and none of it feeds the budget simulation."
913
+ )
914
+ add("")
915
+ add("| Domain | Keyword | Monthly searches | CPC (avg) | Competitor pays for it | In market pool |")
916
+ add("| --- | --- | ---: | ---: | --- | --- |")
917
+ for domain in confirmed:
918
+ domain_rows = sorted(
919
+ (r for r in footprint_rows if r.get("domain") == domain),
920
+ key=lambda r: r.get("search_volume") or 0,
921
+ reverse=True,
922
+ )[:5]
923
+ for row in domain_rows:
924
+ add(
925
+ f"| {domain} | {row['keyword']} | "
926
+ f"{_fmt(row.get('search_volume'))} | "
927
+ f"{_fmt(row.get('cpc'), money=True)} | "
928
+ f"{'yes' if row.get('paid_by') else 'no'} | "
929
+ f"{'yes' if row.get('topical') else 'no'} |"
930
+ )
931
+ add("")
932
+
933
+ add("### Candidate-set forecast" + (f" — ${test_budget_usd:,.0f} test-budget arithmetic" if test_budget_usd is not None else " — provider monthly estimates; no test budget supplied"))
934
+ add("")
935
+ if simulation_coverage:
936
+ c = simulation_coverage
937
+ add(f"Scenario: {c['kind']}; {c['submitted_count']}/{c['requested_count']} requested keywords, "
938
+ f"{len(c['excluded'])} excluded, {len(c['omitted_keywords'])} omitted by cap; "
939
+ f"match `{c['match']}`, bid assumption ${c['bid']:g}, period `{c['forecast_period']}`.")
940
+ add(c["limitations"])
941
+ add("")
942
+ if simulation_omitted_reason:
943
+ add(
944
+ f"Simulation omitted: {simulation_omitted_reason}. Rather than "
945
+ "simulating on excluded queries, this run reports the limitation "
946
+ "explicitly — revise seeds toward commercial category phrasing "
947
+ "and re-run if a paid-search estimate is needed."
948
+ )
949
+ add("")
950
+ if budget_rows:
951
+ add("| Simulated keyword | Bid | Est. avg CPC | Est. clicks/month at bid | Est. cost/month | Est. clicks within test budget |")
952
+ add("| --- | ---: | ---: | ---: | ---: | ---: |")
953
+ elif not simulation_omitted_reason:
954
+ add("Forecast unavailable: the provider returned no usable result for this keyword set. "
955
+ "Expected clicks and cost are unknown, not zero. No budget feasibility estimate is available.")
956
+ for row in budget_rows:
957
+ add(
958
+ f"| {row['keyword']} | {_fmt(row.get('bid'), money=True)} | "
959
+ f"{_fmt(row.get('average_cpc'), money=True)} | "
960
+ f"{_fmt(row.get('monthly_clicks_at_bid'))} | "
961
+ f"{_fmt(row.get('monthly_cost_at_bid'), money=True)} | "
962
+ f"{_fmt(row.get('est_clicks_within_test_budget'))} |"
963
+ )
964
+ add("")
965
+ if budget_rows:
966
+ add(
967
+ "Estimates come from Google's ad-traffic forecast at the stated bid "
968
+ "(`ad_traffic_by_keywords`); clicks within the test budget are a "
969
+ "linear pro-rata of the monthly forecast and ignore learning-phase "
970
+ "effects. They anchor feasibility math; they do not guarantee auction "
971
+ "outcomes."
972
+ )
973
+ if simulation_coverage and simulation_coverage["kind"] == "explicit_scenario":
974
+ add("This forecast uses the caller's explicit scenario, not an automatically selected seed pool.")
975
+ else:
976
+ add("Simulation keywords are drawn only from the eligible seed and expansion pool — "
977
+ "never from competitor footprints, competitor-brand queries, or free/education-intent queries.")
978
+ add("")
979
+
980
+ add("### Method and coverage")
981
+ add("")
982
+ add(
983
+ f"- Metered DataForSEO tasks recorded for this section: "
984
+ f"{totals['tasks_spent']} (≈ ${totals['estimated_cost_usd']:.2f}), "
985
+ "accumulated across attempts; cached replays add no new spend."
986
+ )
987
+ add(
988
+ f"- Coverage caps: {SERP_HEADS} SERP heads at depth {SERP_DEPTH}, "
989
+ f"{CONFIRM_COMPETITORS} competitor keyword footprints at "
990
+ f"{RANKED_KEYWORDS_LIMIT} keywords each, {BUDGET_SIM_KEYWORDS} "
991
+ "budget-simulation keywords."
992
+ )
993
+ if expansion_rejected_count or expansion_veto_rejected_count:
994
+ add(
995
+ f"- Expansion drift guard rejected {expansion_rejected_count} "
996
+ "Google-suggested keywords sharing no seed content token, plus "
997
+ f"{expansion_veto_rejected_count} more via the leading-brand "
998
+ "veto (both groups listed with their reasons in the evidence "
999
+ "file) — foreign-brand drift like 'ai canvas' -> 'canva ai' "
1000
+ "stays out of the demand model."
1001
+ )
1002
+ add(
1003
+ "- Not covered here: non-Google channels, organic difficulty, "
1004
+ "seasonality time series, and any interpretation of which "
1005
+ "segment/offer/channel combination to pursue (downstream concern)."
1006
+ )
1007
+ add("")
1008
+ add("### Sources")
1009
+ add("")
1010
+ add(
1011
+ f"- DataForSEO Google Ads endpoints (`search_volume`, "
1012
+ f"`keywords_for_keywords`, `ad_traffic_by_keywords`), Labs "
1013
+ f"(`ranked_keywords`, `competitors_domain`), and SERP advanced — "
1014
+ f"retrieved {retrieved_on} via the OS tools gateway; evidence rows in "
1015
+ f"`{EVIDENCE_REL.as_posix()}`."
1016
+ )
1017
+ return "\n".join(lines).strip() + "\n"
1018
+
1019
+
1020
+ # ── orchestration ──────────────────────────────────────────────────────────
1021
+
1022
+ @dataclass
1023
+ class RunInputs:
1024
+ seeds: list[str]
1025
+ seed_domains: list[str]
1026
+ location_code: int
1027
+ language_code: str
1028
+ test_budget_usd: float
1029
+ scenario: dict | None = None
1030
+
1031
+
1032
+ def run_pipeline(client: ToolsClient, inputs: RunInputs) -> dict[str, Any]:
1033
+ locale = {"location_code": inputs.location_code, "language_code": inputs.language_code}
1034
+ forecast = _load_script("keyword-forecast.py", "pmr_keyword_forecast")
1035
+ scenario = forecast.validate_scenario(inputs.scenario, locale) if inputs.scenario is not None else None
1036
+ if type(inputs.location_code) is not int or inputs.location_code <= 0 or not re.fullmatch(r"[a-z]{2,3}(?:-[a-z]{2})?", inputs.language_code):
1037
+ raise KeywordEconomicsError("location-code and normalized language-code are invalid")
1038
+ if inputs.test_budget_usd is not None and (not math.isfinite(inputs.test_budget_usd) or inputs.test_budget_usd <= 0):
1039
+ raise KeywordEconomicsError("test-budget-usd must be finite and positive")
1040
+
1041
+ seed_volume = client.call("search-volume", {"keywords": inputs.seeds, **locale})
1042
+ expansion = client.call("keywords-for-keywords", {"keywords": inputs.seeds, **locale})
1043
+
1044
+ seed_rows = [normalize_keyword_row(r, "search_volume") for r in seed_volume.get("result") or []]
1045
+ expansion_rows = [normalize_keyword_row(r, "keywords_for_keywords") for r in expansion.get("result") or []]
1046
+ # Google's expansion drifts into foreign brands ("ai canvas" -> "canva
1047
+ # ai", observed live at 135k volume): an expansion keyword joins the
1048
+ # market pool only when it shares a seed content token. Seeds themselves
1049
+ # are the declared market and always stay.
1050
+ seed_anchor_tokens = content_tokens(inputs.seeds)
1051
+ kept_expansion: list[dict[str, Any]] = []
1052
+ rejected_expansion: list[dict[str, Any]] = []
1053
+ for row in expansion_rows:
1054
+ if row is None:
1055
+ continue
1056
+ if is_topical(row["keyword"], seed_anchor_tokens):
1057
+ kept_expansion.append(row)
1058
+ else:
1059
+ rejected_expansion.append(row)
1060
+ # Second pass: tokens dominating the rejected set are foreign brands;
1061
+ # keywords that slipped through on a single shared seed token ('canva ai
1062
+ # text' via 'text') get vetoed too.
1063
+ veto_tokens = build_foreign_brand_veto(
1064
+ [row["keyword"] for row in rejected_expansion], seed_anchor_tokens
1065
+ )
1066
+ veto_rejected: list[dict[str, Any]] = []
1067
+ if veto_tokens:
1068
+ still_kept: list[dict[str, Any]] = []
1069
+ for row in kept_expansion:
1070
+ tokens = row["keyword"].split()
1071
+ if tokens and tokens[0] in veto_tokens:
1072
+ veto_rejected.append(row)
1073
+ else:
1074
+ still_kept.append(row)
1075
+ kept_expansion = still_kept
1076
+ rejected_expansion.sort(key=lambda r: r.get("search_volume") or 0, reverse=True)
1077
+ veto_rejected.sort(key=lambda r: r.get("search_volume") or 0, reverse=True)
1078
+ expansion_rejected_rows = [
1079
+ {
1080
+ "type": "expansion_rejected",
1081
+ "keyword": row["keyword"],
1082
+ "intent_signal": row["intent_signal"],
1083
+ "search_volume": row.get("search_volume"),
1084
+ "reason": "no_seed_token_overlap",
1085
+ "tier": "C",
1086
+ }
1087
+ for row in rejected_expansion[:20]
1088
+ ] + [
1089
+ {
1090
+ "type": "expansion_rejected",
1091
+ "keyword": row["keyword"],
1092
+ "intent_signal": row["intent_signal"],
1093
+ "search_volume": row.get("search_volume"),
1094
+ "reason": "foreign_brand_veto",
1095
+ "tier": "C",
1096
+ }
1097
+ for row in veto_rejected[:20]
1098
+ ]
1099
+ # The MARKET pool: seeds + the anchored part of Google's expansion. This
1100
+ # pool alone defines the demand model; competitor footprints may only
1101
+ # annotate it (topical intersection) and never replace it.
1102
+ market_pool = merge_keyword_rows(
1103
+ [r for r in seed_rows if r],
1104
+ kept_expansion,
1105
+ )
1106
+ base_ranked = rank_keywords(market_pool)
1107
+ if not base_ranked:
1108
+ raise KeywordEconomicsError(
1109
+ "no seed or expanded keyword has non-zero search volume; the seed "
1110
+ "list does not map to measurable demand — revise seeds in the "
1111
+ "workflow and re-run"
1112
+ )
1113
+ seed_rows_alive = sum(
1114
+ 1
1115
+ for r in seed_rows
1116
+ if r and (r.get("search_volume") or 0) > 0
1117
+ )
1118
+ # Topicality anchors come from the SEEDS ONLY (same set that gates the
1119
+ # expansion above).
1120
+ market_tokens = seed_anchor_tokens
1121
+
1122
+ heads = pick_serp_heads(base_ranked)
1123
+ observations = [
1124
+ extract_serp_observation(
1125
+ head,
1126
+ client.call("serp", {"keyword": head, "depth": SERP_DEPTH, **locale}).get("result") or [],
1127
+ )
1128
+ for head in heads
1129
+ ]
1130
+
1131
+ keyword_volumes = {
1132
+ row["keyword"]: int(row.get("search_volume") or 0) for row in base_ranked
1133
+ }
1134
+ competitor_rows = tally_competitors(
1135
+ observations, inputs.seed_domains, keyword_volumes
1136
+ )
1137
+ confirmed = confirmed_competitors(
1138
+ competitor_rows, inputs.seed_domains, seed_keywords=inputs.seeds
1139
+ )
1140
+ named_domains = {registrable_domain(domain) for domain in inputs.seed_domains}
1141
+ for row in competitor_rows:
1142
+ row["confirmation_status"] = (
1143
+ "founder_seed" if row["domain"] in named_domains and row["domain"] in confirmed
1144
+ else "confirmed" if row["domain"] in confirmed
1145
+ else "unconfirmed"
1146
+ )
1147
+ if row["confirmation_status"] == "unconfirmed" and row["kind"] == "candidate":
1148
+ row["confirmation_reason"] = "insufficient_category_relevant_serp_overlap"
1149
+
1150
+ # Competitor footprints stay a SEPARATE evidence set; only rows sharing a
1151
+ # category-bearing token with the market join the pool (annotated), so a
1152
+ # broad competitor cannot inject an unrelated high-volume query into the
1153
+ # demand tables (review finding #2).
1154
+ footprint_rows: list[dict[str, Any]] = []
1155
+ topical_rows: list[dict[str, Any]] = []
1156
+ for domain in confirmed:
1157
+ response = client.call(
1158
+ "ranked-keywords",
1159
+ {"target": domain, "limit": RANKED_KEYWORDS_LIMIT, **locale},
1160
+ )
1161
+ entry = (response.get("result") or [{}])[0]
1162
+ for item in entry.get("items") or []:
1163
+ row = normalize_ranked_row(item, domain)
1164
+ if row is None:
1165
+ continue
1166
+ topical = is_topical(row["keyword"], market_tokens) and not (
1167
+ set(row["keyword"].split()) & veto_tokens
1168
+ )
1169
+ footprint_rows.append(
1170
+ {
1171
+ **row,
1172
+ "type": "competitor_keyword",
1173
+ "domain": domain,
1174
+ "topical": topical,
1175
+ }
1176
+ )
1177
+ if topical:
1178
+ topical_rows.append({**row, "from_competitor_footprint": True})
1179
+
1180
+ market_pool = merge_keyword_rows(market_pool, topical_rows)
1181
+ ranked_pool = rank_keywords(market_pool)
1182
+
1183
+ overlap_domains: list[str] = []
1184
+ if confirmed:
1185
+ overlap = client.call(
1186
+ "competitors-domain",
1187
+ {"target": confirmed[0], "limit": 20, **locale},
1188
+ )
1189
+ entry = (overlap.get("result") or [{}])[0]
1190
+ for item in entry.get("items") or []:
1191
+ domain = registrable_domain(item.get("domain") or "")
1192
+ if domain and classify_domain(domain) == "candidate" and domain not in confirmed:
1193
+ overlap_domains.append(domain)
1194
+
1195
+ # Budget-simulation keywords come STRICTLY from the seed+expansion pool —
1196
+ # never from competitor footprints, even topical ones. They also estimate
1197
+ # CATEGORY economics, so competitor-brand queries (bidding on 'miro …' is
1198
+ # a separate tactic) and explicit free-intent queries are excluded; when
1199
+ # nothing trustworthy remains the simulation is OMITTED rather than
1200
+ # silently bypassing the filters.
1201
+ competitor_brands = {
1202
+ brand_token(d)
1203
+ for d in confirmed + [registrable_domain(x) for x in inputs.seed_domains]
1204
+ }
1205
+ competitor_brands.discard("")
1206
+ # A brand token that is itself category vocabulary (whiteboard.chat ->
1207
+ # 'whiteboard') must not exclude core category queries.
1208
+ competitor_brands = {
1209
+ brand
1210
+ for brand in competitor_brands
1211
+ if not any(_tokens_match(brand, anchor) for anchor in seed_anchor_tokens)
1212
+ }
1213
+ _BRAND_SUFFIXES = ("com", "io", "app", "ai", "co", "net", "org")
1214
+
1215
+ def _token_is_brand(token: str) -> bool:
1216
+ normalized = re.sub(r"[^a-z0-9]", "", token)
1217
+ for brand in competitor_brands:
1218
+ if normalized == brand:
1219
+ return True
1220
+ # 'monday.com' -> 'mondaycom' must match brand 'monday'.
1221
+ if normalized.startswith(brand) and normalized[len(brand):] in _BRAND_SUFFIXES:
1222
+ return True
1223
+ return False
1224
+
1225
+ def sim_eligible(row: dict[str, Any]) -> bool:
1226
+ if any(_token_is_brand(token) for token in row["keyword"].split()):
1227
+ return False
1228
+ if has_free_intent(row["keyword"]):
1229
+ return False
1230
+ return True
1231
+
1232
+ eligible = [r for r in base_ranked if sim_eligible(r)]
1233
+ sim_rows = eligible[:BUDGET_SIM_KEYWORDS]
1234
+ requested_keywords = scenario["keywords"] if scenario else [r["keyword"] for r in eligible]
1235
+ sim_keywords = requested_keywords[:BUDGET_SIM_KEYWORDS]
1236
+ simulation_omitted_reason = None
1237
+ if not sim_keywords:
1238
+ simulation_omitted_reason = (
1239
+ "no trustworthy simulation keywords remained after excluding "
1240
+ "competitor-brand and free-intent queries"
1241
+ )
1242
+ bid = median([r.get("cpc_high") for r in sim_rows]) or median(
1243
+ [r.get("cpc") for r in sim_rows]
1244
+ ) or 1.0
1245
+ # The forecast endpoint requires whole USD, unlike observed auction CPCs.
1246
+ # Choose a valid exploratory assumption before spending, never after failure.
1247
+ bid_normalized_to_int = not scenario and float(bid) != math.ceil(float(bid))
1248
+ # Preserve the numeric representation used by existing successful cache keys.
1249
+ bid = scenario["bid"] if scenario else float(max(1, min(math.ceil(float(bid)), 500)))
1250
+ match = scenario["match"] if scenario else "exact"
1251
+ budget_rows: list[dict[str, Any]] = []
1252
+ if sim_keywords:
1253
+ ad_traffic = client.call(
1254
+ "ad-traffic",
1255
+ {"keywords": sim_keywords, "bid": bid, "match": match, **locale},
1256
+ )
1257
+ budget_rows = build_budget_rows(
1258
+ sim_keywords, bid, ad_traffic.get("result") or [], inputs.test_budget_usd
1259
+ )
1260
+ simulation_coverage = forecast.coverage(
1261
+ requested=requested_keywords, submitted=sim_keywords, locale=locale, bid=bid, match=match, scenario=scenario,
1262
+ excluded=[] if scenario else [{"keyword": r["keyword"], "reason": "competitor_brand_or_free_education_intent"}
1263
+ for r in base_ranked if not sim_eligible(r)],
1264
+ )
1265
+ simulation_coverage["response_rows"] = len(budget_rows)
1266
+ simulation_coverage["response_status"] = "available" if budget_rows else "unavailable"
1267
+
1268
+ for domain in overlap_domains:
1269
+ competitor_rows.append(
1270
+ {
1271
+ "type": "competitor_domain",
1272
+ "domain": domain,
1273
+ "kind": "candidate",
1274
+ "found_via": ["keyword_overlap"],
1275
+ "serp_paid_appearances": 0,
1276
+ "serp_organic_appearances": 0,
1277
+ "score": 0,
1278
+ "tier": "B",
1279
+ }
1280
+ )
1281
+
1282
+ # Durable spend provenance comes from the response ledger, not this
1283
+ # attempt's counters — a cache-only replay reports the original metered
1284
+ # spend and renders byte-identical artifacts (review finding #1).
1285
+ totals = client.ledger_totals()
1286
+ retrieved_on = (
1287
+ totals["first_retrieved_on"]
1288
+ if totals["first_retrieved_on"] == totals["last_retrieved_on"]
1289
+ else f"{totals['first_retrieved_on']}..{totals['last_retrieved_on']}"
1290
+ )
1291
+
1292
+ evidence = {
1293
+ "schema_version": SCHEMA_VERSION,
1294
+ "section_id": SECTION_ID,
1295
+ "generated_by": Path(__file__).name,
1296
+ "retrieved_on": retrieved_on,
1297
+ "locale": locale,
1298
+ "simulation": simulation_coverage,
1299
+ "request_hashes": sorted(p.stem for p in client._ledger_entries),
1300
+ "inputs": {
1301
+ "seed_keywords": inputs.seeds,
1302
+ "seed_domains": inputs.seed_domains,
1303
+ "test_budget_usd": inputs.test_budget_usd,
1304
+ "bid_normalized_to_int": bid_normalized_to_int,
1305
+ "scenario_sha256": simulation_coverage["scenario_sha256"],
1306
+ "simulation_omitted_reason": simulation_omitted_reason,
1307
+ "expansion_rejected_count": len(rejected_expansion),
1308
+ "expansion_veto_rejected_count": len(veto_rejected),
1309
+ },
1310
+ "rows": (
1311
+ ranked_pool
1312
+ + footprint_rows
1313
+ + expansion_rejected_rows
1314
+ + observations
1315
+ + competitor_rows
1316
+ + budget_rows
1317
+ ),
1318
+ "totals": {
1319
+ "tasks_spent": totals["tasks_spent"],
1320
+ "estimated_cost_usd": totals["estimated_cost_usd"],
1321
+ },
1322
+ }
1323
+
1324
+ section_markdown = render_section(
1325
+ locale=locale,
1326
+ seeds=inputs.seeds,
1327
+ seed_rows_alive=seed_rows_alive,
1328
+ keyword_rows=market_pool,
1329
+ footprint_rows=footprint_rows,
1330
+ observations=observations,
1331
+ competitor_rows=competitor_rows,
1332
+ confirmed=confirmed,
1333
+ budget_rows=budget_rows,
1334
+ overlap_domains=overlap_domains,
1335
+ totals=totals,
1336
+ test_budget_usd=inputs.test_budget_usd,
1337
+ retrieved_on=retrieved_on,
1338
+ expansion_rejected_count=len(rejected_expansion),
1339
+ expansion_veto_rejected_count=len(veto_rejected),
1340
+ simulation_omitted_reason=simulation_omitted_reason,
1341
+ simulation_coverage=simulation_coverage,
1342
+ )
1343
+ return {
1344
+ "evidence": evidence,
1345
+ "section_markdown": section_markdown,
1346
+ "totals": totals,
1347
+ "attempt": {
1348
+ "tasks_spent": client.tasks_spent,
1349
+ "cache_hits": client.cache_hits,
1350
+ },
1351
+ }
1352
+
1353
+
1354
+ def _write_atomic(path: Path, content: str) -> None:
1355
+ _load_script("research-artifact-io.py", "pmr_artifact_io").write_atomic(path, content)
1356
+
1357
+
1358
+ def run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
1359
+ workspace = workspace.resolve()
1360
+ runtime = workspace / RUNTIME_REL
1361
+ cache_dir = runtime / "cache"
1362
+ runtime.mkdir(parents=True, exist_ok=True)
1363
+
1364
+ api_key = (os.environ.get("OS_API_KEY") or "").strip()
1365
+ if not api_key:
1366
+ raise KeywordEconomicsError("OS_API_KEY is required")
1367
+ os_url = (os.environ.get("OS_URL") or DEFAULT_OS_URL).strip() or DEFAULT_OS_URL
1368
+
1369
+ seeds = parse_terms(args.seed_keywords, MAX_SEED_KEYWORDS)
1370
+ if not seeds:
1371
+ raise KeywordEconomicsError(
1372
+ "at least one seed keyword is required for the assigned search scope"
1373
+ )
1374
+ seed_domains = parse_terms(args.seed_domains or "", MAX_SEED_DOMAINS)
1375
+
1376
+ inputs = RunInputs(
1377
+ seeds=seeds,
1378
+ seed_domains=seed_domains,
1379
+ location_code=args.location_code,
1380
+ language_code=args.language_code,
1381
+ test_budget_usd=args.test_budget_usd,
1382
+ scenario=json.loads(Path(args.scenario).read_text()) if getattr(args, "scenario", None) else None,
1383
+ )
1384
+ client = ToolsClient(
1385
+ os_url=os_url,
1386
+ api_key=api_key,
1387
+ cache_dir=cache_dir,
1388
+ max_tasks=args.max_tasks,
1389
+ )
1390
+
1391
+ io = _load_script("research-artifact-io.py", "pmr_artifact_io")
1392
+ publication = _load_script("keyword-locale-publication.py", "pmr_keyword_publication")
1393
+ upsert = _load_script("upsert-markdown-section.py", "pmr_upsert_markdown_section")
1394
+ persist = _load_script("persist-research-report.py", "pmr_persist_research_report")
1395
+ key = publication.locale_key(inputs.location_code, inputs.language_code)
1396
+ io.bind_context(workspace, "keyword_economics", context_file=getattr(args, "context_file", None))
1397
+ context_hash = io.context_binding(workspace, "keyword_economics")
1398
+ with io.file_lock(runtime / "locales" / key / ".run.lock", timeout=1):
1399
+ publication.begin(workspace, key, context_hash, io)
1400
+ try:
1401
+ outcome = run_pipeline(client, inputs)
1402
+ return publication.publish(workspace, key, context_hash, outcome, io, upsert, persist)
1403
+ except Exception:
1404
+ publication.failed(workspace, key, context_hash, io)
1405
+ raise
1406
+
1407
+
1408
+ def main() -> int:
1409
+ parser = argparse.ArgumentParser(description=__doc__)
1410
+ parser.add_argument("--context-file", type=Path, help="Explicit additional context for this measurement.")
1411
+ parser.add_argument("--workspace", type=Path, required=True)
1412
+ parser.add_argument(
1413
+ "--seed-keywords",
1414
+ required=True,
1415
+ help="Comma/semicolon-separated candidate search phrases (3-20).",
1416
+ )
1417
+ parser.add_argument(
1418
+ "--seed-domains",
1419
+ default="",
1420
+ help="Comma-separated competitor domains supplied for this research.",
1421
+ )
1422
+ parser.add_argument("--location-code", type=int, default=2840)
1423
+ parser.add_argument("--language-code", default="en")
1424
+ parser.add_argument("--test-budget-usd", type=float, default=None, help="Optional task-specific budget for conditional arithmetic; no budget is chosen by this runner.")
1425
+ parser.add_argument("--max-tasks", type=int, default=MAX_TASKS_PER_RUN)
1426
+ parser.add_argument("--scenario", type=Path, help="Explicit homogeneous forecast scenario JSON; validated before paid calls")
1427
+ args = parser.parse_args()
1428
+
1429
+ try:
1430
+ state = run(args.workspace, args)
1431
+ except (KeywordEconomicsError, ValueError, OSError, RuntimeError) as exc:
1432
+ print(f"{DISPLAY_NAME}: {exc}", file=sys.stderr)
1433
+ return 1
1434
+ print(
1435
+ f"{DISPLAY_NAME}: {state['status']} — {state['tasks_spent']} metered tasks "
1436
+ f"(≈ ${state['estimated_cost_usd']:.2f}), evidence at {state['evidence_path']}, "
1437
+ f"publication {state['publication_mode']} {state['report_commit_sha'][:12]}"
1438
+ )
1439
+ return 0 if state["status"] == "completed" else 1
1440
+
1441
+
1442
+ if __name__ == "__main__":
1443
+ raise SystemExit(main())