@egbe-ai/opencode 0.0.0-stage → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (479) hide show
  1. package/LICENSE +4 -0
  2. package/README.md +124 -2
  3. package/index.js +5 -0
  4. package/package.json +55 -4
  5. package/plugin.js +230 -0
  6. package/session-log.js +315 -0
  7. package/skills/.egbe-catalog/manifest.json +1 -0
  8. package/skills/egbe-os-analytics/SKILL.md +499 -0
  9. package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
  10. package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
  11. package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
  12. package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
  13. package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
  14. package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
  15. package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
  16. package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
  17. package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
  18. package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
  19. package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
  20. package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
  21. package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
  22. package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
  23. package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
  24. package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
  25. package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
  26. package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
  27. package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
  28. package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
  29. package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
  30. package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
  31. package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
  32. package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
  33. package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
  34. package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
  35. package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
  36. package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
  37. package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
  38. package/skills/egbe-os-db/SKILL.md +174 -0
  39. package/skills/egbe-os-deploy/SKILL.md +412 -0
  40. package/skills/egbe-os-design/SKILL.md +182 -0
  41. package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
  42. package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
  43. package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
  44. package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
  45. package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
  46. package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
  47. package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
  48. package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
  49. package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
  50. package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
  51. package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
  52. package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
  53. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
  54. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
  55. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
  56. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
  57. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
  58. package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
  59. package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
  60. package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
  61. package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
  62. package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
  63. package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
  64. package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
  65. package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
  66. package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
  67. package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
  68. package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
  69. package/skills/egbe-os-destination-readiness/references/report.md +116 -0
  70. package/skills/egbe-os-domains/SKILL.md +189 -0
  71. package/skills/egbe-os-errors/SKILL.md +259 -0
  72. package/skills/egbe-os-git/SKILL.md +275 -0
  73. package/skills/egbe-os-git/references/issues.md +33 -0
  74. package/skills/egbe-os-google-ads/SKILL.md +163 -0
  75. package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
  76. package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
  77. package/skills/egbe-os-google-ads/references/conversions.md +153 -0
  78. package/skills/egbe-os-google-ads/references/delivery.md +128 -0
  79. package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
  80. package/skills/egbe-os-google-ads/references/reporting.md +105 -0
  81. package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
  82. package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
  83. package/skills/egbe-os-google-auth/SKILL.md +285 -0
  84. package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
  85. package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
  86. package/skills/egbe-os-images/SKILL.md +163 -0
  87. package/skills/egbe-os-images/references/stock-images.md +21 -0
  88. package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
  89. package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
  90. package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
  91. package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
  92. package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
  93. package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
  94. package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
  95. package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
  96. package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
  97. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
  98. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
  99. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
  100. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
  101. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
  102. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
  103. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
  104. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
  105. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
  106. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
  107. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
  108. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
  109. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
  110. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
  111. package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
  112. package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
  113. package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
  114. package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
  115. package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
  116. package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
  117. package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
  118. package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
  119. package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
  120. package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
  121. package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
  122. package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
  123. package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
  124. package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
  125. package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
  126. package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
  127. package/skills/egbe-os-llm/SKILL.md +160 -0
  128. package/skills/egbe-os-mail/SKILL.md +180 -0
  129. package/skills/egbe-os-me/SKILL.md +119 -0
  130. package/skills/egbe-os-meta-ads/SKILL.md +114 -0
  131. package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
  132. package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
  133. package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
  134. package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
  135. package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
  136. package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
  137. package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
  138. package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
  139. package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
  140. package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
  141. package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
  142. package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
  143. package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
  144. package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
  145. package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
  146. package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
  147. package/skills/egbe-os-notifications/SKILL.md +78 -0
  148. package/skills/egbe-os-openai-ads/SKILL.md +149 -0
  149. package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
  150. package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
  151. package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
  152. package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
  153. package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
  154. package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
  155. package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
  156. package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
  157. package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
  158. package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
  159. package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
  160. package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
  161. package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
  162. package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
  163. package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
  164. package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
  165. package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
  166. package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
  167. package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
  168. package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
  169. package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
  170. package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
  171. package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
  172. package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
  173. package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
  174. package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
  175. package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
  176. package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
  177. package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
  178. package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
  179. package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
  180. package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
  181. package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
  182. package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
  183. package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
  184. package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
  185. package/skills/egbe-os-payments/SKILL.md +215 -0
  186. package/skills/egbe-os-payments/agents/openai.yaml +4 -0
  187. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
  188. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
  189. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
  190. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
  191. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
  192. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
  193. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
  194. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
  195. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
  196. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
  197. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
  198. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
  199. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
  200. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
  201. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
  202. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
  203. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
  204. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
  205. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
  206. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
  207. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
  208. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
  209. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
  210. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
  211. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
  212. package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
  213. package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
  214. package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
  215. package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
  216. package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
  217. package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
  218. package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
  219. package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
  220. package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
  221. package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
  222. package/skills/egbe-os-playbook/SKILL.md +79 -0
  223. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
  224. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
  225. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
  226. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
  227. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
  228. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
  229. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
  230. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
  231. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
  232. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
  233. package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
  234. package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
  235. package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
  236. package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
  237. package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
  238. package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
  239. package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
  240. package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
  241. package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
  242. package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
  243. package/skills/egbe-os-playbook/playbooks/product.md +67 -0
  244. package/skills/egbe-os-playbook/references/catalog.md +127 -0
  245. package/skills/egbe-os-playbook/references/changes.md +167 -0
  246. package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
  247. package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
  248. package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
  249. package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
  250. package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
  251. package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
  252. package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
  253. package/skills/egbe-os-product-auth/SKILL.md +151 -0
  254. package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
  255. package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
  256. package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
  257. package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
  258. package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
  259. package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
  260. package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
  261. package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
  262. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
  263. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
  264. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
  265. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
  266. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
  267. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
  268. package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
  269. package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
  270. package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
  271. package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
  272. package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
  273. package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
  274. package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
  275. package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
  276. package/skills/egbe-os-product-build/SKILL.md +111 -0
  277. package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
  278. package/skills/egbe-os-product-build/references/capabilities.md +25 -0
  279. package/skills/egbe-os-product-build/references/engineering.md +72 -0
  280. package/skills/egbe-os-product-build/references/execution.md +55 -0
  281. package/skills/egbe-os-product-build/references/foundations.md +93 -0
  282. package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
  283. package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
  284. package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
  285. package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
  286. package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
  287. package/skills/egbe-os-product-market-research/SKILL.md +159 -0
  288. package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
  289. package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
  290. package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
  291. package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
  292. package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
  293. package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
  294. package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
  295. package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
  296. package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
  297. package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
  298. package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
  299. package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
  300. package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
  301. package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
  302. package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
  303. package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
  304. package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
  305. package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
  306. package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
  307. package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
  308. package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
  309. package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
  310. package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
  311. package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
  312. package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
  313. package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
  314. package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
  315. package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
  316. package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
  317. package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
  318. package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
  319. package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
  320. package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
  321. package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
  322. package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
  323. package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
  324. package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
  325. package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
  326. package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
  327. package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
  328. package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
  329. package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
  330. package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
  331. package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
  332. package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
  333. package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
  334. package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
  335. package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
  336. package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
  337. package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
  338. package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
  339. package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
  340. package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
  341. package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
  342. package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
  343. package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
  344. package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
  345. package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
  346. package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
  347. package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
  348. package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
  349. package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
  350. package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
  351. package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
  352. package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
  353. package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
  354. package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
  355. package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
  356. package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
  357. package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
  358. package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
  359. package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
  360. package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
  361. package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
  362. package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
  363. package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
  364. package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
  365. package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
  366. package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
  367. package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
  368. package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
  369. package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
  370. package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
  371. package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
  372. package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
  373. package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
  374. package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
  375. package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
  376. package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
  377. package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
  378. package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
  379. package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
  380. package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
  381. package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
  382. package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
  383. package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
  384. package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
  385. package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
  386. package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
  387. package/skills/egbe-os-product-report/SKILL.md +150 -0
  388. package/skills/egbe-os-product-report/references/artifacts.md +84 -0
  389. package/skills/egbe-os-product-report/references/brief.md +34 -0
  390. package/skills/egbe-os-product-report/references/daily.md +76 -0
  391. package/skills/egbe-os-product-report/references/sources.md +68 -0
  392. package/skills/egbe-os-product-routines/SKILL.md +166 -0
  393. package/skills/egbe-os-product-routines/references/catalog.md +22 -0
  394. package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
  395. package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
  396. package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
  397. package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
  398. package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
  399. package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
  400. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
  401. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
  402. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
  403. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
  404. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
  405. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
  406. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
  407. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
  408. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
  409. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
  410. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
  411. package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
  412. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
  413. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
  414. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
  415. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
  416. package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
  417. package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
  418. package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
  419. package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
  420. package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
  421. package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
  422. package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
  423. package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
  424. package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
  425. package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
  426. package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
  427. package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
  428. package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
  429. package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
  430. package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
  431. package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
  432. package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
  433. package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
  434. package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
  435. package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
  436. package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
  437. package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
  438. package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
  439. package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
  440. package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
  441. package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
  442. package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
  443. package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
  444. package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
  445. package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
  446. package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
  447. package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
  448. package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
  449. package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
  450. package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
  451. package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
  452. package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
  453. package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
  454. package/skills/egbe-os-report-ui/SKILL.md +133 -0
  455. package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
  456. package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
  457. package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
  458. package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
  459. package/skills/egbe-os-report-ui/references/components.md +209 -0
  460. package/skills/egbe-os-research/SKILL.md +125 -0
  461. package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
  462. package/skills/egbe-os-scrape/SKILL.md +160 -0
  463. package/skills/egbe-os-search/SKILL.md +182 -0
  464. package/skills/egbe-os-search/references/google-places.md +81 -0
  465. package/skills/egbe-os-support/SKILL.md +149 -0
  466. package/skills/egbe-os-telemetry/SKILL.md +344 -0
  467. package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
  468. package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
  469. package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
  470. package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
  471. package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
  472. package/skills/egbe-os-tools/SKILL.md +180 -0
  473. package/skills/egbe-os-tools/references/theona.md +26 -0
  474. package/skills/egbe-os-wallet/SKILL.md +181 -0
  475. package/skills/egbe-setup/EGBE.md +484 -0
  476. package/skills/egbe-setup/SKILL.md +116 -0
  477. package/skills/egbe-setup/references/project.md +80 -0
  478. package/state.js +37 -0
  479. package/update.js +361 -0
@@ -0,0 +1,2514 @@
1
+ #!/usr/bin/env python3
2
+ """Collect a bounded competitor comparison sample for caller-scoped research.
3
+
4
+ Discovery records offer-query results, related-domain reports, supplied profiles,
5
+ comparison roles and ad observations. Collection caps and sample order are not
6
+ business-priority rules. The caller owns comparison criteria and interpretation;
7
+ source identity, profile URLs, aliases, cache and receipts remain validated.
8
+
9
+ Caching, spend provenance, budgets and retries mirror the other measured
10
+ runners: every provider response is cached, starts are journaled before the
11
+ call, replays add no spend.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import sys as _artifact_sys
17
+ from pathlib import Path as _ArtifactPath
18
+ _artifact_sys.path.insert(0, str(_ArtifactPath(__file__).resolve().parent))
19
+ from egbe_artifacts import artifact_path as _artifact_path
20
+
21
+ import runpy as _egbe_runpy
22
+ from pathlib import Path as _EgbePath
23
+ _egbe_adapter = _EgbePath(__file__).with_name("connection_identity.py")
24
+ _connection_headers = _egbe_runpy.run_path(str(_egbe_adapter))["connection_headers"] if _egbe_adapter.is_file() else lambda _url: {}
25
+
26
+ import argparse
27
+ import hashlib
28
+ import importlib.util
29
+ import json
30
+ import math
31
+ import os
32
+ import re
33
+ import sys
34
+ import time
35
+ import urllib.error
36
+ import urllib.parse
37
+ import urllib.request
38
+ from dataclasses import asdict, dataclass, field
39
+ from datetime import date, timedelta
40
+ from pathlib import Path
41
+ from types import ModuleType
42
+ from typing import Any, Callable
43
+
44
+ SCHEMA_VERSION = 1
45
+ SECTION_ID = "competitor_set"
46
+ SECTION_HEADING = "Competitor set"
47
+ DISPLAY_NAME = "Competitor Set"
48
+
49
+ DEFAULT_OS_URL = "https://os.hz.egbe.dev"
50
+ RUNTIME_REL = Path(_artifact_path("egbe-docs/research/market/work")) / SECTION_ID
51
+ EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/competitors.json"))
52
+ REPORT_REL = Path(_artifact_path("egbe-docs/research/market/full-report.md"))
53
+ INTAKE_REL = Path(_artifact_path("egbe-docs/research/market/intake.json"))
54
+ CANDIDATES_FILENAME = "candidates.json"
55
+ LISTICLES_FILENAME = "listicles.json"
56
+ PROFILES_FILENAME = "profiles.json"
57
+ DECISIONS_FILENAME = "decisions.json"
58
+ REVIEW_FILENAME = "review.json"
59
+ DECISIONS_SCHEMA_VERSION = 2
60
+
61
+ MAX_COMPETITORS = 10
62
+ MAX_NAMED_DOMAINS = 10
63
+ MAX_PAGE_DOMAINS = 10
64
+ MAX_KEYWORD_DOMAINS = 20
65
+ MAX_CATEGORY_PHRASES = 3
66
+ MAX_BRAND_QUERIES = 3
67
+ MAX_CANDIDATES = 150
68
+ # Bounds of the agent's enrichment work (profiles per round, domains it may
69
+ # add from its own reading, listicle pages it reads).
70
+ # At most 40 profiles in all: the agent's reading, not the pool, is the cost.
71
+ MAX_PROFILES_ROUND1 = 25
72
+ MAX_PROFILES_ROUND2 = 10
73
+ MAX_AGENT_ADDITIONS = 5
74
+ MAX_LISTICLES_TO_READ = 12
75
+ LISTICLES_PER_QUERY = 5
76
+ MAX_LISTICLE_ENTRIES = 60
77
+ ROUND2_SEEDS = 3
78
+ MAX_AD_CHECKS = 25
79
+ AD_CHECK_MONTHS = 6
80
+ AD_CHECK_EXTRA_MARKETS = 2
81
+ KEYWORD_ADS_ROWS = 50
82
+ # SpyFu's keyword ad history reaches back years; only an advertiser seen on a
83
+ # phrase within this window counts as buying traffic on it now.
84
+ KEYWORD_ADS_RECENT_DAYS = 365
85
+ LISTICLES_PER_SITE = 2
86
+ COMPETITORS_DOMAIN_LIMIT = 20
87
+ SPYFU_COMPETITORS = 20
88
+ # One result page: deeper SERP tasks are assembled from several pages and
89
+ # DataForSEO rejects the whole task when one page cannot be retrieved
90
+ # (status 40106, "partial results"), observed live on 2026-10-01.
91
+ SERP_DEPTH = 10
92
+ BRAVE_COUNT = 20
93
+ META_ADS_LIMIT = 25
94
+ GTM_CONTAINERS_PER_SITE = 2
95
+ FETCH_MAX_BYTES = 1_000_000
96
+ FETCH_TIMEOUT_SECONDS = 25.0
97
+ RETRY_DELAY_SECONDS = 20.0
98
+ EXCLUDED_ROWS_RENDERED = 25
99
+ PROFILED_ROWS_RENDERED = 40
100
+ _TASK_ERROR_RE = re.compile(r"HTTP 400 .*dataforseo_task_error", re.DOTALL)
101
+
102
+ MAX_DATAFORSEO_TASKS_PER_ATTEMPT = 15
103
+ MAX_SEARCH_CALLS_PER_ATTEMPT = 12
104
+ MAX_SPYFU_CALLS_PER_ATTEMPT = 40
105
+ MAX_META_SEARCHES_PER_ATTEMPT = 25
106
+ MAX_FETCHES_PER_ATTEMPT = 60
107
+
108
+ DEFAULT_LOCATION_CODE = 2840
109
+ DEFAULT_LANGUAGE_CODE = "en"
110
+ DEFAULT_MARKET = "US"
111
+ DEFAULT_EU_COUNTRY = "DE"
112
+ # DataForSEO location codes for the markets an offer card usually names.
113
+ LOCATION_CODES = {
114
+ "US": 2840, "UK": 2826, "CA": 2124, "AU": 2036, "NZ": 2554, "IE": 2372, "DE": 2276, "FR": 2250,
115
+ "ES": 2724, "IT": 2380, "NL": 2528, "SE": 2752, "CZ": 2203, "PL": 2616, "PT": 2620, "AT": 2040,
116
+ "CH": 2756, "BE": 2056, "DK": 2208, "NO": 2578, "FI": 2246, "IN": 2356, "SG": 2702, "ZA": 2710,
117
+ "BR": 2076, "MX": 2484,
118
+ }
119
+ # SpyFu research markets (SpyFu says UK, not GB).
120
+ SPYFU_MARKETS = frozenset({
121
+ "AR", "AT", "AU", "BE", "BR", "CA", "CH", "DE", "DK", "ES", "FR", "IE", "IN", "IT", "JP", "MX",
122
+ "NL", "NO", "NZ", "PL", "PT", "SE", "SG", "TR", "UA", "UK", "US", "ZA",
123
+ })
124
+ EU_COUNTRIES = frozenset({
125
+ "AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE", "IT", "LV", "LT",
126
+ "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE",
127
+ })
128
+
129
+ DECISION_TIERS = ("direct", "indirect", "substitute", "excluded")
130
+ OFFER_MATCHES = ("same", "comparable", "different", "none")
131
+ RESOLUTIONS = ("accepted", "rejected")
132
+ MISSING_RESOLUTIONS = ("added", "rejected")
133
+ PROVENANCE_ORDER = ("operator_named", "product_page", "discovered", "keyword_economics", "agent_research")
134
+ AD_ORDER = {"ads_seen": 0, "pixels_only": 1, "unknown": 2, "none_detected": 3}
135
+ AD_LABELS = {"ads_seen": "ads seen", "pixels_only": "ad pixels only", "unknown": "ad check inconclusive", "none_detected": "no ads detected"}
136
+ GROUP_LABELS = {0: "direct", 1: "indirect", 2: "other scoped"}
137
+ # Tag channels from the Channel Footprint signature list that mean paid media.
138
+ AD_TAG_CHANNELS = frozenset({
139
+ "meta_ads", "google_ads", "reddit_ads", "linkedin_ads", "tiktok_ads", "microsoft_ads", "x_ads",
140
+ "pinterest_ads", "snap_ads", "quora_ads", "display_native",
141
+ })
142
+
143
+ # Signal strength: offer signals (ads and rankings on the offer's own search
144
+ # phrases, listicle mentions) outweigh relationship hits. It orders the
145
+ # profiling list and breaks ties; it never decides a tier, and naming a
146
+ # competitor (founder or product page) adds nothing.
147
+ SCORE_CATEGORY_PAID = 3.0
148
+ SCORE_KEYWORD_ADS = 2.0
149
+ SCORE_CATEGORY_ORGANIC = 2.0
150
+ SCORE_SECOND_ENGINE = 1.0
151
+ SCORE_BRAND_QUERY = 1.0
152
+ SCORE_LISTICLE = 1.0
153
+ SCORE_RELATIONSHIP = 1.0
154
+ SCORE_KEYWORD_ECONOMICS = 1.0
155
+ CAP_CATEGORY_PAID = 2
156
+ CAP_KEYWORD_ADS = 3
157
+ CAP_CATEGORY_ORGANIC = 3
158
+ CAP_SECOND_ENGINE = 3
159
+ CAP_BRAND_QUERY = 2
160
+ CAP_LISTICLE = 3
161
+ CAP_RELATIONSHIP = 4
162
+
163
+ _LISTICLE_RE = re.compile(r"\b(alternatives?|best|top \d+|vs\.?|comparison|compare)\b", re.IGNORECASE)
164
+ _DOMAIN_RE = re.compile(r"^[a-z0-9][a-z0-9.-]{0,251}\.[a-z]{2,24}$")
165
+ _URL_RE = re.compile(r"^https?://[^\s]+$", re.IGNORECASE)
166
+ _DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
167
+ _SEARCH_PAGE_KEYS = {"q", "query", "search", "keyword", "keywords", "term"}
168
+ PAGES_PER_CANDIDATE = 3
169
+
170
+
171
+ class CompetitorSetError(RuntimeError):
172
+ """Terminal runner failure; runtime state is preserved for resume."""
173
+
174
+
175
+ class BudgetExhausted(CompetitorSetError):
176
+ """A per-attempt call budget is spent; always propagates so a rerun
177
+ resumes from the cache instead of publishing a partial search."""
178
+
179
+
180
+ def _load_script(filename: str, module_name: str) -> ModuleType:
181
+ path = Path(__file__).resolve().parent / filename
182
+ spec = importlib.util.spec_from_file_location(module_name, path)
183
+ if spec is None or spec.loader is None:
184
+ raise CompetitorSetError(f"cannot load bundled script {filename}")
185
+ module = importlib.util.module_from_spec(spec)
186
+ sys.modules[spec.name] = module
187
+ previous = sys.dont_write_bytecode
188
+ sys.dont_write_bytecode = True
189
+ try:
190
+ spec.loader.exec_module(module)
191
+ finally:
192
+ sys.dont_write_bytecode = previous
193
+ return module
194
+
195
+
196
+ # Cache entries are stored gzip-compressed; plain entries of older runs replay too.
197
+ IO = _load_script("research-artifact-io.py", "pmr_competitor_set_io")
198
+ _KEYWORD_ECONOMICS: ModuleType | None = None
199
+ _CHANNEL_FOOTPRINT: ModuleType | None = None
200
+
201
+
202
+ def _keyword_economics() -> ModuleType:
203
+ """Domain normalization and the noise/directory lists live in the Keyword
204
+ Economics runner; load them from there instead of copying the lists."""
205
+ global _KEYWORD_ECONOMICS
206
+ if _KEYWORD_ECONOMICS is None:
207
+ _KEYWORD_ECONOMICS = _load_script("run-keyword-economics.py", "pmr_keyword_economics_for_competitor_set")
208
+ return _KEYWORD_ECONOMICS
209
+
210
+
211
+ def _channel_footprint() -> ModuleType:
212
+ """The ad-tag signatures, the SpyFu spend row and the Meta identity rules
213
+ live in the Channel Footprint runner; the ad check reuses them."""
214
+ global _CHANNEL_FOOTPRINT
215
+ if _CHANNEL_FOOTPRINT is None:
216
+ _CHANNEL_FOOTPRINT = _load_script("run-channel-footprint.py", "pmr_channel_footprint_for_competitor_set")
217
+ return _CHANNEL_FOOTPRINT
218
+
219
+
220
+ def registrable_domain(value: str) -> str:
221
+ return _keyword_economics().registrable_domain(value)
222
+
223
+
224
+ def classify_domain(domain: str) -> str:
225
+ return _keyword_economics().classify_domain(domain)
226
+
227
+
228
+ def brand_token(domain: str) -> str:
229
+ return _keyword_economics().brand_token(domain)
230
+
231
+
232
+ def hosted_seller(domain: str) -> bool:
233
+ """A seller's own subdomain on a course or creator host (kept whole by
234
+ the shared domain normalization)."""
235
+ return _keyword_economics().hosted_seller(domain)
236
+
237
+
238
+ def _domain(value: Any) -> str:
239
+ domain = registrable_domain(str(value or ""))
240
+ return domain if domain and _DOMAIN_RE.match(domain) else ""
241
+
242
+
243
+ def _text(value: Any, cap: int = 240) -> str:
244
+ text = " ".join(str(value if value is not None else "").split())
245
+ return text if len(text) <= cap else text[: cap - 1] + "…"
246
+
247
+
248
+ def _url(value: Any) -> str:
249
+ text = str(value or "").strip()
250
+ return text if _URL_RE.match(text) and len(text) <= 800 else ""
251
+
252
+
253
+ def _number(value: Any) -> float | None:
254
+ if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0:
255
+ raise ValueError("not a non-negative number")
256
+ return float(value)
257
+
258
+
259
+ def _money(value: float | None) -> str:
260
+ if value is None:
261
+ return "—"
262
+ return f"${value:,.0f}"
263
+
264
+
265
+ # ── OS client: metered provider calls + free page fetches ───────────────────
266
+
267
+ @dataclass
268
+ class OsClient:
269
+ os_url: str
270
+ api_key: str
271
+ cache_dir: Path
272
+ max_dataforseo_tasks: int = MAX_DATAFORSEO_TASKS_PER_ATTEMPT
273
+ max_search_calls: int = MAX_SEARCH_CALLS_PER_ATTEMPT
274
+ max_spyfu_calls: int = MAX_SPYFU_CALLS_PER_ATTEMPT
275
+ max_meta_searches: int = MAX_META_SEARCHES_PER_ATTEMPT
276
+ max_fetches: int = MAX_FETCHES_PER_ATTEMPT
277
+ dataforseo_tasks: int = 0
278
+ search_calls: int = 0
279
+ spyfu_calls: int = 0
280
+ meta_searches: int = 0
281
+ fetches: int = 0
282
+ cache_hits: int = 0
283
+ touched: set[str] = field(default_factory=set)
284
+ request: Callable[..., dict[str, Any]] | None = None
285
+ fetch: Callable[[str], tuple[int, str]] | None = None
286
+ sleep: Callable[[float], None] | None = None
287
+
288
+ def _http(self, path: str, payload: dict[str, Any], timeout: float, *, idempotency_key: str | None = None) -> dict[str, Any]:
289
+ url = f"{self.os_url.rstrip('/')}{path}"
290
+ headers = {
291
+ "Authorization": f"Bearer {self.api_key}",
292
+ "Content-Type": "application/json",
293
+ "Accept": "application/json",
294
+ }
295
+ if idempotency_key:
296
+ headers["Idempotency-Key"] = idempotency_key
297
+ req = urllib.request.Request(url, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST")
298
+ try:
299
+ for _name, _value in _connection_headers(req.full_url).items():
300
+ req.add_unredirected_header(_name, _value)
301
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
302
+ envelope = json.loads(resp.read().decode("utf-8"))
303
+ except urllib.error.HTTPError as exc:
304
+ detail = exc.read().decode("utf-8", errors="replace")[:500]
305
+ raise CompetitorSetError(f"OS {path} failed: HTTP {exc.code} {detail}") from exc
306
+ except urllib.error.URLError as exc:
307
+ raise CompetitorSetError(f"OS {path} unreachable: {exc.reason}") from exc
308
+ except (OSError, ValueError) as exc:
309
+ raise CompetitorSetError(f"OS {path} failed in transit: {type(exc).__name__}: {str(exc)[:200]}") from exc
310
+ data = envelope.get("data")
311
+ if envelope.get("code") not in (None, 200) or not isinstance(data, dict):
312
+ raise CompetitorSetError(f"OS {path} returned no data: {str(envelope)[:300]}")
313
+ return data
314
+
315
+ def _fetch_url(self, url: str) -> tuple[int, str]:
316
+ """Plain GET of a public page; the status and a bounded body."""
317
+ req = urllib.request.Request(url, headers={
318
+ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
319
+ "Accept": "text/html,application/xhtml+xml,application/javascript;q=0.9,*/*;q=0.8",
320
+ "Accept-Language": "en-US,en;q=0.9",
321
+ })
322
+ try:
323
+ for _name, _value in _connection_headers(req.full_url).items():
324
+ req.add_unredirected_header(_name, _value)
325
+ with urllib.request.urlopen(req, timeout=FETCH_TIMEOUT_SECONDS) as resp:
326
+ body = resp.read(FETCH_MAX_BYTES + 1)
327
+ status = int(getattr(resp, "status", 200) or 200)
328
+ except urllib.error.HTTPError as exc:
329
+ return exc.code, ""
330
+ except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
331
+ raise CompetitorSetError(f"fetch of {url} failed: {getattr(exc, 'reason', exc)}") from exc
332
+ return status, body[:FETCH_MAX_BYTES].decode("utf-8", errors="replace")
333
+
334
+ def _cache_read(self, cache_file: Path) -> dict[str, Any] | None:
335
+ try:
336
+ return IO.read_cache_entry(cache_file)
337
+ except (OSError, ValueError, EOFError):
338
+ IO.drop_cache_entry(cache_file)
339
+ return None
340
+
341
+ def _cache_write(self, cache_file: Path, entry: dict[str, Any]) -> None:
342
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
343
+ IO.write_cache_entry(cache_file, entry)
344
+
345
+ def _journal_start(self, digest: str) -> None:
346
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
347
+ with (self.cache_dir / "starts.log").open("a", encoding="utf-8") as handle:
348
+ handle.write(f"{date.today().isoformat()} {digest}\n")
349
+
350
+ def _prior_start_count(self, digest: str) -> int:
351
+ try:
352
+ lines = (self.cache_dir / "starts.log").read_text(encoding="utf-8").splitlines()
353
+ except OSError:
354
+ return 0
355
+ return sum(1 for line in lines if line.endswith(f" {digest}"))
356
+
357
+ def _metered(self, kind: str, path: str, payload: dict[str, Any], timeout: float, *, budget_attr: str, budget_max: int, label: str) -> tuple[dict[str, Any], str]:
358
+ key_raw = json.dumps({"kind": kind, "path": path, "payload": payload}, sort_keys=True)
359
+ digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
360
+ cache_file = self.cache_dir / f"{digest}.json"
361
+ self.touched.add(digest)
362
+ entry = self._cache_read(cache_file)
363
+ if entry is not None:
364
+ self.cache_hits += 1
365
+ response = entry.get("response")
366
+ return (response if isinstance(response, dict) else {}), entry.get("retrieved_on") or date.today().isoformat()
367
+
368
+ transport = self.request or self._http
369
+ attempts = 0
370
+ wire_key = f"{digest}-{self._prior_start_count(digest)}"
371
+ while True:
372
+ # Metered starts count BEFORE the transport call (fail closed) and
373
+ # are journaled durably: a failed start may still have billed.
374
+ if getattr(self, budget_attr) >= budget_max:
375
+ raise BudgetExhausted(
376
+ f"{label} budget exhausted ({budget_max} per attempt); rerun the same command to "
377
+ "continue from the cache"
378
+ )
379
+ setattr(self, budget_attr, getattr(self, budget_attr) + 1)
380
+ self._journal_start(digest)
381
+ try:
382
+ response = transport(path, payload, timeout, idempotency_key=wire_key)
383
+ break
384
+ except CompetitorSetError as exc:
385
+ if (kind == "dataforseo" and _TASK_ERROR_RE.search(str(exc))) or (
386
+ kind in ("spyfu", "meta") and re.search(r"HTTP 4(?:00|22)\b", str(exc))
387
+ ):
388
+ # A rejected task (bad input, partial result pages, a
389
+ # market SpyFu does not cover) is a fact about this query,
390
+ # not a transient failure: cache it so replays stay
391
+ # byte-stable and never re-spend.
392
+ response = {"unavailable": str(exc)[:300]}
393
+ break
394
+ if attempts >= 1 or not re.search(r"HTTP 5\d\d", str(exc)):
395
+ raise
396
+ attempts += 1
397
+ (self.sleep or time.sleep)(RETRY_DELAY_SECONDS)
398
+ retrieved_on = date.today().isoformat()
399
+ self._cache_write(cache_file, {"kind": kind, "path": path, "payload": payload, "response": response, "retrieved_on": retrieved_on})
400
+ return (response if isinstance(response, dict) else {}), retrieved_on
401
+
402
+ def dataforseo(self, operation: str, payload: dict[str, Any]) -> tuple[dict[str, Any], str]:
403
+ return self._metered(
404
+ "dataforseo", f"/tools/dataforseo/{operation}", payload, 180.0,
405
+ budget_attr="dataforseo_tasks", budget_max=self.max_dataforseo_tasks, label="DataForSEO task",
406
+ )
407
+
408
+ def brave(self, query: str, count: int, country: str) -> tuple[dict[str, Any], str]:
409
+ return self._metered(
410
+ "search", "/search/web/brave", {"query": query, "count": count, "country": country}, 60.0,
411
+ budget_attr="search_calls", budget_max=self.max_search_calls, label="web search",
412
+ )
413
+
414
+ def spyfu(self, operation: str, payload: dict[str, Any]) -> tuple[dict[str, Any], str]:
415
+ return self._metered(
416
+ "spyfu", f"/tools/spyfu/{operation}", payload, 90.0,
417
+ budget_attr="spyfu_calls", budget_max=self.max_spyfu_calls, label="SpyFu lookup",
418
+ )
419
+
420
+ def meta_library(self, payload: dict[str, Any]) -> tuple[dict[str, Any], str]:
421
+ return self._metered(
422
+ "meta", "/tools/meta-ad-library/search", payload, 90.0,
423
+ budget_attr="meta_searches", budget_max=self.max_meta_searches, label="Meta Ad Library search",
424
+ )
425
+
426
+ def fetch_page(self, url: str) -> tuple[dict[str, Any], str]:
427
+ """Free, cached GET of a public page. A timeout, a rate limit or a
428
+ server error is reported for this attempt and never cached."""
429
+ key_raw = json.dumps({"kind": "fetch", "url": url}, sort_keys=True)
430
+ digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
431
+ cache_file = self.cache_dir / f"{digest}.json"
432
+ self.touched.add(digest)
433
+ entry = self._cache_read(cache_file)
434
+ if entry is not None:
435
+ self.cache_hits += 1
436
+ cached = entry.get("response")
437
+ return (cached if isinstance(cached, dict) else {}), entry.get("retrieved_on") or date.today().isoformat()
438
+ if self.fetches >= self.max_fetches:
439
+ raise BudgetExhausted(f"page fetch budget exhausted ({self.max_fetches} per attempt); rerun the same command to continue from the cache")
440
+ self.fetches += 1
441
+ transport = self.fetch or self._fetch_url
442
+ retrieved_on = date.today().isoformat()
443
+ try:
444
+ status, body = transport(url)
445
+ except CompetitorSetError as exc:
446
+ return {"status": None, "body": "", "error": str(exc)[:200]}, retrieved_on
447
+ if status == 429 or status >= 500:
448
+ return {"status": status, "body": "", "error": f"HTTP {status}"}, retrieved_on
449
+ response = {"status": status, "body": body}
450
+ self._cache_write(cache_file, {"kind": "fetch", "url": url, "response": response, "retrieved_on": retrieved_on})
451
+ return response, retrieved_on
452
+
453
+ def ledger_totals(self) -> dict[str, Any]:
454
+ """Durable provenance over entries THIS execution touched. A cached
455
+ provider rejection is a journaled start, not a successful call: it
456
+ never counts as evidence that the market was searched."""
457
+ counts = {"dataforseo": 0, "search": 0, "spyfu": 0, "meta": 0, "fetch": 0}
458
+ rejected = 0
459
+ dates: list[str] = []
460
+ if self.cache_dir.is_dir():
461
+ for digest in sorted(self.touched):
462
+ try:
463
+ entry = IO.read_cache_entry(self.cache_dir / f"{digest}.json")
464
+ except (OSError, ValueError, EOFError):
465
+ continue
466
+ if not isinstance(entry, dict):
467
+ continue
468
+ kind = entry.get("kind")
469
+ response = entry.get("response") if isinstance(entry.get("response"), dict) else {}
470
+ if response.get("unavailable"):
471
+ rejected += 1
472
+ elif kind in counts:
473
+ counts[kind] += 1
474
+ if isinstance(entry.get("retrieved_on"), str):
475
+ dates.append(entry["retrieved_on"])
476
+ today = date.today().isoformat()
477
+ try:
478
+ starts_recorded = sum(1 for line in (self.cache_dir / "starts.log").read_text(encoding="utf-8").splitlines() if line)
479
+ except OSError:
480
+ starts_recorded = 0
481
+ provider_calls = counts["dataforseo"] + counts["search"] + counts["spyfu"] + counts["meta"]
482
+ return {
483
+ "dataforseo_calls": counts["dataforseo"],
484
+ "search_calls": counts["search"],
485
+ "spyfu_calls": counts["spyfu"],
486
+ "meta_searches": counts["meta"],
487
+ "page_fetches": counts["fetch"],
488
+ "provider_calls": provider_calls,
489
+ "rejected_calls": rejected,
490
+ "provider_starts_recorded": starts_recorded,
491
+ "first_retrieved_on": min(dates) if dates else today,
492
+ "last_retrieved_on": max(dates) if dates else today,
493
+ }
494
+
495
+
496
+ def _call_with_context(client: OsClient, subject: str, label: str, fn: Callable[[], Any]) -> Any:
497
+ try:
498
+ return fn()
499
+ except BudgetExhausted:
500
+ raise
501
+ except CompetitorSetError as exc:
502
+ raise CompetitorSetError(
503
+ f"{label} for `{subject}` failed: {exc} — this attempt had already started "
504
+ f"{client.dataforseo_tasks}/{client.max_dataforseo_tasks} DataForSEO task(s) and "
505
+ f"{client.search_calls}/{client.max_search_calls} web search(es); see "
506
+ "GET /me/limits?resource=dataforseo:tasks for remaining daily quota."
507
+ ) from exc
508
+
509
+
510
+ # ── inputs: the offer card and the two kinds of named competitors ───────────
511
+
512
+ def parse_terms(raw: str, cap: int) -> list[str]:
513
+ seen: list[str] = []
514
+ for term in (raw or "").split(","):
515
+ value = " ".join(term.split()).strip().lower()
516
+ if value and value not in seen:
517
+ seen.append(value)
518
+ return seen[:cap]
519
+
520
+
521
+ def parse_domains(raw: Any, cap: int) -> list[str]:
522
+ terms = raw if isinstance(raw, list) else str(raw or "").split(",")
523
+ domains: list[str] = []
524
+ for term in terms:
525
+ domain = _domain(term)
526
+ if domain and domain not in domains:
527
+ domains.append(domain)
528
+ return domains[:cap]
529
+
530
+
531
+ def normalize_market(value: Any) -> str:
532
+ code = str(value or "").strip().upper()
533
+ return "UK" if code == "GB" else code
534
+
535
+
536
+ def _brave_country(market: str) -> str:
537
+ return "gb" if market == "UK" else market.lower()
538
+
539
+
540
+ def load_intake(path: Path) -> dict[str, Any] | None:
541
+ """Read optional typed operation inputs; never infer labels from prose."""
542
+ if not path.is_file():
543
+ return None
544
+ try:
545
+ raw = json.loads(path.read_text(encoding="utf-8"))
546
+ except (OSError, json.JSONDecodeError) as exc:
547
+ raise CompetitorSetError(f"{path.name} is not valid JSON: {exc}") from exc
548
+ return validate_intake(raw)
549
+
550
+
551
+ def validate_intake(raw: Any) -> dict[str, Any]:
552
+ """Validate supplied fields, not the completeness of a research brief."""
553
+ if not isinstance(raw, dict) or type(raw.get("schema_version")) is not int or raw["schema_version"] != 1:
554
+ raise CompetitorSetError("intake must be an object with schema_version 1")
555
+
556
+ def text(value, field):
557
+ if value is not None and not isinstance(value, str):
558
+ raise CompetitorSetError(f"intake {field} must be text or null")
559
+ return " ".join((value or "").split()) or None
560
+
561
+ def strings(value, field):
562
+ if value is None:
563
+ return []
564
+ if not isinstance(value, list) or any(not isinstance(item, str) or not item.strip() for item in value):
565
+ raise CompetitorSetError(f"intake {field} must be a list of nonempty strings")
566
+ return list(dict.fromkeys(item.strip() for item in value))
567
+
568
+ def boolean(value, field):
569
+ if value is not None and type(value) is not bool:
570
+ raise CompetitorSetError(f"intake {field} must be boolean or null")
571
+ return value
572
+
573
+ def domains(value, field, cap):
574
+ values = strings(value, field)
575
+ if any(not _domain(item) for item in values):
576
+ raise CompetitorSetError(f"intake {field} contains an invalid domain")
577
+ normalized = parse_domains(values, len(values))
578
+ if len(normalized) > cap:
579
+ raise CompetitorSetError(f"intake {field} exceeds the collection limit of {cap}")
580
+ return normalized
581
+
582
+ def languages(value, field):
583
+ values = list(dict.fromkeys(item.lower() for item in strings(value, field)))
584
+ if len(values) > 40 or any(not re.fullmatch(r"[a-z]{2,3}(?:-[a-z0-9]{2,8})*", item) for item in values):
585
+ raise CompetitorSetError(f"intake {field} must contain explicit ISO/BCP-47 language codes")
586
+ return values
587
+
588
+ offer_raw = raw.get("offer")
589
+ if offer_raw is not None and not isinstance(offer_raw, dict):
590
+ raise CompetitorSetError("intake offer must be an object or null")
591
+ offer_raw = offer_raw or {}
592
+ offer: dict[str, Any] = {}
593
+ for key in ("name", "job", "buyer", "offer_class", "comparable_classes", "duration", "price_text", "price_floor_basis", "basis"):
594
+ value = text(offer_raw.get(key), f"offer.{key}")
595
+ if value is not None:
596
+ offer[key] = value
597
+ offer["url"] = _url(offer_raw.get("url"))
598
+ if offer_raw.get("url") and not offer["url"]:
599
+ raise CompetitorSetError("intake offer.url must be an HTTP(S) address")
600
+ for key in ("price_usd", "price_floor_usd"):
601
+ value = offer_raw.get(key)
602
+ if value is None:
603
+ offer[key] = None
604
+ continue
605
+ try:
606
+ offer[key] = _number(value)
607
+ except ValueError:
608
+ raise CompetitorSetError(f"intake offer.{key} must be a finite non-negative number or null") from None
609
+ phrases = strings(offer_raw.get("category_phrases"), "offer.category_phrases")
610
+ phrases = parse_terms(",".join(p.replace(",", " ") for p in phrases), len(phrases))
611
+ if len(phrases) > MAX_CATEGORY_PHRASES:
612
+ raise CompetitorSetError(f"intake offer.category_phrases exceeds the paid-query limit of {MAX_CATEGORY_PHRASES}")
613
+ offer["category_phrases"] = phrases
614
+ offer["confirmed_by_operator"] = boolean(offer_raw.get("confirmed_by_operator"), "offer.confirmed_by_operator")
615
+ offer["inferred"] = boolean(offer_raw.get("inferred"), "offer.inferred")
616
+ if offer["confirmed_by_operator"] and offer["inferred"]:
617
+ raise CompetitorSetError("intake offer cannot be both confirmed and inferred")
618
+ # Without a stated basis, neither a confirmation nor an inference is established.
619
+ if not offer.get("basis"):
620
+ offer["confirmed_by_operator"] = None
621
+ offer["inferred"] = None
622
+ markets: list[str] = []
623
+ for market in strings(raw.get("markets"), "markets"):
624
+ code = normalize_market(market)
625
+ if not re.fullmatch(r"[A-Z]{2}", code) or code == "EU":
626
+ raise CompetitorSetError("intake markets must contain explicit two-letter country codes")
627
+ if code not in markets:
628
+ markets.append(code)
629
+ market_languages = raw.get("market_languages") or {}
630
+ if not isinstance(market_languages, dict):
631
+ raise CompetitorSetError("intake market_languages must be an object")
632
+ by_market = {}
633
+ for country, values in market_languages.items():
634
+ code = normalize_market(country).replace("UK", "GB")
635
+ if not re.fullmatch(r"[A-Z]{2}", code) or code == "EU" or code in by_market:
636
+ raise CompetitorSetError("intake market_languages needs unique explicit country codes")
637
+ by_market[code] = languages(values, f"market_languages.{country}")
638
+ if not by_market[code]:
639
+ raise CompetitorSetError(f"intake market_languages.{country} cannot be empty")
640
+ named = domains(raw.get("operator_named"), "operator_named", MAX_NAMED_DOMAINS)
641
+ asked = boolean(raw.get("operator_named_asked"), "operator_named_asked")
642
+ if named and asked is False:
643
+ raise CompetitorSetError("intake operator_named contradicts operator_named_asked=false")
644
+ ltv = raw.get("ltv_usd")
645
+ if ltv is not None:
646
+ try:
647
+ ltv = _number(ltv)
648
+ except ValueError:
649
+ raise CompetitorSetError("intake ltv_usd must be a finite non-negative number or null") from None
650
+ ltv_provenance = raw.get("ltv_provenance")
651
+ if ltv_provenance not in (None, "given", "estimated", "unknown"):
652
+ raise CompetitorSetError("intake ltv_provenance must be given, estimated, unknown or null")
653
+ if ltv is None and ltv_provenance in ("given", "estimated"):
654
+ raise CompetitorSetError("intake ltv_provenance has no amount")
655
+ ltv_basis = text(raw.get("ltv_basis"), "ltv_basis")
656
+ if not ltv_basis:
657
+ ltv_provenance = None
658
+ return {
659
+ "offer": offer,
660
+ "operator_named": named,
661
+ "operator_named_asked": asked,
662
+ "page_mentioned": domains(raw.get("page_mentioned"), "page_mentioned", MAX_PAGE_DOMAINS),
663
+ "product_domain": _domain(raw.get("product_domain")),
664
+ "markets": markets,
665
+ "languages": languages(raw.get("languages"), "languages"),
666
+ "market_languages": by_market,
667
+ "ltv_usd": ltv,
668
+ "ltv_provenance": ltv_provenance,
669
+ "ltv_basis": ltv_basis,
670
+ }
671
+
672
+
673
+ def domains_from_keyword_evidence(workspace: Path, cap: int) -> list[str]:
674
+ evidence = _load_script("research-evidence.py", "pmr_evidence_for_competitor_set")
675
+ return evidence.keyword_inputs(workspace, consumer=SECTION_ID, row_type="competitor_domain", field="domain", cap=cap)
676
+
677
+
678
+ def category_queries(phrases: list[str]) -> list[str]:
679
+ """Per category phrase: the phrase itself (who pays for and ranks on the
680
+ offer's own search) and its "best" list (listicles)."""
681
+ queries: list[str] = []
682
+ for phrase in phrases:
683
+ for query in (phrase, f"best {phrase}"):
684
+ if query not in queries:
685
+ queries.append(query)
686
+ return queries
687
+
688
+
689
+ def brand_queries(named_domains: list[str]) -> list[str]:
690
+ """Alternatives queries for competitors named by the founder only, with their
691
+ own budget so they never displace the category queries."""
692
+ queries: list[str] = []
693
+ for domain in named_domains:
694
+ brand = brand_token(domain)
695
+ query = f"{brand} alternatives" if brand else ""
696
+ if query and query not in queries:
697
+ queries.append(query)
698
+ return queries[:MAX_BRAND_QUERIES]
699
+
700
+
701
+ @dataclass
702
+ class RunInputs:
703
+ named_domains: list[str]
704
+ page_domains: list[str]
705
+ keyword_domains: list[str]
706
+ product_domain: str
707
+ category_phrases: list[str]
708
+ category_queries: list[str]
709
+ brand_queries: list[str]
710
+ offer: dict[str, Any]
711
+ price_floor_usd: float | None
712
+ markets: list[str]
713
+ location_code: int
714
+ language_code: str
715
+ country: str
716
+ spyfu_market: str
717
+ eu_country: str
718
+ intake_path: str | None
719
+ intake_sha256: str | None
720
+ input_annotations: dict[str, Any] = field(default_factory=dict)
721
+
722
+
723
+ def _arg(args: argparse.Namespace, name: str, default: Any) -> Any:
724
+ value = getattr(args, name, None)
725
+ return default if value is None else value
726
+
727
+
728
+ def resolve_inputs(workspace: Path, args: argparse.Namespace) -> RunInputs:
729
+ intake_arg = str(_arg(args, "intake", "") or "")
730
+ intake_path = Path(intake_arg) if intake_arg else workspace / INTAKE_REL
731
+ if not intake_path.is_absolute():
732
+ intake_path = workspace / intake_path
733
+ if intake_arg and not intake_path.is_file():
734
+ raise CompetitorSetError(f"the supplied intake file does not exist: {intake_arg}")
735
+ intake = load_intake(intake_path) or {}
736
+ offer = dict(intake.get("offer") or {})
737
+
738
+ product_domain = _domain(_arg(args, "product_domain", "")) or intake.get("product_domain") or _domain(offer.get("url"))
739
+ named_flag = str(_arg(args, "domains", "") or "")
740
+ named = parse_domains(named_flag, MAX_NAMED_DOMAINS) if named_flag.strip() else list(intake.get("operator_named") or [])
741
+ page_flag = str(_arg(args, "page_domains", "") or "")
742
+ page = parse_domains(page_flag, MAX_PAGE_DOMAINS) if page_flag.strip() else list(intake.get("page_mentioned") or [])
743
+ named = [d for d in named if d != product_domain][:MAX_NAMED_DOMAINS]
744
+ page = [d for d in page if d != product_domain and d not in named][:MAX_PAGE_DOMAINS]
745
+ keyword_domains = [d for d in domains_from_keyword_evidence(workspace, MAX_KEYWORD_DOMAINS) if d != product_domain and d not in named and d not in page]
746
+ phrases_flag = str(_arg(args, "seed_keywords", "") or "")
747
+ phrases = parse_terms(phrases_flag, MAX_CATEGORY_PHRASES) if phrases_flag.strip() else list(offer.get("category_phrases") or [])
748
+ if not named and not phrases:
749
+ raise CompetitorSetError(
750
+ "no competitor domains or seed queries available: supply typed operation inputs "
751
+ f"in {INTAKE_REL.as_posix()}, or pass --seed-keywords and/or --domains "
752
+ "for the assigned comparison (see the workflow doc)"
753
+ )
754
+ floor_flag = _arg(args, "price_floor_usd", None)
755
+ try:
756
+ floor = _number(floor_flag) if floor_flag is not None else offer.get("price_floor_usd")
757
+ except ValueError:
758
+ raise CompetitorSetError("--price-floor-usd must be a non-negative number") from None
759
+ offer["category_phrases"] = phrases
760
+
761
+ markets = list(intake.get("markets") or [])
762
+ country_flag = normalize_market(_arg(args, "country", ""))
763
+ if country_flag:
764
+ markets = [country_flag] + [m for m in markets if m != country_flag]
765
+ if not markets:
766
+ markets = [DEFAULT_MARKET]
767
+ primary = markets[0]
768
+ location_flag = _arg(args, "location_code", None)
769
+ location_code = int(location_flag) if location_flag is not None else LOCATION_CODES.get(primary, DEFAULT_LOCATION_CODE)
770
+ eu_flag = normalize_market(_arg(args, "eu_country", ""))
771
+ eu_country = eu_flag or next((m for m in markets if m in EU_COUNTRIES), DEFAULT_EU_COUNTRY)
772
+ intake_sha = hashlib.sha256(intake_path.read_bytes()).hexdigest() if intake_path.is_file() else None
773
+ try:
774
+ intake_rel = intake_path.relative_to(workspace).as_posix()
775
+ except ValueError:
776
+ intake_rel = intake_path.as_posix()
777
+ return RunInputs(
778
+ named_domains=named,
779
+ page_domains=page,
780
+ keyword_domains=keyword_domains,
781
+ product_domain=product_domain,
782
+ category_phrases=phrases,
783
+ category_queries=category_queries(phrases),
784
+ brand_queries=brand_queries(named),
785
+ offer=offer,
786
+ price_floor_usd=floor,
787
+ markets=markets,
788
+ location_code=location_code,
789
+ language_code=str(_arg(args, "language_code", DEFAULT_LANGUAGE_CODE) or DEFAULT_LANGUAGE_CODE),
790
+ country=_brave_country(primary),
791
+ spyfu_market=primary,
792
+ eu_country=eu_country,
793
+ intake_path=intake_rel if intake_sha else None,
794
+ intake_sha256=intake_sha,
795
+ input_annotations={
796
+ key: intake.get(key)
797
+ for key in ("operator_named_asked", "languages", "market_languages", "ltv_usd", "ltv_provenance", "ltv_basis")
798
+ } | {"operator_named_asked": True if named_flag.strip() else intake.get("operator_named_asked")},
799
+ )
800
+
801
+
802
+ def validate_resume_inputs(workspace: Path, args: argparse.Namespace, inputs: RunInputs) -> None:
803
+ """Check explicit changes without rebuilding the saved collection scope."""
804
+ comparisons = {
805
+ "domains": (inputs.named_domains, lambda value: [d for d in parse_domains(value, MAX_NAMED_DOMAINS) if d != inputs.product_domain]),
806
+ "page_domains": (inputs.page_domains, lambda value: [d for d in parse_domains(value, MAX_PAGE_DOMAINS) if d != inputs.product_domain and d not in inputs.named_domains]),
807
+ "product_domain": (inputs.product_domain, _domain),
808
+ "seed_keywords": (inputs.category_phrases, lambda value: parse_terms(value, MAX_CATEGORY_PHRASES)),
809
+ "price_floor_usd": (inputs.price_floor_usd, _number),
810
+ "location_code": (inputs.location_code, int),
811
+ "language_code": (inputs.language_code, str),
812
+ "country": (inputs.markets[0], normalize_market),
813
+ "eu_country": (inputs.eu_country, normalize_market),
814
+ }
815
+ for name, (expected, normalize) in comparisons.items():
816
+ supplied = _arg(args, name, None)
817
+ if supplied in (None, ""):
818
+ continue
819
+ try:
820
+ matches = normalize(supplied) == expected
821
+ except (TypeError, ValueError):
822
+ matches = False
823
+ if not matches:
824
+ raise CompetitorSetError("competitor collection inputs changed; retain this study and use a new research ID")
825
+
826
+ # Source applicability is separate from replay identity. Retained sources
827
+ # must still match, but a clone need not carry the original source file.
828
+ if inputs.intake_path:
829
+ source = Path(inputs.intake_path)
830
+ source = source if source.is_absolute() else workspace / source
831
+ if source.is_file() and hashlib.sha256(source.read_bytes()).hexdigest() != inputs.intake_sha256:
832
+ raise CompetitorSetError("competitor collection inputs changed: declared intake source changed; review saved evidence applicability")
833
+ intake_arg = str(_arg(args, "intake", "") or "")
834
+ if intake_arg:
835
+ source = Path(intake_arg)
836
+ source = source if source.is_absolute() else workspace / source
837
+ if not source.is_file():
838
+ raise CompetitorSetError(f"the supplied intake file does not exist: {intake_arg}")
839
+ if hashlib.sha256(source.read_bytes()).hexdigest() != inputs.intake_sha256:
840
+ raise CompetitorSetError("competitor collection inputs changed; retain this study and use a new research ID")
841
+
842
+
843
+ def bind_inputs(workspace: Path, args: argparse.Namespace) -> RunInputs:
844
+ """Freeze the collection inputs before provider calls; output caps are not inputs."""
845
+ path = workspace / RUNTIME_REL / "inputs.json"
846
+ saved = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else None
847
+ if saved is not None:
848
+ if not isinstance(saved, dict) or saved.get("schema_version") != 1 or not isinstance(saved.get("inputs"), dict):
849
+ raise CompetitorSetError("invalid saved competitor inputs; preserve the state and inspect it")
850
+ digest = hashlib.sha256(json.dumps(saved["inputs"], sort_keys=True, ensure_ascii=False).encode()).hexdigest()
851
+ if saved.get("sha256") != digest:
852
+ raise CompetitorSetError("saved competitor inputs failed their integrity check")
853
+ inputs = RunInputs(**saved["inputs"])
854
+ validate_resume_inputs(workspace, args, inputs)
855
+ return inputs
856
+ inputs = resolve_inputs(workspace, args)
857
+ payload = asdict(inputs)
858
+ current = {"schema_version": 1, "inputs": payload,
859
+ "sha256": hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest()}
860
+ _write_atomic(path, json.dumps(current, indent=2, sort_keys=True, ensure_ascii=False) + "\n")
861
+ return inputs
862
+
863
+
864
+ # ── candidates ──────────────────────────────────────────────────────────────
865
+
866
+ @dataclass
867
+ class Candidate:
868
+ domain: str
869
+ provenance: list[str] = field(default_factory=list)
870
+ sources: list[str] = field(default_factory=list)
871
+ paid_queries: list[str] = field(default_factory=list)
872
+ organic_queries: list[str] = field(default_factory=list)
873
+ search_queries: list[str] = field(default_factory=list)
874
+ brand_pairs: list[str] = field(default_factory=list)
875
+ keyword_ads_phrases: list[str] = field(default_factory=list)
876
+ keyword_ads_rows: int = 0
877
+ keyword_ads_last_seen: str | None = None
878
+ listicle_pages: list[str] = field(default_factory=list)
879
+ overlap_seeds: list[str] = field(default_factory=list)
880
+ overlap_intersections: int = 0
881
+ ppc_seeds: list[str] = field(default_factory=list)
882
+ spyfu_common_keywords: int = 0
883
+ listed_for: list[str] = field(default_factory=list)
884
+ found_in: str | None = None
885
+ round: int = 1
886
+ aliases: list[str] = field(default_factory=list)
887
+ pages: list[str] = field(default_factory=list)
888
+
889
+ def add_provenance(self, kind: str) -> None:
890
+ if kind not in self.provenance:
891
+ self.provenance.append(kind)
892
+
893
+ def add_source(self, source: str) -> None:
894
+ if source not in self.sources:
895
+ self.sources.append(source)
896
+
897
+ def listed(self, query: str) -> None:
898
+ if query not in self.listed_for:
899
+ self.listed_for.append(query)
900
+
901
+ def seen_on(self, url: str) -> None:
902
+ """The first pages that ranked or advertised: profiling starts there
903
+ (a reseller's or host's page shows what the domain really sells)."""
904
+ page = _url(url.split("#", 1)[0])
905
+ if page and page not in self.pages and len(self.pages) < PAGES_PER_CANDIDATE:
906
+ self.pages.append(page)
907
+
908
+ @staticmethod
909
+ def _add(values: list[str], value: str) -> None:
910
+ if value not in values:
911
+ values.append(value)
912
+
913
+ @property
914
+ def serp_paid_hits(self) -> int:
915
+ return len(self.paid_queries)
916
+
917
+ @property
918
+ def serp_organic_hits(self) -> int:
919
+ return len(self.organic_queries)
920
+
921
+ @property
922
+ def search_hits(self) -> int:
923
+ return len(self.search_queries)
924
+
925
+ @property
926
+ def brand_hits(self) -> int:
927
+ return len(self.brand_pairs)
928
+
929
+ def offer_signals(self) -> int:
930
+ """Distinct offer signals: one per query and engine, one per phrase a
931
+ recent advertiser bought."""
932
+ return self.serp_paid_hits + self.serp_organic_hits + self.search_hits + len(self.keyword_ads_phrases)
933
+
934
+ def relationship_hits(self) -> int:
935
+ return len(set(self.overlap_seeds)) + len(set(self.ppc_seeds))
936
+
937
+ def round1_score(self) -> float:
938
+ score = SCORE_CATEGORY_PAID * min(self.serp_paid_hits, CAP_CATEGORY_PAID)
939
+ score += SCORE_KEYWORD_ADS * min(len(self.keyword_ads_phrases), CAP_KEYWORD_ADS)
940
+ score += SCORE_CATEGORY_ORGANIC * min(self.serp_organic_hits, CAP_CATEGORY_ORGANIC)
941
+ score += SCORE_SECOND_ENGINE * min(self.search_hits, CAP_SECOND_ENGINE)
942
+ score += SCORE_BRAND_QUERY * min(self.brand_hits, CAP_BRAND_QUERY)
943
+ score += SCORE_LISTICLE * min(len(self.listicle_pages), CAP_LISTICLE)
944
+ if "keyword_economics" in self.provenance:
945
+ score += SCORE_KEYWORD_ECONOMICS
946
+ return round(score, 2)
947
+
948
+ def score(self) -> float:
949
+ return round(self.round1_score() + SCORE_RELATIONSHIP * min(self.relationship_hits(), CAP_RELATIONSHIP), 2)
950
+
951
+ def signal_count(self) -> int:
952
+ """Independent places the domain showed up: offer signals, listicle
953
+ pages and alternatives-query hits."""
954
+ return self.offer_signals() + len(self.listicle_pages) + self.brand_hits
955
+
956
+ def offer_signal(self) -> bool:
957
+ """Worth profiling: ranks or pays on Google for the offer's own
958
+ search, or shows up in two independent places (two queries, two
959
+ engines, an advertised phrase plus a listicle, two listicles); a
960
+ single second-engine result, a single advertised phrase or a single
961
+ listicle mention alone is a weak hit."""
962
+ return bool(self.serp_paid_hits or self.serp_organic_hits or self.signal_count() >= 2)
963
+
964
+ def absorb(self, alias: "Candidate") -> None:
965
+ """Take over the discovery facts of another domain of the same
966
+ company (idempotent: every list is a set)."""
967
+ for name in ("provenance", "sources", "paid_queries", "organic_queries", "search_queries", "brand_pairs", "keyword_ads_phrases",
968
+ "listicle_pages", "overlap_seeds", "ppc_seeds", "listed_for"):
969
+ values = getattr(self, name)
970
+ for value in getattr(alias, name):
971
+ Candidate._add(values, value)
972
+ for page in alias.pages:
973
+ self.seen_on(page)
974
+ if alias.keyword_ads_last_seen and (self.keyword_ads_last_seen is None or alias.keyword_ads_last_seen > self.keyword_ads_last_seen):
975
+ self.keyword_ads_last_seen = alias.keyword_ads_last_seen
976
+ self.overlap_intersections = max(self.overlap_intersections, alias.overlap_intersections)
977
+ self.spyfu_common_keywords = max(self.spyfu_common_keywords, alias.spyfu_common_keywords)
978
+ Candidate._add(self.aliases, alias.domain)
979
+
980
+ def round1_signal(self) -> bool:
981
+ return bool(self.serp_paid_hits or self.keyword_ads_phrases or self.serp_organic_hits or self.search_hits
982
+ or self.brand_hits or self.listicle_pages or "keyword_economics" in self.provenance)
983
+
984
+ def primary_provenance(self) -> str:
985
+ return next((p for p in PROVENANCE_ORDER if p in self.provenance), "discovered")
986
+
987
+ def ordered_provenance(self) -> list[str]:
988
+ return [p for p in PROVENANCE_ORDER if p in self.provenance]
989
+
990
+ def row(self) -> dict[str, Any]:
991
+ return {
992
+ "type": "candidate",
993
+ "domain": self.domain,
994
+ "provenance": self.ordered_provenance(),
995
+ "round": self.round,
996
+ "sources": list(self.sources),
997
+ "source_count": len(self.sources),
998
+ "serp_paid_hits": self.serp_paid_hits,
999
+ "serp_organic_hits": self.serp_organic_hits,
1000
+ "search_hits": self.search_hits,
1001
+ "brand_hits": self.brand_hits,
1002
+ "keyword_ads_hits": len(self.keyword_ads_phrases),
1003
+ "listicle_pages": len(self.listicle_pages),
1004
+ "relationship_hits": self.relationship_hits(),
1005
+ "overlap_intersections": self.overlap_intersections,
1006
+ "spyfu_ppc_hits": len(set(self.ppc_seeds)),
1007
+ "spyfu_common_keywords": self.spyfu_common_keywords,
1008
+ "listed_for": list(self.listed_for[:8]),
1009
+ "pages": list(self.pages),
1010
+ "found_in": self.found_in,
1011
+ "aliases": list(self.aliases),
1012
+ "score": self.score(),
1013
+ "tier": "B",
1014
+ }
1015
+
1016
+
1017
+ def _candidate(pool: dict[str, Candidate], domain: str, round_no: int = 1) -> Candidate:
1018
+ if domain not in pool:
1019
+ pool[domain] = Candidate(domain=domain, round=round_no)
1020
+ return pool[domain]
1021
+
1022
+
1023
+ def _serp_items(response: dict[str, Any]) -> list[dict[str, Any]]:
1024
+ result = response.get("result")
1025
+ entry = (result or [{}])[0] if isinstance(result, list) else {}
1026
+ items = entry.get("items") if isinstance(entry, dict) else None
1027
+ return [item for item in (items or []) if isinstance(item, dict)]
1028
+
1029
+
1030
+ def _listicle(url: str, title: str) -> bool:
1031
+ return bool(_LISTICLE_RE.search(title or "")) or bool(_LISTICLE_RE.search(url or ""))
1032
+
1033
+
1034
+ def _summary(round_no: int, query: str, kind: str, source: str, retrieved: int, found: int, listicles: list[str], unavailable: str | None = None) -> dict[str, Any]:
1035
+ row = {"type": "search_summary", "round": round_no, "query": query, "query_kind": kind, "source": source,
1036
+ "retrieved": retrieved, "candidates": found, "listicles": listicles, "tier": "A"}
1037
+ if unavailable:
1038
+ row["unavailable"] = unavailable[:200]
1039
+ return row
1040
+
1041
+
1042
+ @dataclass
1043
+ class Discovery:
1044
+ pool: dict[str, Candidate]
1045
+ directories: dict[str, list[str]]
1046
+ summaries: list[dict[str, Any]]
1047
+
1048
+
1049
+ def run_round1(client: OsClient, inputs: RunInputs) -> Discovery:
1050
+ """Discovery from the offer: category phrases first, then the founder's
1051
+ brands; core Google results fail the attempt, SpyFu and Brave are best
1052
+ effort."""
1053
+ pool: dict[str, Candidate] = {}
1054
+ directories: dict[str, list[str]] = {}
1055
+ summaries: list[dict[str, Any]] = []
1056
+ for domain in inputs.named_domains:
1057
+ _candidate(pool, domain).add_provenance("operator_named")
1058
+ for domain in inputs.page_domains:
1059
+ _candidate(pool, domain).add_provenance("product_page")
1060
+ for domain in inputs.keyword_domains:
1061
+ candidate = _candidate(pool, domain)
1062
+ candidate.add_provenance("keyword_economics")
1063
+ candidate.add_source("keyword_economics")
1064
+
1065
+ locale = {"location_code": inputs.location_code, "language_code": inputs.language_code}
1066
+ queries = [(q, "category") for q in inputs.category_queries] + [(q, "brand") for q in inputs.brand_queries]
1067
+
1068
+ def admit(domain: str, query: str) -> str:
1069
+ """`candidate`, `skip` or `directory` for a result domain."""
1070
+ if not domain or domain == inputs.product_domain:
1071
+ return "skip"
1072
+ kind = classify_domain(domain)
1073
+ if kind == "noise":
1074
+ return "skip"
1075
+ if kind == "directory":
1076
+ directories.setdefault(domain, []).append(query)
1077
+ return "directory"
1078
+ return "candidate"
1079
+
1080
+ for query, kind in queries:
1081
+ response, _ = _call_with_context(client, query, "DataForSEO serp", lambda q=query: client.dataforseo(
1082
+ "serp", {"keyword": q, "depth": SERP_DEPTH, **locale}
1083
+ ))
1084
+ if response.get("unavailable"):
1085
+ summaries.append(_summary(1, query, kind, "dataforseo_serp", 0, 0, [], str(response["unavailable"])))
1086
+ continue
1087
+ retrieved = found = 0
1088
+ listicles: list[str] = []
1089
+ for item in _serp_items(response):
1090
+ item_type = item.get("type")
1091
+ if item_type not in ("paid", "organic"):
1092
+ continue
1093
+ retrieved += 1
1094
+ url = str(item.get("url") or "")
1095
+ domain = _domain(item.get("domain") or url)
1096
+ verdict = admit(domain, query)
1097
+ if verdict == "skip":
1098
+ continue
1099
+ if verdict == "directory" or (item_type == "organic" and _listicle(url, str(item.get("title") or ""))):
1100
+ if url and len(listicles) < LISTICLES_PER_QUERY and url not in listicles:
1101
+ listicles.append(url)
1102
+ continue
1103
+ candidate = _candidate(pool, domain)
1104
+ candidate.add_provenance("discovered")
1105
+ if kind == "category":
1106
+ if item_type == "paid":
1107
+ candidate.add_source("serp_paid")
1108
+ Candidate._add(candidate.paid_queries, query)
1109
+ else:
1110
+ candidate.add_source("serp_organic")
1111
+ Candidate._add(candidate.organic_queries, query)
1112
+ else:
1113
+ candidate.add_source("brand_query")
1114
+ Candidate._add(candidate.brand_pairs, f"serp:{query}")
1115
+ candidate.listed(query)
1116
+ candidate.seen_on(url)
1117
+ found += 1
1118
+ summaries.append(_summary(1, query, kind, "dataforseo_serp", retrieved, found, listicles))
1119
+
1120
+ # SpyFu: the domains that bought Google ads on the offer's own phrases.
1121
+ for phrase in inputs.category_phrases if client.max_spyfu_calls > 0 else []:
1122
+ query = f"ads:{phrase}"
1123
+ try:
1124
+ response, retrieved_on = client.spyfu("keyword-ads", {"keyword": phrase, "country_code": inputs.spyfu_market, "max_rows": KEYWORD_ADS_ROWS})
1125
+ except BudgetExhausted:
1126
+ raise
1127
+ except CompetitorSetError as exc:
1128
+ summaries.append(_summary(1, query, "category", "spyfu_keyword_ads", 0, 0, [], str(exc)))
1129
+ continue
1130
+ if response.get("unavailable"):
1131
+ summaries.append(_summary(1, query, "category", "spyfu_keyword_ads", 0, 0, [], str(response["unavailable"])))
1132
+ continue
1133
+ # Recency is measured from the day the answer was retrieved, so a
1134
+ # cached replay classifies the same advertisers the same way.
1135
+ try:
1136
+ cutoff = (date.fromisoformat(retrieved_on) - timedelta(days=KEYWORD_ADS_RECENT_DAYS)).isoformat()
1137
+ except ValueError:
1138
+ cutoff = (date.today() - timedelta(days=KEYWORD_ADS_RECENT_DAYS)).isoformat()
1139
+ retrieved = 0
1140
+ advertisers: list[str] = []
1141
+ for item in response.get("rows") or []:
1142
+ if not isinstance(item, dict):
1143
+ continue
1144
+ retrieved += 1
1145
+ seen = str(item.get("seen_on") or "")[:10]
1146
+ if not _DATE_RE.match(seen) or seen < cutoff:
1147
+ continue # an advertiser of years ago does not buy this traffic now
1148
+ urls = item.get("urls") if isinstance(item.get("urls"), list) else []
1149
+ domain = _domain(item.get("advertiser") or (urls[0] if urls else ""))
1150
+ if admit(domain, query) != "candidate":
1151
+ continue
1152
+ candidate = _candidate(pool, domain)
1153
+ candidate.add_provenance("discovered")
1154
+ candidate.add_source("spyfu_keyword_ads")
1155
+ Candidate._add(candidate.keyword_ads_phrases, phrase)
1156
+ candidate.keyword_ads_rows += 1
1157
+ if candidate.keyword_ads_last_seen is None or seen > candidate.keyword_ads_last_seen:
1158
+ candidate.keyword_ads_last_seen = seen
1159
+ candidate.listed(query)
1160
+ for ad_url in urls[:1]:
1161
+ candidate.seen_on(str(ad_url))
1162
+ Candidate._add(advertisers, domain)
1163
+ summaries.append(_summary(1, query, "category", "spyfu_keyword_ads", retrieved, len(advertisers), []))
1164
+
1165
+ for query, kind in queries:
1166
+ try:
1167
+ response, _ = client.brave(query, BRAVE_COUNT, inputs.country)
1168
+ except BudgetExhausted:
1169
+ raise # the attempt stops; the next invocation finishes the queries from the cache
1170
+ except CompetitorSetError as exc:
1171
+ # Best-effort second engine: an undeployed provider or a provider
1172
+ # rate limit is recorded, never a failed section.
1173
+ summaries.append(_summary(1, query, kind, "brave", 0, 0, [], str(exc)))
1174
+ continue
1175
+ retrieved = found = 0
1176
+ listicles = []
1177
+ for item in response.get("results") or []:
1178
+ if not isinstance(item, dict):
1179
+ continue
1180
+ retrieved += 1
1181
+ url = str(item.get("url") or "")
1182
+ domain = _domain(url)
1183
+ verdict = admit(domain, query)
1184
+ if verdict == "skip":
1185
+ continue
1186
+ if verdict == "directory" or _listicle(url, str(item.get("title") or "")):
1187
+ if url and len(listicles) < LISTICLES_PER_QUERY and url not in listicles:
1188
+ listicles.append(url)
1189
+ continue
1190
+ candidate = _candidate(pool, domain)
1191
+ candidate.add_provenance("discovered")
1192
+ if kind == "category":
1193
+ candidate.add_source("brave")
1194
+ Candidate._add(candidate.search_queries, query)
1195
+ else:
1196
+ candidate.add_source("brand_query")
1197
+ Candidate._add(candidate.brand_pairs, f"brave:{query}")
1198
+ candidate.listed(query)
1199
+ candidate.seen_on(url)
1200
+ found += 1
1201
+ summaries.append(_summary(1, query, kind, "brave", retrieved, found, listicles))
1202
+ return Discovery(pool=pool, directories=directories, summaries=summaries)
1203
+
1204
+
1205
+ def listicles_to_read(summaries: list[dict[str, Any]], product_domain: str) -> list[str]:
1206
+ """The bounded reading list: "best" lists of the category phrases first,
1207
+ then the other category results, then the brand queries; directories and
1208
+ the product's own pages are never on it."""
1209
+ def priority(row: dict[str, Any]) -> tuple[int, int]:
1210
+ if row.get("query_kind") == "brand":
1211
+ group = 2
1212
+ else:
1213
+ group = 0 if str(row.get("query") or "").startswith("best ") else 1
1214
+ return group, 0 if row.get("source") == "dataforseo_serp" else 1
1215
+
1216
+ urls: list[str] = []
1217
+ per_site: dict[str, int] = {}
1218
+ for row in sorted((s for s in summaries if s.get("round") == 1), key=priority):
1219
+ for url in row.get("listicles") or []:
1220
+ domain = _domain(url)
1221
+ if not domain or domain == product_domain or classify_domain(domain) == "directory" or url in urls:
1222
+ continue
1223
+ if _search_page(url) or per_site.get(domain, 0) >= LISTICLES_PER_SITE:
1224
+ continue # a site's own search results are no list; two pages per site keep the views varied
1225
+ per_site[domain] = per_site.get(domain, 0) + 1
1226
+ urls.append(url)
1227
+ return urls[:MAX_LISTICLES_TO_READ]
1228
+
1229
+
1230
+ def _search_page(url: str) -> bool:
1231
+ parsed = urllib.parse.urlparse(url)
1232
+ keys = {key.lower() for key in urllib.parse.parse_qs(parsed.query)}
1233
+ return bool(keys & _SEARCH_PAGE_KEYS) or "/search" in parsed.path.lower()
1234
+
1235
+
1236
+ def run_round2(client: OsClient, inputs: RunInputs, discovery: Discovery, seeds: list[str]) -> None:
1237
+ """Keyword overlap and PPC competitors of the confirmed direct domains.
1238
+ The overlap report is core evidence; SpyFu is best effort."""
1239
+ pool, summaries = discovery.pool, discovery.summaries
1240
+ locale = {"location_code": inputs.location_code, "language_code": inputs.language_code}
1241
+
1242
+ def admit(domain: str, seed: str, query: str) -> Candidate | None:
1243
+ if not domain or domain in (seed, inputs.product_domain):
1244
+ return None
1245
+ kind = classify_domain(domain)
1246
+ if kind == "noise":
1247
+ return None
1248
+ if kind == "directory":
1249
+ discovery.directories.setdefault(domain, []).append(query)
1250
+ return None
1251
+ candidate = _candidate(pool, domain, 2)
1252
+ candidate.add_provenance("discovered")
1253
+ return candidate
1254
+
1255
+ for seed in seeds:
1256
+ query = f"overlap:{seed}"
1257
+ response, _ = _call_with_context(client, seed, "DataForSEO competitors-domain", lambda s=seed: client.dataforseo(
1258
+ "competitors-domain", {"target": s, "limit": COMPETITORS_DOMAIN_LIMIT, **locale}
1259
+ ))
1260
+ if response.get("unavailable"):
1261
+ summaries.append(_summary(2, query, "relationship", "dataforseo_competitors_domain", 0, 0, [], str(response["unavailable"])))
1262
+ continue
1263
+ retrieved = found = 0
1264
+ for item in _serp_items(response):
1265
+ retrieved += 1
1266
+ candidate = admit(_domain(item.get("domain")), seed, query)
1267
+ if candidate is None:
1268
+ continue
1269
+ candidate.add_source("dataforseo_overlap")
1270
+ if seed not in candidate.overlap_seeds:
1271
+ candidate.overlap_seeds.append(seed)
1272
+ intersections = item.get("intersections")
1273
+ if isinstance(intersections, int) and not isinstance(intersections, bool):
1274
+ candidate.overlap_intersections = max(candidate.overlap_intersections, intersections)
1275
+ candidate.listed(query)
1276
+ found += 1
1277
+ summaries.append(_summary(2, query, "relationship", "dataforseo_competitors_domain", retrieved, found, []))
1278
+
1279
+ for seed in seeds if client.max_spyfu_calls > 0 else []:
1280
+ query = f"ppc:{seed}"
1281
+ try:
1282
+ response, _ = client.spyfu("ppc-competitors", {"domain": seed, "country_code": inputs.spyfu_market, "max_rows": SPYFU_COMPETITORS})
1283
+ except BudgetExhausted:
1284
+ raise
1285
+ except CompetitorSetError as exc:
1286
+ summaries.append(_summary(2, query, "relationship", "spyfu_ppc_competitors", 0, 0, [], str(exc)))
1287
+ continue
1288
+ if response.get("unavailable"):
1289
+ summaries.append(_summary(2, query, "relationship", "spyfu_ppc_competitors", 0, 0, [], str(response["unavailable"])))
1290
+ continue
1291
+ retrieved = found = 0
1292
+ for item in response.get("rows") or []:
1293
+ if not isinstance(item, dict):
1294
+ continue
1295
+ retrieved += 1
1296
+ candidate = admit(_domain(item.get("domain")), seed, query)
1297
+ if candidate is None:
1298
+ continue
1299
+ candidate.add_source("spyfu_ppc")
1300
+ if seed not in candidate.ppc_seeds:
1301
+ candidate.ppc_seeds.append(seed)
1302
+ common = item.get("common_paid_keywords")
1303
+ if isinstance(common, int) and not isinstance(common, bool):
1304
+ candidate.spyfu_common_keywords = max(candidate.spyfu_common_keywords, common)
1305
+ candidate.listed(query)
1306
+ found += 1
1307
+ summaries.append(_summary(2, query, "relationship", "spyfu_ppc_competitors", retrieved, found, []))
1308
+
1309
+
1310
+ # ── the agent's enrichment files ────────────────────────────────────────────
1311
+
1312
+ def _load_json(path: Path) -> Any:
1313
+ if not path.is_file():
1314
+ return None
1315
+ try:
1316
+ return json.loads(path.read_text(encoding="utf-8"))
1317
+ except (OSError, json.JSONDecodeError) as exc:
1318
+ raise CompetitorSetError(f"{path.name} is not valid JSON: {exc}") from exc
1319
+
1320
+
1321
+ def load_listicles(path: Path) -> dict[str, dict[str, Any]]:
1322
+ raw = _load_json(path)
1323
+ if raw is None:
1324
+ return {}
1325
+ if not isinstance(raw, dict) or raw.get("schema_version") != 1 or not isinstance(raw.get("pages"), list):
1326
+ raise CompetitorSetError(f"{path.name} must be an object with schema_version 1 and a pages list")
1327
+ pages: dict[str, dict[str, Any]] = {}
1328
+ for index, page in enumerate(raw["pages"]):
1329
+ if not isinstance(page, dict):
1330
+ raise CompetitorSetError(f"{path.name} page #{index + 1} must be an object")
1331
+ url = _url(page.get("url"))
1332
+ if not url:
1333
+ raise CompetitorSetError(f"{path.name} page #{index + 1} needs the listicle address as url")
1334
+ if url in pages:
1335
+ raise CompetitorSetError(f"{path.name} records {url} twice")
1336
+ status = page.get("status")
1337
+ if status not in ("read", "unreadable"):
1338
+ raise CompetitorSetError(f"{path.name} page {url} needs status read or unreadable")
1339
+ reason = _text(page.get("reason"), 200)
1340
+ if status == "unreadable" and not reason:
1341
+ raise CompetitorSetError(f"{path.name} page {url} is unreadable without a reason")
1342
+ entries_raw = page.get("entries") or []
1343
+ if not isinstance(entries_raw, list) or len(entries_raw) > MAX_LISTICLE_ENTRIES:
1344
+ raise CompetitorSetError(f"{path.name} page {url} needs at most {MAX_LISTICLE_ENTRIES} entries in a list")
1345
+ entries = []
1346
+ for entry in entries_raw:
1347
+ if not isinstance(entry, dict) or not _text(entry.get("name")):
1348
+ raise CompetitorSetError(f"{path.name} page {url} has an entry without a name")
1349
+ domain_raw = entry.get("domain")
1350
+ domain = _domain(domain_raw) if domain_raw else ""
1351
+ if domain_raw and not domain:
1352
+ raise CompetitorSetError(f"{path.name} page {url} has an entry with an invalid domain {domain_raw!r}")
1353
+ entries.append({
1354
+ "name": _text(entry.get("name"), 80), "domain": domain or None,
1355
+ **{key: _text(entry.get(key), 160) or None for key in ("offer", "format", "price_text", "duration", "audience")},
1356
+ })
1357
+ pages[url] = {"url": url, "status": status, "reason": reason or None, "entries": entries}
1358
+ return pages
1359
+
1360
+
1361
+ def load_profiles(path: Path) -> dict[str, dict[str, Any]]:
1362
+ raw = _load_json(path)
1363
+ if raw is None:
1364
+ return {}
1365
+ if not isinstance(raw, dict) or raw.get("schema_version") != 1 or not isinstance(raw.get("profiles"), list):
1366
+ raise CompetitorSetError(f"{path.name} must be an object with schema_version 1 and a profiles list")
1367
+ profiles: dict[str, dict[str, Any]] = {}
1368
+ for index, item in enumerate(raw["profiles"]):
1369
+ if not isinstance(item, dict):
1370
+ raise CompetitorSetError(f"{path.name} profile #{index + 1} must be an object")
1371
+ domain = _domain(item.get("domain"))
1372
+ if not domain:
1373
+ raise CompetitorSetError(f"{path.name} profile #{index + 1} has no valid domain")
1374
+ if domain in profiles:
1375
+ raise CompetitorSetError(f"{path.name} profiles `{domain}` twice")
1376
+ for flag in ("operating", "same_buyer", "same_job"):
1377
+ if not isinstance(item.get(flag), bool):
1378
+ raise CompetitorSetError(f"{path.name} profile for `{domain}` needs a boolean {flag}")
1379
+ if item.get("offer_match") not in OFFER_MATCHES:
1380
+ raise CompetitorSetError(f"{path.name} profile for `{domain}` needs offer_match in {', '.join(OFFER_MATCHES)}")
1381
+ price = item.get("price_usd")
1382
+ try:
1383
+ price_usd = None if price is None else _number(price)
1384
+ except ValueError:
1385
+ raise CompetitorSetError(f"{path.name} profile for `{domain}` needs price_usd as a non-negative number or null") from None
1386
+ for flag in ("marketplace", "focused"):
1387
+ if item.get(flag) is not None and not isinstance(item.get(flag), bool):
1388
+ raise CompetitorSetError(f"{path.name} profile for `{domain}` has a non-boolean {flag}")
1389
+ urls = item.get("evidence_urls")
1390
+ evidence = [u for u in (_url(x) for x in urls) if u] if isinstance(urls, list) else []
1391
+ if not evidence:
1392
+ raise CompetitorSetError(f"{path.name} profile for `{domain}` needs at least one evidence URL (the pages read)")
1393
+ found_raw = item.get("found_in")
1394
+ found_in = _url(found_raw) if found_raw else ""
1395
+ if found_raw and not found_in:
1396
+ raise CompetitorSetError(f"{path.name} profile for `{domain}` has found_in that is not an address")
1397
+ profiles[domain] = {
1398
+ "domain": domain,
1399
+ "name": _text(item.get("name"), 80) or None,
1400
+ "operating": item["operating"],
1401
+ "offer": _text(item.get("offer"), 200) or None,
1402
+ "offer_url": _url(item.get("offer_url")) or None,
1403
+ "offer_class": _text(item.get("offer_class"), 120) or None,
1404
+ "offer_match": item["offer_match"],
1405
+ "price_text": _text(item.get("price_text"), 160) or None,
1406
+ "price_usd": price_usd,
1407
+ "duration": _text(item.get("duration"), 120) or None,
1408
+ "buyer": _text(item.get("buyer"), 200) or None,
1409
+ "same_buyer": item["same_buyer"],
1410
+ "job": _text(item.get("job"), 200) or None,
1411
+ "same_job": item["same_job"],
1412
+ "marketplace": bool(item.get("marketplace")),
1413
+ # False when the offer is a side line of a big multi-product domain (a
1414
+ # university, a large platform): its neighbours are not the offer's.
1415
+ "focused": item.get("focused") is not False,
1416
+ "main_business": _text(item.get("main_business"), 200) or None,
1417
+ "markets": _text(item.get("markets"), 120) or None,
1418
+ "evidence_urls": evidence[:6],
1419
+ "found_in": found_in or None,
1420
+ }
1421
+ return profiles
1422
+
1423
+
1424
+ def load_decisions(path: Path) -> dict[str, dict[str, Any]]:
1425
+ """Validate the agent's tiering file. Every decision names a domain the
1426
+ pool knows (discovered, named, listicle-backed or profiled with the page
1427
+ where the agent found it)."""
1428
+ raw = _load_json(path)
1429
+ if raw is None:
1430
+ return {}
1431
+ if isinstance(raw, dict) and raw.get("schema_version") == 1:
1432
+ raise CompetitorSetError(
1433
+ f"{path.name} has schema_version 1, which predates the offer-led rules: profile the candidates and "
1434
+ f"decide them again with schema_version {DECISIONS_SCHEMA_VERSION} (see the workflow doc)"
1435
+ )
1436
+ if not isinstance(raw, dict) or raw.get("schema_version") != DECISIONS_SCHEMA_VERSION or not isinstance(raw.get("decisions"), list):
1437
+ raise CompetitorSetError(f"{path.name} must be an object with schema_version {DECISIONS_SCHEMA_VERSION} and a decisions list")
1438
+ decisions: dict[str, dict[str, Any]] = {}
1439
+ for index, item in enumerate(raw["decisions"]):
1440
+ if not isinstance(item, dict):
1441
+ raise CompetitorSetError(f"{path.name} decision #{index + 1} must be an object")
1442
+ domain = _domain(item.get("domain"))
1443
+ tier = item.get("competitor_tier")
1444
+ reason = _text(item.get("reason"), 240)
1445
+ if not domain:
1446
+ raise CompetitorSetError(f"{path.name} decision #{index + 1} has no valid domain")
1447
+ if domain in decisions:
1448
+ raise CompetitorSetError(f"{path.name} decides `{domain}` twice")
1449
+ if tier not in DECISION_TIERS:
1450
+ raise CompetitorSetError(f"{path.name} decision for `{domain}` needs competitor_tier in {', '.join(DECISION_TIERS)}")
1451
+ if not reason:
1452
+ raise CompetitorSetError(f"{path.name} decision for `{domain}` needs a reason")
1453
+ name = item.get("name")
1454
+ same_raw = item.get("same_as")
1455
+ same_as = _domain(same_raw) if same_raw else ""
1456
+ if same_raw and (not same_as or same_as == domain):
1457
+ raise CompetitorSetError(f"{path.name} decision for `{domain}` has same_as that is not another domain")
1458
+ if same_as and tier != "excluded":
1459
+ raise CompetitorSetError(f"{path.name} decision for `{domain}` folds it into `{same_as}`: its tier must be excluded")
1460
+ decisions[domain] = {
1461
+ "competitor_tier": tier,
1462
+ "reason": reason,
1463
+ "name": _text(name, 80) if isinstance(name, str) and name.strip() else None,
1464
+ "override_reason": _text(item.get("override_reason"), 240) or None,
1465
+ "same_as": same_as or None,
1466
+ }
1467
+ for domain, decision in decisions.items():
1468
+ target = decisions.get(decision["same_as"] or "")
1469
+ if target is not None and target["same_as"]:
1470
+ raise CompetitorSetError(f"{path.name} folds `{domain}` into `{decision['same_as']}`, which is folded itself; name the final domain")
1471
+ return decisions
1472
+
1473
+
1474
+ def load_review(path: Path) -> dict[str, Any]:
1475
+ raw = _load_json(path)
1476
+ empty = {"rounds": [], "checked": set(), "objections": [], "missing": [], "checked_by_round": {1: set(), 2: set()}}
1477
+ if raw is None:
1478
+ return empty
1479
+ if not isinstance(raw, dict) or raw.get("schema_version") != 1 or not isinstance(raw.get("rounds"), list):
1480
+ raise CompetitorSetError(f"{path.name} must be an object with schema_version 1 and a rounds list")
1481
+ review = empty
1482
+ for index, item in enumerate(raw["rounds"]):
1483
+ if not isinstance(item, dict):
1484
+ raise CompetitorSetError(f"{path.name} round #{index + 1} must be an object")
1485
+ reviewer = _text(item.get("reviewer"), 120)
1486
+ if not reviewer:
1487
+ raise CompetitorSetError(f"{path.name} round #{index + 1} needs the reviewer (a separate agent or pass)")
1488
+ checked = item.get("checked")
1489
+ if not isinstance(checked, list) or not checked:
1490
+ raise CompetitorSetError(f"{path.name} round #{index + 1} needs the checked domains")
1491
+ domains = parse_domains(checked, 1000)
1492
+ review["checked"].update(domains)
1493
+ # The discovery round whose pool this review round looked through (1 when omitted).
1494
+ discovery_round = item.get("discovery_round", 1)
1495
+ if type(discovery_round) is not int or discovery_round not in (1, 2):
1496
+ raise CompetitorSetError(f"{path.name} round #{index + 1} has discovery_round {discovery_round!r}; use 1 or 2")
1497
+ review["checked_by_round"][discovery_round].update(domains)
1498
+ objections = item.get("objections") or []
1499
+ missing = item.get("missing") or []
1500
+ if not isinstance(objections, list) or not isinstance(missing, list):
1501
+ raise CompetitorSetError(f"{path.name} round #{index + 1} needs objections and missing as lists")
1502
+ for objection in objections:
1503
+ domain = _domain(objection.get("domain")) if isinstance(objection, dict) else ""
1504
+ if not domain or not _text(objection.get("objection")):
1505
+ raise CompetitorSetError(f"{path.name} round #{index + 1} has an objection without a domain or text")
1506
+ resolution = objection.get("resolution")
1507
+ if resolution not in (None, "", *RESOLUTIONS):
1508
+ raise CompetitorSetError(f"{path.name} objection on `{domain}` needs resolution accepted or rejected")
1509
+ note = _text(objection.get("note"), 240)
1510
+ review["objections"].append({"round": index + 1, "domain": domain, "objection": _text(objection.get("objection"), 240),
1511
+ "resolution": resolution or None, "note": note or None})
1512
+ for entry in missing:
1513
+ domain = _domain(entry.get("domain")) if isinstance(entry, dict) else ""
1514
+ if not domain:
1515
+ raise CompetitorSetError(f"{path.name} round #{index + 1} lists a missing rival without a valid domain")
1516
+ resolution = entry.get("resolution")
1517
+ if resolution not in (None, "", *MISSING_RESOLUTIONS):
1518
+ raise CompetitorSetError(f"{path.name} missing rival `{domain}` needs resolution added or rejected")
1519
+ review["missing"].append({"round": index + 1, "domain": domain, "why": _text(entry.get("why"), 240),
1520
+ "resolution": resolution or None, "note": _text(entry.get("note"), 240) or None})
1521
+ review["rounds"].append({"round": index + 1, "reviewer": reviewer, "reviewed_on": _text(item.get("reviewed_on"), 10) or None,
1522
+ "discovery_round": discovery_round, "checked": len(domains), "objections": len(objections),
1523
+ "missing": len(missing)})
1524
+ return review
1525
+
1526
+
1527
+ @dataclass
1528
+ class AgentFiles:
1529
+ listicles: dict[str, dict[str, Any]]
1530
+ profiles: dict[str, dict[str, Any]]
1531
+ decisions: dict[str, dict[str, Any]]
1532
+ review: dict[str, Any]
1533
+ paths: dict[str, Path]
1534
+
1535
+
1536
+ def load_agent_files(runtime: Path, decisions_path: Path) -> AgentFiles:
1537
+ paths = {
1538
+ "listicles": runtime / LISTICLES_FILENAME,
1539
+ "profiles": runtime / PROFILES_FILENAME,
1540
+ "decisions": decisions_path,
1541
+ "review": runtime / REVIEW_FILENAME,
1542
+ }
1543
+ return AgentFiles(
1544
+ listicles=load_listicles(paths["listicles"]),
1545
+ profiles=load_profiles(paths["profiles"]),
1546
+ decisions=load_decisions(paths["decisions"]),
1547
+ review=load_review(paths["review"]),
1548
+ paths=paths,
1549
+ )
1550
+
1551
+
1552
+ def add_agent_research(pool: dict[str, Candidate], files: AgentFiles, to_read: list[str], product_domain: str) -> list[str]:
1553
+ """Listicle entries and profiles the agent recorded become candidates
1554
+ with provenance `agent_research` and the page they came from; a profile
1555
+ of a domain no source recorded must say where it was found."""
1556
+ recorded = set(to_read)
1557
+ stray = sorted(url for url in files.listicles if url not in recorded)
1558
+ if stray:
1559
+ raise CompetitorSetError(
1560
+ "listicles.json reads pages the runner did not record: " + ", ".join(stray[:5])
1561
+ + " — keep the reading to the addresses in candidates.json `listicles_to_read`"
1562
+ )
1563
+ for url in to_read:
1564
+ page = files.listicles.get(url)
1565
+ if not page or page["status"] != "read":
1566
+ continue
1567
+ for entry in page["entries"]:
1568
+ domain = entry["domain"]
1569
+ if not domain or domain == product_domain or classify_domain(domain) in ("noise", "directory"):
1570
+ continue
1571
+ if domain not in pool:
1572
+ # Only a domain no source recorded gets the agent's provenance;
1573
+ # for a known one the listicle page is one more source.
1574
+ candidate = _candidate(pool, domain)
1575
+ candidate.add_provenance("agent_research")
1576
+ candidate.found_in = url
1577
+ candidate = pool[domain]
1578
+ candidate.add_source("listicle")
1579
+ if url not in candidate.listicle_pages:
1580
+ candidate.listicle_pages.append(url)
1581
+ additions: list[str] = []
1582
+ for domain, profile in sorted(files.profiles.items()):
1583
+ if domain == product_domain:
1584
+ raise CompetitorSetError(f"profiles.json profiles the product's own domain `{domain}`")
1585
+ if domain in pool or not profile["found_in"]:
1586
+ continue
1587
+ candidate = _candidate(pool, domain)
1588
+ candidate.add_provenance("agent_research")
1589
+ candidate.add_source("agent_research")
1590
+ candidate.found_in = profile["found_in"]
1591
+ additions.append(domain)
1592
+ return additions
1593
+
1594
+
1595
+ def own_additions(pool: dict[str, Candidate], additions: list[str]) -> list[str]:
1596
+ """The agent's own finds, after round 2: a profiled domain that round 2
1597
+ also discovered is a discovery, whatever page the profile names."""
1598
+ own = [domain for domain in additions if "discovered" not in pool[domain].provenance]
1599
+ if len(own) > MAX_AGENT_ADDITIONS:
1600
+ raise CompetitorSetError(f"profiles.json adds {len(own)} domains the discovery never saw; at most {MAX_AGENT_ADDITIONS} per run")
1601
+ return own
1602
+
1603
+
1604
+ # ── profiling list, tier rules and completeness ─────────────────────────────
1605
+
1606
+ def _capped(rows: list[Candidate], cap: int, decisions: dict[str, dict[str, Any]]) -> list[str]:
1607
+ """The first `cap` domains, not counting the ones already decided
1608
+ `excluded`: an obvious non-competitor costs no profile, so it frees its
1609
+ place for the next candidate."""
1610
+ chosen: list[str] = []
1611
+ counted = 0
1612
+ for candidate in rows:
1613
+ excluded = (decisions.get(candidate.domain) or {}).get("competitor_tier") == "excluded"
1614
+ if not excluded:
1615
+ if counted >= cap:
1616
+ continue
1617
+ counted += 1
1618
+ chosen.append(candidate.domain)
1619
+ return chosen
1620
+
1621
+
1622
+ def required_round1(pool: dict[str, Candidate], inputs: RunInputs, decisions: dict[str, dict[str, Any]] | None = None) -> list[str]:
1623
+ """Domains the agent must profile and decide in round 1: every named
1624
+ competitor (founder or product page), then every domain worth
1625
+ profiling (see `offer_signal`), strongest first, up to the cap."""
1626
+ decisions = decisions or {}
1627
+ aliases = {d for d, dec in decisions.items() if dec.get("same_as")}
1628
+ named = [d for d in inputs.named_domains if d in pool]
1629
+ page = [d for d in inputs.page_domains if d in pool and d not in named]
1630
+ fixed = set(named) | set(page)
1631
+ discovered = [c for c in pool.values() if c.domain not in fixed and c.domain not in aliases and c.offer_signal()]
1632
+ # Advertisers on the offer's own search first: the list exists for the ones that buy ads.
1633
+ discovered.sort(key=lambda c: (not (c.serp_paid_hits or c.keyword_ads_phrases), -c.round1_score(), -c.signal_count(), -len(c.sources), c.domain))
1634
+ return named + page + _capped(discovered, MAX_PROFILES_ROUND1, decisions)
1635
+
1636
+
1637
+ def required_round2(pool: dict[str, Candidate], required1: list[str], decisions: dict[str, dict[str, Any]] | None = None) -> list[str]:
1638
+ """A bounded stable-domain sample of relationship results; single hits remain eligible."""
1639
+ decisions = decisions or {}
1640
+ first = set(required1)
1641
+ rows = [
1642
+ c for c in pool.values()
1643
+ if c.domain not in first and not (decisions.get(c.domain) or {}).get("same_as")
1644
+ and c.relationship_hits()
1645
+ ]
1646
+ rows.sort(key=lambda c: c.domain)
1647
+ return _capped(rows, MAX_PROFILES_ROUND2, decisions)
1648
+
1649
+
1650
+ def apply_aliases(pool: dict[str, Candidate], decisions: dict[str, dict[str, Any]]) -> None:
1651
+ """A decision with `same_as` folds a second domain of the same offer (a
1652
+ redirect, an academy domain, a university brand on a partner's program)
1653
+ into the canonical one: its discovery facts count there, and it is
1654
+ recorded as excluded."""
1655
+ for domain, decision in sorted(decisions.items()):
1656
+ canonical = decision.get("same_as")
1657
+ if not canonical or domain not in pool:
1658
+ continue
1659
+ if canonical not in pool:
1660
+ raise CompetitorSetError(
1661
+ f"decisions.json folds `{domain}` into `{canonical}`, which no source recorded: profile `{canonical}` with found_in first"
1662
+ )
1663
+ pool[canonical].absorb(pool[domain])
1664
+
1665
+
1666
+ def price_band(price_usd: float | None, floor: float | None) -> str:
1667
+ if floor is None:
1668
+ return "no_floor"
1669
+ if price_usd is None:
1670
+ return "unknown"
1671
+ return "at_or_above_floor" if price_usd >= floor else "below_floor"
1672
+
1673
+
1674
+ @dataclass
1675
+ class Assessment:
1676
+ domain: str
1677
+ tier: str
1678
+ named: bool
1679
+ profile: dict[str, Any] | None
1680
+ decision: dict[str, Any]
1681
+ price_band: str
1682
+ meets_direct: bool
1683
+ comparable: bool
1684
+ unmet: list[str]
1685
+ group: int | None
1686
+ reason_code: str
1687
+
1688
+
1689
+ def assess(domain: str, decision: dict[str, Any], profile: dict[str, Any] | None, named: bool, floor: float | None) -> Assessment:
1690
+ """Record caller-supplied comparison roles; prices and ad activity are facts,
1691
+ not automatic eligibility or business-priority rules."""
1692
+ tier = decision["competitor_tier"]
1693
+ band = price_band(profile["price_usd"] if profile else None, floor)
1694
+ group = None if tier == "excluded" else {"direct": 0, "indirect": 1}.get(tier, 2)
1695
+ code = ("same_as" if decision.get("same_as") else "excluded") if group is None else tier
1696
+ return Assessment(domain=domain, tier=tier, named=named, profile=profile, decision=decision, price_band=band,
1697
+ meets_direct=tier == "direct", comparable=group is not None, unmet=[], group=group, reason_code=code)
1698
+
1699
+
1700
+ def check_consistency(assessments: dict[str, Assessment]) -> None:
1701
+ """Decisions require reasons and sourced profiles in load_agent_files.
1702
+ No built-in price/buyer/offer formula overrides the caller's scope."""
1703
+ return None
1704
+
1705
+
1706
+ def missing_items(*, domains: set[str], required: list[str], inputs: RunInputs, files: AgentFiles, to_read: list[str],
1707
+ assessments: dict[str, Assessment]) -> dict[str, list[str]]:
1708
+ """Identify missing source records for this chosen collection operation."""
1709
+ listicles = [url for url in to_read if url not in files.listicles]
1710
+ profiles: list[str] = []
1711
+ decisions: list[str] = []
1712
+ for domain in required:
1713
+ decision = files.decisions.get(domain)
1714
+ if decision is None:
1715
+ decisions.append(domain)
1716
+ needs_profile = domain in inputs.named_domains or (decision is not None and decision["competitor_tier"] != "excluded") or decision is None
1717
+ if needs_profile and domain not in files.profiles:
1718
+ profiles.append(domain)
1719
+ for domain in sorted(d for d in files.profiles if d in domains):
1720
+ if domain not in files.decisions and domain not in decisions:
1721
+ decisions.append(domain)
1722
+ for domain in sorted(d for d, dec in files.decisions.items() if d in domains and dec["competitor_tier"] != "excluded"):
1723
+ if domain not in files.profiles and domain not in profiles:
1724
+ profiles.append(domain)
1725
+ missing = {"listicles": listicles, "profiles": profiles, "decisions": decisions}
1726
+ return {key: value for key, value in missing.items() if value}
1727
+
1728
+
1729
+ def build_assessments(pool: dict[str, Candidate], files: AgentFiles, inputs: RunInputs) -> dict[str, Assessment]:
1730
+ return {
1731
+ domain: assess(domain, decision, files.profiles.get(domain), domain in inputs.named_domains, inputs.price_floor_usd)
1732
+ for domain, decision in files.decisions.items() if domain in pool
1733
+ }
1734
+
1735
+
1736
+ def round2_seeds(pool: dict[str, Candidate], assessments: dict[str, Assessment], inputs: RunInputs) -> list[str]:
1737
+ """Up to three confirmed direct competitors (those named by the founder first, then
1738
+ by ads on the offer's phrases), topped up with comparable indirect ones.
1739
+ A domain whose profile says the offer is a side line of a big
1740
+ multi-product site (`focused: false`) is never a seed: its keyword and
1741
+ PPC neighbours are the rest of that site's business, not this offer's.
1742
+ Nor is a seller's subdomain on a course host: the tools measure the host."""
1743
+ def key(item: Assessment) -> tuple:
1744
+ candidate = pool[item.domain]
1745
+ ads = len(candidate.keyword_ads_phrases) + candidate.serp_paid_hits
1746
+ return (0 if item.named else 1, -ads, -candidate.round1_score(), item.domain)
1747
+
1748
+ def usable(item: Assessment) -> bool:
1749
+ return bool(item.profile and item.profile.get("focused", True)) and not hosted_seller(item.domain)
1750
+
1751
+ direct = sorted((a for a in assessments.values() if a.group == 0 and usable(a)), key=key)
1752
+ seeds = [a.domain for a in direct[:ROUND2_SEEDS]]
1753
+ if len(seeds) < ROUND2_SEEDS:
1754
+ comparable = sorted((a for a in assessments.values() if a.group == 1 and usable(a)), key=key)
1755
+ seeds += [a.domain for a in comparable[: ROUND2_SEEDS - len(seeds)]]
1756
+ return seeds
1757
+
1758
+
1759
+ # ── ad check ────────────────────────────────────────────────────────────────
1760
+
1761
+ def scan_home(client: OsClient, domain: str) -> dict[str, Any]:
1762
+ """Home page plus up to two Google Tag Manager containers; the ad tags
1763
+ of the Channel Footprint signature list."""
1764
+ cf = _channel_footprint()
1765
+ page, _ = client.fetch_page(f"https://{domain}/")
1766
+ status = page.get("status")
1767
+ body = str(page.get("body") or "")
1768
+ if not isinstance(status, int):
1769
+ return {"fetch_status": "error", "tags": []}
1770
+ if status >= 400:
1771
+ return {"fetch_status": "blocked" if status in (401, 403, 429, 503) else "error", "tags": []}
1772
+ if not body.strip():
1773
+ return {"fetch_status": "empty", "tags": []}
1774
+ blobs = [body]
1775
+ for container in list(dict.fromkeys(cf.GTM_ID_RE.findall(body)))[:GTM_CONTAINERS_PER_SITE]:
1776
+ gtm, _ = client.fetch_page(f"https://www.googletagmanager.com/gtm.js?id={container}")
1777
+ blobs.append(str(gtm.get("body") or ""))
1778
+ tags = sorted({t["tag"] for t in cf.detect_tags(blobs) if t["channel"] in AD_TAG_CHANNELS})
1779
+ return {"fetch_status": "ok", "tags": tags}
1780
+
1781
+
1782
+ def _meta_matches(rows: list[Any], domain: str) -> int:
1783
+ """Ads whose destination is the domain, or whose page name is the brand
1784
+ when the ad carries no usable destination (the Channel Footprint rule)."""
1785
+ cf = _channel_footprint()
1786
+ brand = brand_token(domain)
1787
+ matched = 0
1788
+ for row in rows:
1789
+ if not isinstance(row, dict):
1790
+ continue
1791
+ links = row.get("ad_creative_link_captions") or []
1792
+ if isinstance(links, str):
1793
+ links = [links]
1794
+ destinations = {cf.usable_destination(x) for x in links if x} - {""} if isinstance(links, list) else set()
1795
+ if destinations:
1796
+ matched += int(domain in destinations)
1797
+ elif cf._name_matches(str(row.get("page_name") or ""), brand):
1798
+ matched += 1
1799
+ return matched
1800
+
1801
+
1802
+ def check_ads(client: OsClient, inputs: RunInputs, candidate: Candidate, *, focused: bool = True) -> dict[str, Any]:
1803
+ """Does the competitor buy media? The round-1 ad facts on the offer's
1804
+ phrases, SpyFu's modeled Google spend (the primary market, then up to two
1805
+ more target markets), ad pixels on the home page and, when nothing on
1806
+ Google shows, the official Meta Ad Library (EU delivery).
1807
+
1808
+ For an offer that is a side line of a multi-product domain (`focused:
1809
+ false`) only the ads on the offer's own phrases count: the domain's
1810
+ overall spend, pixels and Meta ads promote the rest of its business."""
1811
+ cf = _channel_footprint()
1812
+ kinds: list[str] = []
1813
+ facts: list[str] = []
1814
+ lookups: list[str] = []
1815
+ budget: float | None = None
1816
+ answered = False
1817
+ if candidate.keyword_ads_phrases:
1818
+ kinds.append("category_keyword_ads")
1819
+ seen = f", last seen {candidate.keyword_ads_last_seen}" if candidate.keyword_ads_last_seen else ""
1820
+ facts.append("Google ads on " + ", ".join(f"'{p}'" for p in candidate.keyword_ads_phrases) + f" (SpyFu{seen})")
1821
+ if candidate.serp_paid_hits:
1822
+ kinds.append("category_paid_result")
1823
+ facts.append("paid result for " + ", ".join(f"'{q}'" for q in candidate.paid_queries[:3]))
1824
+ if not focused:
1825
+ lookups.append("domain-wide checks skipped: the offer is a side line of a multi-product domain")
1826
+ return {"ad_activity": "ads_seen" if kinds else "unknown", "ad_evidence": facts[:6], "ad_kinds": kinds,
1827
+ "ad_strength": len(kinds), "google_budget_usd": None, "pixel_tags": [], "lookup_gaps": lookups}
1828
+ markets = [m for m in inputs.markets if m in SPYFU_MARKETS][: 1 + AD_CHECK_EXTRA_MARKETS]
1829
+ if hosted_seller(candidate.domain):
1830
+ # SpyFu models the host's domain; that spend is not this seller's.
1831
+ markets = []
1832
+ lookups.append("spyfu: a seller's subdomain on a course host, not measured")
1833
+ for market in markets if client.max_spyfu_calls > 0 else []:
1834
+ try:
1835
+ data, retrieved_on = client.spyfu("domain-stats", {"domain": candidate.domain, "country_code": market, "months": AD_CHECK_MONTHS})
1836
+ except BudgetExhausted:
1837
+ raise
1838
+ except CompetitorSetError as exc:
1839
+ lookups.append(f"spyfu {market}: {str(exc)[:80]}")
1840
+ continue
1841
+ if data.get("unavailable"):
1842
+ lookups.append(f"spyfu {market}: rejected")
1843
+ continue
1844
+ row = cf._paid_search_row(candidate.domain, market, data, retrieved_on)
1845
+ if row["found"]:
1846
+ answered = True
1847
+ if row["months_with_spend"]:
1848
+ kinds.append("google_spend")
1849
+ budget = row["avg_monthly_budget_usd"]
1850
+ amount = f", about {_money(float(budget))}/month" if budget else ""
1851
+ facts.append(f"Google Ads spend in {row['months_with_spend']} of {row['months_checked']} month(s) in {market} (SpyFu{amount})")
1852
+ break
1853
+ scan = scan_home(client, candidate.domain)
1854
+ if scan["fetch_status"] == "ok":
1855
+ answered = True
1856
+ else:
1857
+ lookups.append(f"home page: {scan['fetch_status']}")
1858
+ if not kinds and client.max_meta_searches > 0:
1859
+ try:
1860
+ response, _ = client.meta_library({"query": brand_token(candidate.domain), "countries": [inputs.eu_country],
1861
+ "activeStatus": "ACTIVE", "limit": META_ADS_LIMIT})
1862
+ except BudgetExhausted:
1863
+ raise
1864
+ except CompetitorSetError as exc:
1865
+ lookups.append(f"meta: {str(exc)[:80]}")
1866
+ else:
1867
+ if response.get("unavailable"):
1868
+ lookups.append("meta: rejected")
1869
+ else:
1870
+ answered = True
1871
+ matched = _meta_matches(response.get("data") or [], candidate.domain)
1872
+ if matched:
1873
+ kinds.append("meta_eu")
1874
+ facts.append(f"{matched} active Meta ad(s) delivered in {inputs.eu_country} (EU library)")
1875
+ if scan["tags"]:
1876
+ facts.append("ad pixels on the home page: " + ", ".join(scan["tags"]))
1877
+ verdict = "ads_seen" if kinds else "pixels_only" if scan["tags"] else "none_detected" if answered else "unknown"
1878
+ return {"ad_activity": verdict, "ad_evidence": facts[:6], "ad_kinds": kinds, "ad_strength": len(kinds),
1879
+ "google_budget_usd": budget, "pixel_tags": scan["tags"], "lookup_gaps": lookups[:4]}
1880
+
1881
+
1882
+ # ── composition ─────────────────────────────────────────────────────────────
1883
+
1884
+ def _order_key(item: Assessment, ads: dict[str, dict[str, Any]], pool: dict[str, Candidate]) -> tuple:
1885
+ # Stable inspection order, never a recommendation ranking.
1886
+ return (item.domain,)
1887
+
1888
+
1889
+ def compose(eligible: list[Assessment], ads: dict[str, dict[str, Any]], pool: dict[str, Candidate], cap: int) -> tuple[list[Assessment], list[Assessment]]:
1890
+ """Preserve explicitly named scope, then take a stable-domain bounded sample."""
1891
+ ordered = sorted(eligible, key=lambda a: _order_key(a, ads, pool))
1892
+ reserved = [a for a in ordered if a.named]
1893
+ others = [a for a in ordered if not a.named]
1894
+ room = max(cap - len(reserved), 0)
1895
+ chosen = sorted(reserved + others[:room], key=lambda a: _order_key(a, ads, pool))
1896
+ return chosen, others[room:]
1897
+
1898
+
1899
+ def _selection_text(item: Assessment, ad: dict[str, Any], cap: int, direct_count: int) -> str:
1900
+ return f"{item.tier}; caller-scoped inspection sample; {AD_LABELS[ad['ad_activity']]}"
1901
+
1902
+
1903
+ def _not_eligible_reason(item: Assessment, floor: float | None) -> str:
1904
+ if item.reason_code == "same_as":
1905
+ return f"same offer as {item.decision['same_as']}: {item.decision['reason']}"
1906
+ return item.decision["reason"]
1907
+
1908
+
1909
+ # ── rendering ───────────────────────────────────────────────────────────────
1910
+
1911
+ def _cell(value: Any, cap: int = 150) -> str:
1912
+ text = " ".join(str(value if value is not None and value != "" else "—").split()).replace("|", "\\|")
1913
+ return text if len(text) <= cap else text[: cap - 1] + "…"
1914
+
1915
+
1916
+ def _offer_lines(inputs: RunInputs) -> list[str]:
1917
+ offer = inputs.offer
1918
+ lines = []
1919
+ if offer.get("name") or offer.get("url"):
1920
+ lines.append(f"- Offer: {_cell(offer.get('name') or 'the product', 120)}" + (f" ({offer['url']})" if offer.get("url") else ""))
1921
+ for key, label in (("job", "Job"), ("buyer", "Buyer"), ("offer_class", "Offer class"), ("comparable_classes", "Comparable offer classes"),
1922
+ ("duration", "Duration"), ("price_text", "Price")):
1923
+ if offer.get(key):
1924
+ lines.append(f"- {label}: {_cell(offer[key], 300)}")
1925
+ if inputs.price_floor_usd is not None:
1926
+ basis = f" ({offer['price_floor_basis']})" if offer.get("price_floor_basis") else ""
1927
+ lines.append(f"- Caller-supplied comparison floor: {_money(inputs.price_floor_usd)}{basis}")
1928
+ else:
1929
+ lines.append("- Caller-supplied comparison floor: not set; no default is inferred")
1930
+ lines.append(f"- Category phrases: {', '.join(inputs.category_phrases) or 'none'}")
1931
+ named = ", ".join(inputs.named_domains) or ("explicitly none" if inputs.input_annotations.get("operator_named_asked") is True else "unknown")
1932
+ lines.append(f"- Competitors named by the founder: {named}; mentioned on the product page: {', '.join(inputs.page_domains) or 'not supplied'}")
1933
+ lines.append(f"- Markets: {', '.join(inputs.markets)}; product domain {inputs.product_domain or 'not given'} (excluded, never a search seed)")
1934
+ if inputs.offer.get("confirmed_by_operator"):
1935
+ lines.append("- The founder confirmed this offer: " + _cell(inputs.offer.get("basis"), 300))
1936
+ elif inputs.offer.get("inferred"):
1937
+ lines.append("- The offer is an inference: " + _cell(inputs.offer.get("basis"), 300))
1938
+ elif inputs.intake_path:
1939
+ lines.append("- The offer card is not marked as confirmed by the founder.")
1940
+ else:
1941
+ lines.append(f"- No offer card was recorded ({INTAKE_REL.as_posix()} is missing); the inputs came from the command line.")
1942
+ return lines
1943
+
1944
+
1945
+ def render_section(*, inputs: RunInputs, cap: int, competitors: list[dict[str, Any]], profiled_out: list[dict[str, Any]],
1946
+ excluded: list[dict[str, Any]], summaries: list[dict[str, Any]], pool_size: int, coverage: dict[str, Any],
1947
+ totals: dict[str, Any], retrieved_on: str) -> str:
1948
+ lines: list[str] = []
1949
+ add = lines.append
1950
+ groups = {g: sum(1 for c in competitors if c["selection_group"] == g) for g in GROUP_LABELS.values()}
1951
+ add(
1952
+ f"Competitor discovery retrieved {retrieved_on}: {pool_size} candidate domain(s) from {len(summaries)} provider "
1953
+ f"query/report(s) in two rounds; {coverage['profiles']} profiled; {len(competitors)} competitor(s) kept "
1954
+ f"({groups['direct']} direct, {groups['indirect']} indirect, {groups['other scoped']} "
1955
+ f"other scoped); {len(excluded)} candidate(s) recorded as not included."
1956
+ )
1957
+ add("")
1958
+ add("### Offer searched from")
1959
+ add("")
1960
+ lines.extend(_offer_lines(inputs))
1961
+ add("")
1962
+ add("### Inspection scope")
1963
+ add("")
1964
+ add(f"- Up to {cap} caller-classified competitors, with explicitly named domains retained and the rest in stable domain order.")
1965
+ add("- Direct, indirect and substitute are recorded comparison roles. Price, ad activity and signal counts do not choose a business priority or exclude a substitute.")
1966
+ add("")
1967
+ add("### Competitor set")
1968
+ add("")
1969
+ if competitors:
1970
+ add("| # | Competitor | Tier | Provenance | Offer and format | Price | Ads | Why in the list |")
1971
+ add("| ---: | --- | --- | --- | --- | --- | --- | --- |")
1972
+ for row in competitors:
1973
+ offer = " — ".join(x for x in (row.get("offer"), row.get("offer_class")) if x) or None
1974
+ price = row.get("price_text") or _money(row.get("price_usd"))
1975
+ add(f"| {row['rank']} | {_cell(row['name'], 40)} ({_cell(row['domain'], 40)}) | {row['competitor_tier']} | {_cell(row['provenance_detail'], 60)} "
1976
+ f"| {_cell(offer, 110)} | {_cell(price, 50)} | {_cell('; '.join([AD_LABELS[row['ad_activity']]] + row['ad_evidence'][:2]), 120)} | {_cell(row['selection'], 140)} |")
1977
+ else:
1978
+ add("No competitor was confirmed in this run.")
1979
+ overrides = [row for row in competitors + profiled_out if row.get("override_reason")]
1980
+ if overrides:
1981
+ add("")
1982
+ add("Additional classification rationale: " + "; ".join(f"{r['domain']}: {_cell(r['override_reason'], 160)}" for r in overrides))
1983
+ add("")
1984
+ add("### Profiled, not included")
1985
+ add("")
1986
+ if profiled_out:
1987
+ add("| Domain | Tier | Offer and format | Price | Ads | Why not included |")
1988
+ add("| --- | --- | --- | --- | --- | --- |")
1989
+ for row in profiled_out[:PROFILED_ROWS_RENDERED]:
1990
+ offer = " — ".join(x for x in (row.get("offer"), row.get("offer_class")) if x) or None
1991
+ price = row.get("price_text") or _money(row.get("price_usd"))
1992
+ ads = AD_LABELS.get(row.get("ad_activity") or "", "not checked")
1993
+ add(f"| {_cell(row['domain'], 40)} | {row.get('competitor_tier') or '—'} | {_cell(offer, 110)} | {_cell(price, 50)} | {ads} | {_cell(row['reason'], 160)} |")
1994
+ if len(profiled_out) > PROFILED_ROWS_RENDERED:
1995
+ add("")
1996
+ add(f"{len(profiled_out) - PROFILED_ROWS_RENDERED} more profiled candidate(s) are recorded in the evidence file.")
1997
+ else:
1998
+ add("Every profiled candidate is in the list.")
1999
+ others = [row for row in excluded if not row.get("profiled") and row["reason_code"] not in ("directory", "over_cap")]
2000
+ if others:
2001
+ add("")
2002
+ counts: dict[str, int] = {}
2003
+ for row in others:
2004
+ counts[row["reason_code"]] = counts.get(row["reason_code"], 0) + 1
2005
+ add("Not profiled: " + ", ".join(f"{n} {code.replace('_', ' ')}" for code, n in sorted(counts.items())) + "; examples: "
2006
+ + ", ".join(row["domain"] for row in others[:EXCLUDED_ROWS_RENDERED]) + ".")
2007
+ directories = [row["domain"] for row in excluded if row["reason_code"] == "directory"]
2008
+ if directories:
2009
+ add("")
2010
+ add("Review and listing directories (recorded, never promoted): " + ", ".join(directories[:EXCLUDED_ROWS_RENDERED]) + ".")
2011
+ add("")
2012
+ add("### Discovery and enrichment coverage")
2013
+ add("")
2014
+ add("| Round | Query | Source | Items | Candidates |")
2015
+ add("| ---: | --- | --- | ---: | ---: |")
2016
+ for row in summaries:
2017
+ items = "unavailable" if row.get("unavailable") else str(row["retrieved"])
2018
+ add(f"| {row['round']} | {_cell(row['query'], 80)} | {row['source']} | {items} | {row['candidates']} |")
2019
+ if any(row.get("unavailable") for row in summaries):
2020
+ add("")
2021
+ add("Unavailable queries were rejected by the provider or failed; they are recorded, not retried silently.")
2022
+ add("")
2023
+ add(f"- Listicles: {coverage['listicles_read']} page(s) read, {coverage['listicles_unreadable']} unreadable, "
2024
+ f"{coverage['listicle_entries']} program entries recorded.")
2025
+ add(f"- Profiles: {coverage['profiles']} domain(s) profiled on their own sites; {coverage['agent_additions']} added by the agent "
2026
+ f"from its reading, each with the page it came from.")
2027
+ seeds = ", ".join(coverage["round2_seeds"]) or "none (no confirmed direct or comparable competitor)"
2028
+ add(f"- Round 2 seeds (confirmed direct competitors): {seeds}.")
2029
+ add(f"- Review: {coverage['review_rounds']} optional recorded review round(s); {coverage['objections_accepted']} objection(s) "
2030
+ f"accepted, {coverage['objections_rejected']} rejected; {coverage['missing_added']} missing rival(s) added, "
2031
+ f"{coverage['missing_rejected']} rejected.")
2032
+ add(f"- Ad check: {coverage['ad_checks']} domain(s); " + ", ".join(f"{n} {AD_LABELS[k]}" for k, n in coverage["ad_counts"].items() if n) + ".")
2033
+ add("")
2034
+ add("### Method and coverage")
2035
+ add("")
2036
+ add(
2037
+ f"- Spend recorded: {totals['dataforseo_calls']} answered DataForSEO task(s), {totals['spyfu_calls']} SpyFu lookup(s), "
2038
+ f"{totals['search_calls']} web search(es), {totals['meta_searches']} Meta Ad Library search(es) and {totals['page_fetches']} free page fetch(es)"
2039
+ + (f", plus {totals['rejected_calls']} rejected query(ies) that count as spend but not as evidence" if totals.get("rejected_calls") else "")
2040
+ + ", accumulated across attempts; cached replays add no new spend."
2041
+ )
2042
+ add(
2043
+ "- Signal strength orders the profiling list and breaks ties, never a tier: a paid result on a category query "
2044
+ f"{SCORE_CATEGORY_PAID:g} (up to {CAP_CATEGORY_PAID}), a Google advertiser on a category phrase {SCORE_KEYWORD_ADS:g} "
2045
+ f"(up to {CAP_KEYWORD_ADS}), an organic category result {SCORE_CATEGORY_ORGANIC:g} (up to {CAP_CATEGORY_ORGANIC}), a second-engine "
2046
+ f"result {SCORE_SECOND_ENGINE:g} (up to {CAP_SECOND_ENGINE}), an alternatives-query hit {SCORE_BRAND_QUERY:g} (up to {CAP_BRAND_QUERY}), "
2047
+ f"a listicle page {SCORE_LISTICLE:g} (up to {CAP_LISTICLE}), a round-2 relationship hit {SCORE_RELATIONSHIP:g} (up to {CAP_RELATIONSHIP}). "
2048
+ "Being named by the founder or on the product page adds nothing."
2049
+ )
2050
+ add(
2051
+ f"- Caps: {MAX_CATEGORY_PHRASES} category phrases (two queries each), {MAX_BRAND_QUERIES} alternatives queries for competitors named by the founder; "
2052
+ f"{MAX_LISTICLES_TO_READ} listicle pages read, {MAX_PROFILES_ROUND1} profiles in round 1 and {MAX_PROFILES_ROUND2} in round 2, "
2053
+ f"{MAX_AGENT_ADDITIONS} agent additions, {ROUND2_SEEDS} round-2 seeds, {MAX_AD_CHECKS} ad checks, {cap} competitors."
2054
+ )
2055
+ add("")
2056
+ add("### Sources")
2057
+ add("")
2058
+ add(
2059
+ "- DataForSEO Google SERP (paid and organic) and Labs `competitors_domain`, SpyFu keyword ad history, PPC competitors and "
2060
+ "domain stats, the official Meta Ad Library, Brave web search via OS tools, free home-page fetches, and the executing "
2061
+ f"agent's listicle reading, site profiles and optional review — retrieved {retrieved_on}; the candidate, competitor and "
2062
+ f"exclusion rows in `{EVIDENCE_REL.as_posix()}` are exactly the set every table above is computed from."
2063
+ )
2064
+ return "\n".join(lines).strip() + "\n"
2065
+
2066
+
2067
+ # ── orchestration ───────────────────────────────────────────────────────────
2068
+
2069
+ def _write_atomic(path: Path, content: str) -> None:
2070
+ path.parent.mkdir(parents=True, exist_ok=True)
2071
+ tmp = path.with_suffix(path.suffix + ".tmp")
2072
+ tmp.write_text(content, encoding="utf-8")
2073
+ tmp.replace(path)
2074
+
2075
+
2076
+ def _sha(path: Path) -> str | None:
2077
+ return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None
2078
+
2079
+
2080
+ def run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
2081
+ workspace = workspace.resolve()
2082
+ io = _load_script("research-artifact-io.py", "pmr_artifact_io")
2083
+ with io.section_attempt(workspace, SECTION_ID, context_file=getattr(args, "context_file", None)):
2084
+ return _run(workspace, args)
2085
+
2086
+
2087
+ def _inputs_block(inputs: RunInputs, cap: int) -> dict[str, Any]:
2088
+ return {
2089
+ "offer": inputs.offer,
2090
+ "operator_named": inputs.named_domains,
2091
+ "page_mentioned": inputs.page_domains,
2092
+ "keyword_domains": inputs.keyword_domains,
2093
+ "product_domain": inputs.product_domain,
2094
+ "category_phrases": inputs.category_phrases,
2095
+ "queries": inputs.category_queries + inputs.brand_queries,
2096
+ "price_floor_usd": inputs.price_floor_usd,
2097
+ "markets": inputs.markets,
2098
+ "location_code": inputs.location_code,
2099
+ "language_code": inputs.language_code,
2100
+ "max_competitors": cap,
2101
+ "intake_path": inputs.intake_path,
2102
+ "input_annotations": inputs.input_annotations,
2103
+ }
2104
+
2105
+
2106
+ def _pending(workspace: Path, runtime: Path, decisions_path: Path, client: OsClient, reason: str, next_step: str, missing: dict[str, list[str]]) -> dict[str, Any]:
2107
+ state = {
2108
+ "schema_version": SCHEMA_VERSION,
2109
+ "section_id": SECTION_ID,
2110
+ "status": "pending",
2111
+ "inputs_path": (RUNTIME_REL / "inputs.json").as_posix(),
2112
+ "inputs_sha256": _sha(runtime / "inputs.json"),
2113
+ "research_context": _load_script("research-artifact-io.py", "pmr_input_io").context_binding(workspace, SECTION_ID),
2114
+ "error": None,
2115
+ "reason": reason,
2116
+ "next_step": next_step,
2117
+ "missing": missing,
2118
+ "candidates_path": (RUNTIME_REL / CANDIDATES_FILENAME).as_posix(),
2119
+ "decisions_path": decisions_path.as_posix(),
2120
+ "undecided": missing.get("decisions", []),
2121
+ "attempt_dataforseo_tasks": client.dataforseo_tasks,
2122
+ "attempt_search_calls": client.search_calls,
2123
+ "attempt_spyfu_calls": client.spyfu_calls,
2124
+ "attempt_cache_hits": client.cache_hits,
2125
+ }
2126
+ _write_atomic(runtime / "request.json", json.dumps(state, indent=1))
2127
+ return state
2128
+
2129
+
2130
+ def _write_candidates(runtime: Path, *, inputs: RunInputs, cap: int, files: AgentFiles, discovery: Discovery, to_read: list[str],
2131
+ required1: list[str], required2: list[str] | None, seeds: list[str] | None, missing: dict[str, list[str]], status: str) -> None:
2132
+ profiled = set(files.profiles)
2133
+ rows = sorted(discovery.pool.values(), key=lambda c: (-c.score(), c.domain))
2134
+ payload = {
2135
+ "schema_version": SCHEMA_VERSION,
2136
+ "section_id": SECTION_ID,
2137
+ "status": status,
2138
+ "files": {name: path.as_posix() for name, path in files.paths.items()},
2139
+ "inputs": _inputs_block(inputs, cap),
2140
+ "listicles_to_read": to_read,
2141
+ "profile_required": {"round1": required1, "round2": required2 if required2 is not None else "after round 1 is complete"},
2142
+ "round2_seeds": seeds,
2143
+ "missing": missing,
2144
+ "rules": {
2145
+ "roles": "caller-supplied roles with sourced profile and reason",
2146
+ "price": "recorded comparison only; no automatic floor",
2147
+ "order": "explicitly named scope retained; stable domain inspection sample",
2148
+ },
2149
+ "candidates": [{**c.row(), "required": c.domain in required1 or c.domain in (required2 or []), "profiled": c.domain in profiled} for c in rows],
2150
+ "directories": sorted(discovery.directories),
2151
+ "search_summaries": discovery.summaries,
2152
+ }
2153
+ _write_atomic(runtime / CANDIDATES_FILENAME, json.dumps(payload, indent=1, ensure_ascii=False))
2154
+
2155
+
2156
+ def _pending_message(missing: dict[str, list[str]]) -> str:
2157
+ parts = []
2158
+ for key, label in (("listicles", "listicle page(s) to read"), ("profiles", "domain(s) to profile"), ("decisions", "domain(s) to decide"),
2159
+ ("review", "domain(s) for the reviewer"), ("objections", "objection(s) to resolve"), ("missing_rivals", "missing rival(s) to settle"),
2160
+ ("round2_review", "round-2 domain(s) for a review round of the round-2 pool")):
2161
+ if missing.get(key):
2162
+ parts.append(f"{len(missing[key])} {label}")
2163
+ return ", ".join(parts)
2164
+
2165
+
2166
+ def _run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
2167
+ workspace = workspace.resolve()
2168
+ runtime = workspace / RUNTIME_REL
2169
+ cache_dir = runtime / "cache"
2170
+ runtime.mkdir(parents=True, exist_ok=True)
2171
+
2172
+ api_key = (os.environ.get("OS_API_KEY") or "").strip()
2173
+ if not api_key:
2174
+ raise CompetitorSetError("OS_API_KEY is required")
2175
+ os_url = (os.environ.get("OS_URL") or DEFAULT_OS_URL).strip() or DEFAULT_OS_URL
2176
+
2177
+ cap = max(1, int(_arg(args, "max_competitors", MAX_COMPETITORS)))
2178
+ inputs = bind_inputs(workspace, args)
2179
+ client = OsClient(
2180
+ os_url=os_url,
2181
+ api_key=api_key,
2182
+ cache_dir=cache_dir,
2183
+ max_dataforseo_tasks=int(_arg(args, "max_dataforseo_tasks", MAX_DATAFORSEO_TASKS_PER_ATTEMPT)),
2184
+ max_search_calls=int(_arg(args, "max_search_calls", MAX_SEARCH_CALLS_PER_ATTEMPT)),
2185
+ max_spyfu_calls=int(_arg(args, "max_spyfu_calls", MAX_SPYFU_CALLS_PER_ATTEMPT)),
2186
+ max_meta_searches=int(_arg(args, "max_meta_searches", MAX_META_SEARCHES_PER_ATTEMPT)),
2187
+ max_fetches=int(_arg(args, "max_fetches", MAX_FETCHES_PER_ATTEMPT)),
2188
+ )
2189
+
2190
+ discovery = run_round1(client, inputs)
2191
+ if not discovery.summaries:
2192
+ raise CompetitorSetError("no discovery query could be formed from the inputs: give the offer's category phrases or competitors named by the founder")
2193
+ if not any(not s.get("unavailable") for s in discovery.summaries):
2194
+ # Nothing proves the market was searched: a provider that rejected or
2195
+ # failed every query must not become a "completed" section with an
2196
+ # empty or named-only competitor list.
2197
+ rejected = sum(1 for s in discovery.summaries if s.get("unavailable"))
2198
+ raise CompetitorSetError(
2199
+ f"discovery unavailable: none of the {len(discovery.summaries)} provider queries answered "
2200
+ f"({rejected} rejected or failed); the section cannot complete without a successful search. "
2201
+ "Rejected DataForSEO queries are cached and will not be retried unchanged: change the category phrases "
2202
+ "or retry later if the web search was rate-limited."
2203
+ )
2204
+
2205
+ decisions_arg = str(_arg(args, "decisions", "") or "")
2206
+ decisions_path = Path(decisions_arg) if decisions_arg else runtime / DECISIONS_FILENAME
2207
+ if not decisions_path.is_absolute():
2208
+ decisions_path = workspace / decisions_path
2209
+ files = load_agent_files(runtime, decisions_path)
2210
+ to_read = listicles_to_read(discovery.summaries, inputs.product_domain)
2211
+ additions = add_agent_research(discovery.pool, files, to_read, inputs.product_domain)
2212
+ apply_aliases(discovery.pool, files.decisions)
2213
+ required1 = required_round1(discovery.pool, inputs, files.decisions)
2214
+ round1_domains = set(discovery.pool)
2215
+ assessments = build_assessments(discovery.pool, files, inputs)
2216
+ check_consistency(assessments)
2217
+ missing1 = missing_items(domains=round1_domains, required=required1, inputs=inputs, files=files, to_read=to_read, assessments=assessments)
2218
+ if missing1:
2219
+ _write_candidates(runtime, inputs=inputs, cap=cap, files=files, discovery=discovery, to_read=to_read, required1=required1,
2220
+ required2=None, seeds=None, missing=missing1, status="enrich_round1")
2221
+ return _pending(workspace, runtime, decisions_path, client, "enrichment of round 1 incomplete: " + _pending_message(missing1), "enrich_round1", missing1)
2222
+
2223
+ seeds = round2_seeds(discovery.pool, assessments, inputs)
2224
+ if seeds:
2225
+ run_round2(client, inputs, discovery, seeds)
2226
+ additions = own_additions(discovery.pool, additions)
2227
+ unknown = sorted(d for d in set(files.decisions) | {d for d, p in files.profiles.items() if not p["found_in"]} if d not in discovery.pool)
2228
+ if unknown:
2229
+ raise CompetitorSetError(
2230
+ "decisions or profiles name domains no source recorded: " + ", ".join(unknown[:10])
2231
+ + " — give a profile with found_in (the page where the agent found it), or remove them (the round-2 seeds may have changed)"
2232
+ )
2233
+ apply_aliases(discovery.pool, files.decisions)
2234
+ required2 = required_round2(discovery.pool, required1, files.decisions)
2235
+ assessments = build_assessments(discovery.pool, files, inputs)
2236
+ check_consistency(assessments)
2237
+ missing2 = missing_items(domains=set(discovery.pool), required=required1 + required2, inputs=inputs, files=files, to_read=to_read, assessments=assessments)
2238
+ if missing2:
2239
+ _write_candidates(runtime, inputs=inputs, cap=cap, files=files, discovery=discovery, to_read=to_read, required1=required1,
2240
+ required2=required2, seeds=seeds, missing=missing2, status="enrich_round2")
2241
+ return _pending(workspace, runtime, decisions_path, client, "enrichment of round 2 incomplete: " + _pending_message(missing2), "enrich_round2", missing2)
2242
+
2243
+ # Ad check for every domain that can enter the list (those named by the founder first, then in tier order).
2244
+ eligible = [a for a in assessments.values() if a.group is not None]
2245
+ eligible.sort(key=lambda a: (0 if a.named else 1, a.domain))
2246
+ ads: dict[str, dict[str, Any]] = {}
2247
+ checked_ads: list[str] = []
2248
+ for item in eligible[:max(MAX_AD_CHECKS, sum(1 for a in eligible if a.named))]:
2249
+ ads[item.domain] = check_ads(client, inputs, discovery.pool[item.domain], focused=(item.profile or {}).get("focused") is not False)
2250
+ checked_ads.append(item.domain)
2251
+ for item in eligible:
2252
+ ads.setdefault(item.domain, {"ad_activity": "unknown", "ad_evidence": [f"not checked: over the ad-check cap of {MAX_AD_CHECKS}"],
2253
+ "ad_kinds": [], "ad_strength": 0, "google_budget_usd": None, "pixel_tags": [], "lookup_gaps": []})
2254
+ chosen, over = compose(eligible, ads, discovery.pool, cap)
2255
+ _write_candidates(runtime, inputs=inputs, cap=cap, files=files, discovery=discovery, to_read=to_read, required1=required1,
2256
+ required2=required2, seeds=seeds, missing={}, status="complete")
2257
+
2258
+ direct_count = sum(1 for a in eligible if a.group == 0)
2259
+ pool = discovery.pool
2260
+
2261
+ def profile_fields(domain: str) -> dict[str, Any]:
2262
+ profile = files.profiles.get(domain) or {}
2263
+ return {key: profile.get(key) for key in ("offer", "offer_url", "offer_class", "offer_match", "price_text", "price_usd",
2264
+ "duration", "buyer", "job", "operating", "marketplace", "evidence_urls")}
2265
+
2266
+ def provenance_detail(candidate: Candidate) -> str:
2267
+ primary = candidate.primary_provenance()
2268
+ if primary == "agent_research" and candidate.found_in:
2269
+ return f"agent research ({candidate.found_in})"
2270
+ if primary == "discovered":
2271
+ return "discovered (" + ", ".join(candidate.sources[:3]) + ")"
2272
+ return primary.replace("_", " ")
2273
+
2274
+ competitors: list[dict[str, Any]] = []
2275
+ for rank, item in enumerate(chosen, start=1):
2276
+ candidate = pool[item.domain]
2277
+ ad = ads[item.domain]
2278
+ profile = files.profiles.get(item.domain) or {}
2279
+ competitors.append({
2280
+ "type": "competitor",
2281
+ "rank": rank,
2282
+ "domain": item.domain,
2283
+ "name": item.decision["name"] or profile.get("name") or default_name(item.domain),
2284
+ "competitor_tier": item.tier,
2285
+ "reason": item.decision["reason"],
2286
+ "icp_match": profile.get("same_buyer"),
2287
+ "jtbd_match": profile.get("same_job"),
2288
+ "provenance": candidate.ordered_provenance(),
2289
+ "provenance_detail": provenance_detail(candidate),
2290
+ "found_in": candidate.found_in,
2291
+ **profile_fields(item.domain),
2292
+ "price_band": item.price_band,
2293
+ "comparable": item.comparable,
2294
+ "ad_activity": ad["ad_activity"],
2295
+ "ad_evidence": ad["ad_evidence"],
2296
+ "selection_group": GROUP_LABELS[item.group],
2297
+ "selection": _selection_text(item, ad, cap, direct_count),
2298
+ "override_reason": item.decision.get("override_reason"),
2299
+ "reviewed": item.domain in files.review["checked"],
2300
+ "sources": list(candidate.sources),
2301
+ "source_count": len(candidate.sources),
2302
+ "score": candidate.score(),
2303
+ "decided_by": "agent",
2304
+ "tier": "B" if (candidate.sources and len(candidate.sources) >= 2) or ad["ad_activity"] == "ads_seen" else "C",
2305
+ })
2306
+
2307
+ profiled_out: list[dict[str, Any]] = []
2308
+ excluded: list[dict[str, Any]] = []
2309
+ for item in sorted(over, key=lambda a: _order_key(a, ads, pool)):
2310
+ ad = ads[item.domain]
2311
+ excluded.append({
2312
+ "type": "excluded_candidate", "domain": item.domain, "competitor_tier": item.tier, "reason_code": "over_cap",
2313
+ "reason": f"over the cap of {cap}: {GROUP_LABELS[item.group]}, {AD_LABELS[ad['ad_activity']]}",
2314
+ "decided_by": "cap", "provenance": pool[item.domain].ordered_provenance(), "profiled": item.domain in files.profiles,
2315
+ **profile_fields(item.domain), "price_band": item.price_band, "ad_activity": ad["ad_activity"], "ad_evidence": ad["ad_evidence"],
2316
+ "override_reason": item.decision.get("override_reason"), "listed_for": [], "score": pool[item.domain].score(), "tier": "C",
2317
+ })
2318
+ for domain in sorted(assessments, key=lambda d: (assessments[d].reason_code, -pool[d].score(), d)):
2319
+ item = assessments[domain]
2320
+ if item.group is not None:
2321
+ continue
2322
+ excluded.append({
2323
+ "type": "excluded_candidate", "domain": domain, "competitor_tier": item.tier, "reason_code": item.reason_code,
2324
+ "reason": _not_eligible_reason(item, inputs.price_floor_usd), "decided_by": "agent" if item.tier == "excluded" else "rule",
2325
+ "provenance": pool[domain].ordered_provenance(), "profiled": domain in files.profiles, **profile_fields(domain),
2326
+ "price_band": item.price_band, "override_reason": item.decision.get("override_reason"),
2327
+ "listed_for": list(pool[domain].listed_for[:8]), "score": pool[domain].score(), "tier": "C",
2328
+ })
2329
+ profiled_out = [row for row in excluded if row.get("profiled")]
2330
+ decided = set(assessments)
2331
+ required_all = set(required1) | set(required2)
2332
+ for candidate in sorted(pool.values(), key=lambda c: (-c.score(), c.domain)):
2333
+ if candidate.domain in decided:
2334
+ continue
2335
+ if candidate.domain in required_all:
2336
+ continue # unreachable: completeness requires a decision
2337
+ if candidate.round == 2 and not candidate.round1_signal():
2338
+ code, why = "round2_single_hit", "one relationship hit in round 2; not profiled"
2339
+ elif candidate.offer_signal():
2340
+ code, why = "over_profile_cap", f"over the profiling cap of {MAX_PROFILES_ROUND1}; not profiled"
2341
+ else:
2342
+ code, why = "weak_signal", "a single weak discovery hit; not profiled"
2343
+ excluded.append({
2344
+ "type": "excluded_candidate", "domain": candidate.domain, "reason_code": code, "reason": why, "decided_by": "profiling_list",
2345
+ "provenance": candidate.ordered_provenance(), "profiled": False, "listed_for": list(candidate.listed_for[:8]),
2346
+ "score": candidate.score(), "tier": "C",
2347
+ })
2348
+ for domain, queries in sorted(discovery.directories.items()):
2349
+ excluded.append({"type": "excluded_candidate", "domain": domain, "reason_code": "directory", "reason": "directory",
2350
+ "decided_by": "noise_filter", "profiled": False, "listed_for": sorted(set(queries))[:8], "score": None, "tier": "C"})
2351
+
2352
+ # Every pool domain already has a competitor or exclusion row; the
2353
+ # candidate rows (discovery facts) keep the strongest MAX_CANDIDATES.
2354
+ candidate_rows = [c.row() for c in sorted(pool.values(), key=lambda c: (-c.score(), c.domain))][:MAX_CANDIDATES]
2355
+ totals = client.ledger_totals()
2356
+ retrieved_on = totals["first_retrieved_on"] if totals["first_retrieved_on"] == totals["last_retrieved_on"] else f"{totals['first_retrieved_on']}..{totals['last_retrieved_on']}"
2357
+ listicle_pages = [files.listicles[url] for url in to_read if url in files.listicles]
2358
+ ad_counts = {key: sum(1 for ad in ads.values() if ad["ad_activity"] == key) for key in AD_ORDER}
2359
+ coverage = {
2360
+ "listicles_read": sum(1 for p in listicle_pages if p["status"] == "read"),
2361
+ "listicles_unreadable": sum(1 for p in listicle_pages if p["status"] == "unreadable"),
2362
+ "listicle_entries": sum(len(p["entries"]) for p in listicle_pages),
2363
+ "profiles": sum(1 for d in files.profiles if d in pool),
2364
+ "agent_additions": len(additions),
2365
+ "round2_seeds": seeds,
2366
+ "review_rounds": len(files.review["rounds"]),
2367
+ "objections_accepted": sum(1 for o in files.review["objections"] if o["resolution"] == "accepted"),
2368
+ "objections_rejected": sum(1 for o in files.review["objections"] if o["resolution"] == "rejected"),
2369
+ "missing_added": sum(1 for m in files.review["missing"] if m["resolution"] == "added"),
2370
+ "missing_rejected": sum(1 for m in files.review["missing"] if m["resolution"] == "rejected"),
2371
+ "ad_checks": len(checked_ads),
2372
+ "ad_counts": ad_counts,
2373
+ }
2374
+ evidence = {
2375
+ "schema_version": SCHEMA_VERSION,
2376
+ "section_id": SECTION_ID,
2377
+ "generated_by": Path(__file__).name,
2378
+ "retrieved_on": retrieved_on,
2379
+ "inputs": _inputs_block(inputs, cap),
2380
+ "composition_rule": {
2381
+ "roles": "caller-supplied roles with sourced profile and reason",
2382
+ "price": "recorded comparison only; no automatic floor",
2383
+ "order": "stable domain inspection order, not business priority",
2384
+ "operator_named": "explicitly named scope retained within the requested sample",
2385
+ "max_competitors": cap,
2386
+ },
2387
+ "enrichment": {key: value for key, value in coverage.items() if key != "ad_counts"} | {"ad_counts": ad_counts},
2388
+ "rows": discovery.summaries + candidate_rows + competitors + excluded,
2389
+ "totals": {key: totals[key] for key in ("dataforseo_calls", "search_calls", "spyfu_calls", "meta_searches", "page_fetches", "provider_calls", "rejected_calls")}
2390
+ | {"answered_queries": sum(1 for s in discovery.summaries if not s.get("unavailable"))},
2391
+ }
2392
+ section_markdown = render_section(
2393
+ inputs=inputs, cap=cap, competitors=competitors, profiled_out=profiled_out, excluded=excluded, summaries=discovery.summaries,
2394
+ pool_size=len(pool), coverage=coverage, totals=totals, retrieved_on=retrieved_on,
2395
+ )
2396
+
2397
+ evidence_path = workspace / EVIDENCE_REL
2398
+ evidence_json = json.dumps(evidence, indent=1, ensure_ascii=False)
2399
+ _write_atomic(evidence_path, evidence_json)
2400
+ section_file = runtime / "result.md"
2401
+ _write_atomic(section_file, section_markdown)
2402
+
2403
+ upsert = _load_script("upsert-markdown-section.py", "pmr_upsert_markdown_section")
2404
+ report_path = workspace / REPORT_REL
2405
+ report_path.parent.mkdir(parents=True, exist_ok=True)
2406
+ upsert.upsert_markdown_section(report_path, SECTION_HEADING, section_markdown)
2407
+
2408
+ persist = _load_script("persist-research-report.py", "pmr_persist_research_report")
2409
+ persistence = persist.persist_research_report(
2410
+ workspace,
2411
+ commit_message=f"research: update {SECTION_HEADING.lower()} section",
2412
+ extra_paths=(Path(_artifact_path("egbe-docs/research/market/evidence/competitors.json")),),
2413
+ )
2414
+
2415
+ legacy_source = {"operator_named": "explicit"}
2416
+ state = {
2417
+ "schema_version": SCHEMA_VERSION,
2418
+ "section_id": SECTION_ID,
2419
+ "status": "completed",
2420
+ "domains": [row["domain"] for row in competitors],
2421
+ "domain_sources": {row["domain"]: legacy_source.get(row["provenance"][0], row["provenance"][0]) if row["provenance"] else "discovered" for row in competitors},
2422
+ "explicit_domains": inputs.named_domains,
2423
+ "page_domains": inputs.page_domains,
2424
+ "product_domain": inputs.product_domain,
2425
+ "category_phrases": inputs.category_phrases,
2426
+ "queries": inputs.category_queries + inputs.brand_queries,
2427
+ "price_floor_usd": inputs.price_floor_usd,
2428
+ "round2_seeds": seeds,
2429
+ "evidence_path": EVIDENCE_REL.as_posix(),
2430
+ "evidence_sha256": hashlib.sha256(evidence_json.encode("utf-8")).hexdigest(),
2431
+ "result_path": (RUNTIME_REL / "result.md").as_posix(),
2432
+ "result_sha256": hashlib.sha256(section_markdown.encode("utf-8")).hexdigest(),
2433
+ "intake_sha256": inputs.intake_sha256,
2434
+ "intake_path": inputs.intake_path,
2435
+ "inputs_path": (RUNTIME_REL / "inputs.json").as_posix(),
2436
+ "inputs_sha256": _sha(runtime / "inputs.json"),
2437
+ "decisions_sha256": _sha(files.paths["decisions"]),
2438
+ "profiles_sha256": _sha(files.paths["profiles"]),
2439
+ "listicles_sha256": _sha(files.paths["listicles"]),
2440
+ "review_sha256": _sha(files.paths["review"]),
2441
+ "provider_calls": totals["provider_calls"],
2442
+ "dataforseo_calls": totals["dataforseo_calls"],
2443
+ "search_calls": totals["search_calls"],
2444
+ "spyfu_calls": totals["spyfu_calls"],
2445
+ "meta_searches": totals["meta_searches"],
2446
+ "page_fetches": totals["page_fetches"],
2447
+ "rejected_calls": totals["rejected_calls"],
2448
+ "answered_queries": evidence["totals"]["answered_queries"],
2449
+ "provider_starts_recorded": totals["provider_starts_recorded"],
2450
+ "attempt_dataforseo_tasks": client.dataforseo_tasks,
2451
+ "attempt_search_calls": client.search_calls,
2452
+ "attempt_spyfu_calls": client.spyfu_calls,
2453
+ "attempt_meta_searches": client.meta_searches,
2454
+ "attempt_cache_hits": client.cache_hits,
2455
+ "report_commit_sha": persistence.commit_sha,
2456
+ "publication_mode": persistence.publication_mode,
2457
+ "research_context": _load_script("research-artifact-io.py", "pmr_input_io").context_binding(workspace, SECTION_ID),
2458
+ "error": None,
2459
+ }
2460
+ _write_atomic(runtime / "request.json", json.dumps(state, indent=1))
2461
+ return state
2462
+
2463
+
2464
+ def default_name(domain: str) -> str:
2465
+ stem = registrable_domain(domain).split(".")[0]
2466
+ return stem[:1].upper() + stem[1:] if stem else domain
2467
+
2468
+
2469
+ def main() -> int:
2470
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
2471
+ parser.add_argument("--workspace", type=Path, required=True)
2472
+ parser.add_argument("--context-file", default=None, help="Optional explicitly selected provider-safe research context.")
2473
+ parser.add_argument("--intake", default="", help=f"Optional typed operation inputs (default: {INTAKE_REL.as_posix()}).")
2474
+ parser.add_argument("--domains", default="", help="Competitor domains named by the founder; overrides the intake's list.")
2475
+ parser.add_argument("--page-domains", default="", help="Competitors the product page mentions; ordinary candidates. Overrides the intake's list.")
2476
+ parser.add_argument("--product-domain", default="", help="The product's own domain; never a candidate and never a search seed.")
2477
+ parser.add_argument("--seed-keywords", default="", help="1-3 category phrases of the offer; override the intake's offer card.")
2478
+ parser.add_argument("--price-floor-usd", type=float, default=None, help="Comparable price floor; overrides the intake's offer card.")
2479
+ parser.add_argument("--decisions", default="", help=f"Path to the tiering decisions file (default: {RUNTIME_REL.as_posix()}/{DECISIONS_FILENAME}).")
2480
+ parser.add_argument("--max-competitors", type=int, default=MAX_COMPETITORS)
2481
+ parser.add_argument("--location-code", type=int, default=None, help="DataForSEO location (default: the intake's first market, else US).")
2482
+ parser.add_argument("--language-code", default="", help="Provider query language (default: en); retain the saved language on resume.")
2483
+ parser.add_argument("--country", default="", help="Primary market as a two-letter code (default: the intake's first market, else US).")
2484
+ parser.add_argument("--eu-country", default="", help="EU country for the official Meta Ad Library check (default: the first EU market, else DE).")
2485
+ parser.add_argument("--max-dataforseo-tasks", type=int, default=MAX_DATAFORSEO_TASKS_PER_ATTEMPT)
2486
+ parser.add_argument("--max-search-calls", type=int, default=MAX_SEARCH_CALLS_PER_ATTEMPT)
2487
+ parser.add_argument("--max-spyfu-calls", type=int, default=MAX_SPYFU_CALLS_PER_ATTEMPT, help="SpyFu lookups per attempt (0 disables the source).")
2488
+ parser.add_argument("--max-meta-searches", type=int, default=MAX_META_SEARCHES_PER_ATTEMPT)
2489
+ parser.add_argument("--max-fetches", type=int, default=MAX_FETCHES_PER_ATTEMPT)
2490
+ args = parser.parse_args()
2491
+
2492
+ try:
2493
+ state = run(args.workspace, args)
2494
+ except (CompetitorSetError, ValueError) as exc:
2495
+ print(f"{DISPLAY_NAME}: {exc}", file=sys.stderr)
2496
+ return 1
2497
+ if state["status"] == "pending":
2498
+ print(
2499
+ f"{DISPLAY_NAME}: {state['reason']} — read {state['candidates_path']} (`missing`, `profile_required`, "
2500
+ f"`listicles_to_read`), write the enrichment files next to it (see the workflow doc), then rerun the same "
2501
+ "command; cached discovery adds no new spend.",
2502
+ file=sys.stderr,
2503
+ )
2504
+ return 2
2505
+ print(
2506
+ f"{DISPLAY_NAME}: completed — {len(state['domains'])} competitor(s), {state['dataforseo_calls']} DataForSEO "
2507
+ f"task(s), {state['spyfu_calls']} SpyFu lookup(s), {state['search_calls']} web search(es), evidence at "
2508
+ f"{state['evidence_path']}, report commit {(state['report_commit_sha'] or '')[:12] or 'n/a'}"
2509
+ )
2510
+ return 0
2511
+
2512
+
2513
+ if __name__ == "__main__":
2514
+ raise SystemExit(main())