@egbe-ai/opencode 0.0.0-stage → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (479) hide show
  1. package/LICENSE +4 -0
  2. package/README.md +124 -2
  3. package/index.js +5 -0
  4. package/package.json +55 -4
  5. package/plugin.js +230 -0
  6. package/session-log.js +315 -0
  7. package/skills/.egbe-catalog/manifest.json +1 -0
  8. package/skills/egbe-os-analytics/SKILL.md +499 -0
  9. package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
  10. package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
  11. package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
  12. package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
  13. package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
  14. package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
  15. package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
  16. package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
  17. package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
  18. package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
  19. package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
  20. package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
  21. package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
  22. package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
  23. package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
  24. package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
  25. package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
  26. package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
  27. package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
  28. package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
  29. package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
  30. package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
  31. package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
  32. package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
  33. package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
  34. package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
  35. package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
  36. package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
  37. package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
  38. package/skills/egbe-os-db/SKILL.md +174 -0
  39. package/skills/egbe-os-deploy/SKILL.md +412 -0
  40. package/skills/egbe-os-design/SKILL.md +182 -0
  41. package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
  42. package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
  43. package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
  44. package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
  45. package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
  46. package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
  47. package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
  48. package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
  49. package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
  50. package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
  51. package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
  52. package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
  53. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
  54. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
  55. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
  56. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
  57. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
  58. package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
  59. package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
  60. package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
  61. package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
  62. package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
  63. package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
  64. package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
  65. package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
  66. package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
  67. package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
  68. package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
  69. package/skills/egbe-os-destination-readiness/references/report.md +116 -0
  70. package/skills/egbe-os-domains/SKILL.md +189 -0
  71. package/skills/egbe-os-errors/SKILL.md +259 -0
  72. package/skills/egbe-os-git/SKILL.md +275 -0
  73. package/skills/egbe-os-git/references/issues.md +33 -0
  74. package/skills/egbe-os-google-ads/SKILL.md +163 -0
  75. package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
  76. package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
  77. package/skills/egbe-os-google-ads/references/conversions.md +153 -0
  78. package/skills/egbe-os-google-ads/references/delivery.md +128 -0
  79. package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
  80. package/skills/egbe-os-google-ads/references/reporting.md +105 -0
  81. package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
  82. package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
  83. package/skills/egbe-os-google-auth/SKILL.md +285 -0
  84. package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
  85. package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
  86. package/skills/egbe-os-images/SKILL.md +163 -0
  87. package/skills/egbe-os-images/references/stock-images.md +21 -0
  88. package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
  89. package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
  90. package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
  91. package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
  92. package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
  93. package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
  94. package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
  95. package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
  96. package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
  97. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
  98. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
  99. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
  100. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
  101. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
  102. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
  103. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
  104. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
  105. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
  106. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
  107. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
  108. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
  109. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
  110. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
  111. package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
  112. package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
  113. package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
  114. package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
  115. package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
  116. package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
  117. package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
  118. package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
  119. package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
  120. package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
  121. package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
  122. package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
  123. package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
  124. package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
  125. package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
  126. package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
  127. package/skills/egbe-os-llm/SKILL.md +160 -0
  128. package/skills/egbe-os-mail/SKILL.md +180 -0
  129. package/skills/egbe-os-me/SKILL.md +119 -0
  130. package/skills/egbe-os-meta-ads/SKILL.md +114 -0
  131. package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
  132. package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
  133. package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
  134. package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
  135. package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
  136. package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
  137. package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
  138. package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
  139. package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
  140. package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
  141. package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
  142. package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
  143. package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
  144. package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
  145. package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
  146. package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
  147. package/skills/egbe-os-notifications/SKILL.md +78 -0
  148. package/skills/egbe-os-openai-ads/SKILL.md +149 -0
  149. package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
  150. package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
  151. package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
  152. package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
  153. package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
  154. package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
  155. package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
  156. package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
  157. package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
  158. package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
  159. package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
  160. package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
  161. package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
  162. package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
  163. package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
  164. package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
  165. package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
  166. package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
  167. package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
  168. package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
  169. package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
  170. package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
  171. package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
  172. package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
  173. package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
  174. package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
  175. package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
  176. package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
  177. package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
  178. package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
  179. package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
  180. package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
  181. package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
  182. package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
  183. package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
  184. package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
  185. package/skills/egbe-os-payments/SKILL.md +215 -0
  186. package/skills/egbe-os-payments/agents/openai.yaml +4 -0
  187. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
  188. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
  189. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
  190. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
  191. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
  192. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
  193. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
  194. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
  195. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
  196. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
  197. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
  198. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
  199. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
  200. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
  201. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
  202. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
  203. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
  204. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
  205. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
  206. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
  207. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
  208. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
  209. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
  210. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
  211. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
  212. package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
  213. package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
  214. package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
  215. package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
  216. package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
  217. package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
  218. package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
  219. package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
  220. package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
  221. package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
  222. package/skills/egbe-os-playbook/SKILL.md +79 -0
  223. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
  224. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
  225. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
  226. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
  227. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
  228. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
  229. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
  230. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
  231. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
  232. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
  233. package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
  234. package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
  235. package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
  236. package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
  237. package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
  238. package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
  239. package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
  240. package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
  241. package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
  242. package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
  243. package/skills/egbe-os-playbook/playbooks/product.md +67 -0
  244. package/skills/egbe-os-playbook/references/catalog.md +127 -0
  245. package/skills/egbe-os-playbook/references/changes.md +167 -0
  246. package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
  247. package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
  248. package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
  249. package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
  250. package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
  251. package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
  252. package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
  253. package/skills/egbe-os-product-auth/SKILL.md +151 -0
  254. package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
  255. package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
  256. package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
  257. package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
  258. package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
  259. package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
  260. package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
  261. package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
  262. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
  263. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
  264. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
  265. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
  266. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
  267. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
  268. package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
  269. package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
  270. package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
  271. package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
  272. package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
  273. package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
  274. package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
  275. package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
  276. package/skills/egbe-os-product-build/SKILL.md +111 -0
  277. package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
  278. package/skills/egbe-os-product-build/references/capabilities.md +25 -0
  279. package/skills/egbe-os-product-build/references/engineering.md +72 -0
  280. package/skills/egbe-os-product-build/references/execution.md +55 -0
  281. package/skills/egbe-os-product-build/references/foundations.md +93 -0
  282. package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
  283. package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
  284. package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
  285. package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
  286. package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
  287. package/skills/egbe-os-product-market-research/SKILL.md +159 -0
  288. package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
  289. package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
  290. package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
  291. package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
  292. package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
  293. package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
  294. package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
  295. package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
  296. package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
  297. package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
  298. package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
  299. package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
  300. package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
  301. package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
  302. package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
  303. package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
  304. package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
  305. package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
  306. package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
  307. package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
  308. package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
  309. package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
  310. package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
  311. package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
  312. package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
  313. package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
  314. package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
  315. package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
  316. package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
  317. package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
  318. package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
  319. package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
  320. package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
  321. package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
  322. package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
  323. package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
  324. package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
  325. package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
  326. package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
  327. package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
  328. package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
  329. package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
  330. package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
  331. package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
  332. package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
  333. package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
  334. package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
  335. package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
  336. package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
  337. package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
  338. package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
  339. package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
  340. package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
  341. package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
  342. package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
  343. package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
  344. package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
  345. package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
  346. package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
  347. package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
  348. package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
  349. package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
  350. package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
  351. package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
  352. package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
  353. package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
  354. package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
  355. package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
  356. package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
  357. package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
  358. package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
  359. package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
  360. package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
  361. package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
  362. package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
  363. package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
  364. package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
  365. package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
  366. package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
  367. package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
  368. package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
  369. package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
  370. package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
  371. package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
  372. package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
  373. package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
  374. package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
  375. package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
  376. package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
  377. package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
  378. package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
  379. package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
  380. package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
  381. package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
  382. package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
  383. package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
  384. package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
  385. package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
  386. package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
  387. package/skills/egbe-os-product-report/SKILL.md +150 -0
  388. package/skills/egbe-os-product-report/references/artifacts.md +84 -0
  389. package/skills/egbe-os-product-report/references/brief.md +34 -0
  390. package/skills/egbe-os-product-report/references/daily.md +76 -0
  391. package/skills/egbe-os-product-report/references/sources.md +68 -0
  392. package/skills/egbe-os-product-routines/SKILL.md +166 -0
  393. package/skills/egbe-os-product-routines/references/catalog.md +22 -0
  394. package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
  395. package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
  396. package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
  397. package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
  398. package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
  399. package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
  400. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
  401. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
  402. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
  403. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
  404. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
  405. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
  406. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
  407. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
  408. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
  409. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
  410. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
  411. package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
  412. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
  413. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
  414. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
  415. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
  416. package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
  417. package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
  418. package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
  419. package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
  420. package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
  421. package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
  422. package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
  423. package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
  424. package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
  425. package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
  426. package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
  427. package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
  428. package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
  429. package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
  430. package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
  431. package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
  432. package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
  433. package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
  434. package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
  435. package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
  436. package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
  437. package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
  438. package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
  439. package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
  440. package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
  441. package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
  442. package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
  443. package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
  444. package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
  445. package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
  446. package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
  447. package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
  448. package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
  449. package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
  450. package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
  451. package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
  452. package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
  453. package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
  454. package/skills/egbe-os-report-ui/SKILL.md +133 -0
  455. package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
  456. package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
  457. package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
  458. package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
  459. package/skills/egbe-os-report-ui/references/components.md +209 -0
  460. package/skills/egbe-os-research/SKILL.md +125 -0
  461. package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
  462. package/skills/egbe-os-scrape/SKILL.md +160 -0
  463. package/skills/egbe-os-search/SKILL.md +182 -0
  464. package/skills/egbe-os-search/references/google-places.md +81 -0
  465. package/skills/egbe-os-support/SKILL.md +149 -0
  466. package/skills/egbe-os-telemetry/SKILL.md +344 -0
  467. package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
  468. package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
  469. package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
  470. package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
  471. package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
  472. package/skills/egbe-os-tools/SKILL.md +180 -0
  473. package/skills/egbe-os-tools/references/theona.md +26 -0
  474. package/skills/egbe-os-wallet/SKILL.md +181 -0
  475. package/skills/egbe-setup/EGBE.md +484 -0
  476. package/skills/egbe-setup/SKILL.md +116 -0
  477. package/skills/egbe-setup/references/project.md +80 -0
  478. package/state.js +37 -0
  479. package/update.js +361 -0
@@ -0,0 +1,2258 @@
1
+ #!/usr/bin/env python3
2
+ """Run the deterministic Channel Footprint section: where competitors buy traffic.
3
+
4
+ Ad libraries say whether a competitor advertises on a platform; nothing in the
5
+ other measured sections says WHERE its traffic comes from or which platforms
6
+ it pays. This runner answers that per competitor from five kinds of
7
+ observation, each cheap and reproducible:
8
+
9
+ - tracking tags on the competitor's own pages (home page plus its Google
10
+ Tag Manager containers): ad-platform pixels, native-ad networks, review
11
+ marketplace trackers and affiliate platforms reveal where conversions are
12
+ bought, including platforms without any public ad library;
13
+ - ad-library presence and age on Google, Meta (US page search plus the
14
+ official EU delivery data), LinkedIn, TikTok (EU library) and Reddit,
15
+ through OS scrape actors and the official Meta endpoint;
16
+ - the public traffic-source split and monthly visit estimate (a Similarweb
17
+ actor, or the SimilarWeb tools provider once it is configured);
18
+ - partner signals from web search: affiliate program pages, coupon and
19
+ promo-code listings, sponsorship mentions;
20
+ - page history from the Wayback Machine: domain age and how often the home
21
+ page changed over the last two years.
22
+
23
+ The staged runner preserves a normalized ad corpus, validates context-bound
24
+ agent relevance decisions, and reports scoped activity, duration, coverage and
25
+ comparison roles without selecting a business priority. No LLM runs
26
+ inside the runner. Cache replays add no spend; durable cumulative provider
27
+ starts and per-attempt budgets bound collection and sample expansion.
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import sys as _artifact_sys
33
+ from pathlib import Path as _ArtifactPath
34
+ _artifact_sys.path.insert(0, str(_ArtifactPath(__file__).resolve().parent))
35
+ from egbe_artifacts import artifact_path as _artifact_path
36
+
37
+ import runpy as _egbe_runpy
38
+ from pathlib import Path as _EgbePath
39
+ _egbe_adapter = _EgbePath(__file__).with_name("connection_identity.py")
40
+ _connection_headers = _egbe_runpy.run_path(str(_egbe_adapter))["connection_headers"] if _egbe_adapter.is_file() else lambda _url: {}
41
+
42
+ import argparse
43
+ import base64
44
+ import hashlib
45
+ import html as html_lib
46
+ import http.client
47
+ import importlib.util
48
+ import json
49
+ import os
50
+ import re
51
+ import sys
52
+ import time
53
+ import urllib.error
54
+ import urllib.parse
55
+ import urllib.request
56
+ from dataclasses import dataclass, field
57
+ from datetime import date, datetime, timedelta, timezone
58
+ from pathlib import Path
59
+ from types import ModuleType
60
+ from typing import Any, Callable
61
+
62
+ SCHEMA_VERSION = 2
63
+ SECTION_ID = "channel_footprint"
64
+ SECTION_HEADING = "Channel footprint"
65
+ DISPLAY_NAME = "Channel Footprint"
66
+
67
+ DEFAULT_OS_URL = "https://os.hz.egbe.dev"
68
+ RUNTIME_REL = Path(_artifact_path("egbe-docs/research/market/work")) / SECTION_ID
69
+ EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/channel-footprint.json"))
70
+ REPORT_REL = Path(_artifact_path("egbe-docs/research/market/full-report.md"))
71
+ SOURCES_CONFIG_REL = Path("references/config/channel-footprint-sources.v1.json")
72
+ WAYBACK_CDX_URL = "https://web.archive.org/cdx/search/cdx"
73
+
74
+ MAX_COMPETITORS = 10
75
+ MAX_EXPLICIT_DOMAINS = 10
76
+ MAX_ACTOR_RUNS_PER_ATTEMPT = 12
77
+ MAX_META_SEARCHES_PER_ATTEMPT = 10
78
+ MAX_SEARCH_CALLS_PER_ATTEMPT = 30
79
+ MAX_FETCHES_PER_ATTEMPT = 40
80
+ MAX_CDX_CALLS_PER_ATTEMPT = 25
81
+ # One SpyFu domain-stats lookup per competitor (12 months = 12 rows, ~$0.006).
82
+ MAX_SPYFU_CALLS_PER_ATTEMPT = 12
83
+ SPYFU_MONTHS = 12
84
+ FETCH_MAX_BYTES = 2_000_000
85
+ FETCH_TIMEOUT_SECONDS = 25.0
86
+ CDX_TIMEOUT_SECONDS = 60.0
87
+ CDX_PAUSE_SECONDS = 1.0
88
+ CDX_ROW_LIMIT = 500
89
+ GTM_CONTAINERS_PER_SITE = 3
90
+ HISTORY_MONTHS = 24
91
+ BRAVE_COUNT = 10
92
+ PARTNER_URLS_PER_KIND = 3
93
+ ACTOR_RETRY_DELAY_SECONDS = 20.0
94
+ # The OS host refuses connections for about 40 s while a gateway on it
95
+ # redeploys (observed live 2026-10-02); one retry after this delay covers it.
96
+ OS_UNREACHABLE_RETRY_DELAY_SECONDS = 45.0
97
+ FREE_RETRY_DELAY_SECONDS = 5.0
98
+ DEFAULT_COUNTRY = "US"
99
+ DEFAULT_EU_COUNTRY = "DE"
100
+ # Product relevance by the creative classifier's semantic verdict: small
101
+ # batches keep each answer bounded; a decided sample may expand once more
102
+ # per round, so a few rounds settle a collect.
103
+ MAX_IMAGE_FETCHES_PER_ATTEMPT = 500
104
+ MAX_IMAGE_BYTES = 4 * 1024 * 1024
105
+ IMAGE_TIMEOUT_SECONDS = 30.0
106
+ RELEVANCE_PATH = "/v1/media/classify-creative"
107
+ RELEVANCE_BATCH = 4
108
+ MAX_RELEVANCE_CALLS_PER_ATTEMPT = 40
109
+ RELEVANCE_TIMEOUT_SECONDS = 330.0
110
+ MODEL_RELEVANCE_ROUNDS = 3
111
+ PRESENT_SHARE_PCT = 0.0 # Any positive modeled share is an observation, not a success threshold.
112
+
113
+ ALL_SOURCES = ("pixels", "history", "traffic", "google", "spyfu", "meta", "linkedin", "partners", "tiktok", "reddit")
114
+ CHANNELS = (
115
+ "google_ads", "youtube_ads", "meta_ads", "linkedin_ads", "tiktok_ads", "reddit_ads", "microsoft_ads", "x_ads",
116
+ "pinterest_ads", "snap_ads", "quora_ads", "display_native", "review_marketplace", "affiliate", "paid_search", "paid_social",
117
+ )
118
+ CHANNEL_LABELS = {
119
+ "google_ads": "Google Ads", "youtube_ads": "YouTube (video)", "meta_ads": "Meta", "linkedin_ads": "LinkedIn",
120
+ "tiktok_ads": "TikTok", "reddit_ads": "Reddit", "microsoft_ads": "Microsoft Ads", "x_ads": "X",
121
+ "pinterest_ads": "Pinterest", "snap_ads": "Snapchat", "quora_ads": "Quora", "display_native": "Display / native",
122
+ "review_marketplace": "Review marketplaces", "affiliate": "Affiliate / partners", "paid_search": "Paid search (category)", "paid_social": "Paid social (category)",
123
+ }
124
+ TAG_MANAGER_SIGNATURES: tuple[tuple[str, str], ...] = (
125
+ ("google_tag_manager", r"googletagmanager\.com/gtm\.js|\bGTM-[A-Z0-9]{4,10}\b"),
126
+ ("segment", r"cdn\.segment\.com/analytics\.js|\banalytics\.load\s*\("),
127
+ ("tealium", r"tags\.tiqcdn\.com"),
128
+ ("adobe_launch", r"assets\.adobedtm\.com"),
129
+ )
130
+ LINKEDIN_COMPANY_RE = re.compile(r"linkedin\.com/(?:company|school|showcase)/([A-Za-z0-9._%-]+)", re.IGNORECASE)
131
+ AFFILIATE_NETWORK_DOMAINS = {
132
+ "impact.com", "partnerstack.com", "shareasale.com", "cj.com", "awin.com", "rakutenadvertising.com",
133
+ "rewardful.com", "firstpromoter.com", "tapfiliate.com", "refersion.com", "getrewardful.com",
134
+ "affiliateprogramdb.com", "lasso.net", "affpaying.com", "offervault.com",
135
+ }
136
+ COUPON_DOMAINS = {
137
+ "retailmenot.com", "joinhoney.com", "coupons.com", "dealspotr.com", "couponfollow.com", "slickdeals.net",
138
+ "wethrift.com", "knoji.com", "couponbirds.com", "coupert.com", "simplycodes.com", "capitaloneshopping.com",
139
+ "rakuten.com", "groupon.com", "couponcabin.com", "offers.com", "promocodes.com", "hotdeals.com",
140
+ }
141
+ TRAFFIC_CHANNEL_MAP = {
142
+ "paid_search": "paid_search", "search_paid": "paid_search", "paid_social": "paid_social", "social_paid": "paid_social",
143
+ "display": "display_native", "display_ads": "display_native", "displayads": "display_native",
144
+ "paid_referrals": "display_native", "affiliates": "affiliate", "affiliate": "affiliate",
145
+ }
146
+
147
+ # (tag id, channel, regex over page HTML and tag-manager container JS)
148
+ TAG_SIGNATURES: tuple[tuple[str, str, str], ...] = (
149
+ ("meta_pixel", "meta_ads", r"connect\.facebook\.net/[^\"']*/fbevents\.js|\bfbq\s*\("),
150
+ ("google_ads_tag", "google_ads", r"googleads\.g\.doubleclick\.net|googleadservices\.com/pagead/conversion|\bAW-\d{6,}"),
151
+ ("reddit_pixel", "reddit_ads", r"redditstatic\.com/ads/pixel\.js|\brdt\s*\(\s*['\"]init"),
152
+ ("linkedin_insight", "linkedin_ads", r"snap\.licdn\.com/li\.lms-analytics/insight\.min\.js|_linkedin_partner_id"),
153
+ ("tiktok_pixel", "tiktok_ads", r"analytics\.tiktok\.com/i18n/pixel/events\.js|\bttq\.load\s*\("),
154
+ ("microsoft_uet", "microsoft_ads", r"bat\.bing\.com/bat\.js|\buetq\b"),
155
+ ("x_pixel", "x_ads", r"static\.ads-twitter\.com/uwt\.js|\btwq\s*\(\s*['\"](?:config|init)"),
156
+ ("pinterest_tag", "pinterest_ads", r"s\.pinimg\.com/ct/core\.js|\bpintrk\s*\("),
157
+ ("snap_pixel", "snap_ads", r"sc-static\.net/scevent\.min\.js|\bsnaptr\s*\("),
158
+ ("quora_pixel", "quora_ads", r"a\.quora\.com/qevents\.js|\bqp\s*\(\s*['\"]init"),
159
+ ("taboola", "display_native", r"cdn\.taboola\.com/libtrc|\b_tfa\b"),
160
+ ("outbrain", "display_native", r"amplify\.outbrain\.com/cp/obtp\.js|\bobApi\s*\("),
161
+ ("criteo", "display_native", r"static\.criteo\.net/js/ld/ld\.js"),
162
+ ("capterra_tracker", "review_marketplace", r"ct\.capterra\.com/capterra_tracker"),
163
+ ("g2_tracker", "review_marketplace", r"tracking\.g2crowd\.com|g2crowd\.com/attribution"),
164
+ ("partnerstack", "affiliate", r"js\.partnerstack\.com|\bgrsm\.io\b|growsumo"),
165
+ ("impact", "affiliate", r"utt\.impactcdn\.com|impactradius"),
166
+ ("rewardful", "affiliate", r"r\.wdfl\.co|\brewardful\b"),
167
+ ("firstpromoter", "affiliate", r"cdn\.firstpromoter\.com|\bfpr\s*\(\s*['\"]init"),
168
+ ("tapfiliate", "affiliate", r"script\.tapfiliate\.com"),
169
+ ("refersion", "affiliate", r"refersion\.com/tracker"),
170
+ ("shareasale", "affiliate", r"shareasale\.com/sale\.cfm|static\.shareasale\.com/json/shareasale\.js"),
171
+ )
172
+ GTM_ID_RE = re.compile(r"\bGTM-[A-Z0-9]{4,10}\b")
173
+ PARTNER_KINDS: tuple[tuple[str, str], ...] = (
174
+ # (kind, query template); precision rules live in `_partner_hit`.
175
+ ("affiliate_program", '"{brand}" affiliate program'),
176
+ ("coupon_listing", '"{brand}" promo code coupon'),
177
+ ("sponsorship", '"sponsored by {brand}" OR "{brand} sponsored"'),
178
+ )
179
+ _DOMAIN_RE = re.compile(r"^[a-z0-9][a-z0-9.-]{0,251}\.[a-z]{2,24}$")
180
+
181
+
182
+ class ChannelFootprintError(RuntimeError):
183
+ """Terminal runner failure; runtime state is preserved for resume."""
184
+
185
+
186
+ class SourceUnavailable(ChannelFootprintError):
187
+ """A provider rejected the request itself (invalid input, unsupported
188
+ actor): a fact about the source, cached so reruns never re-spend."""
189
+
190
+
191
+ class BudgetExhausted(ChannelFootprintError):
192
+ """The per-attempt budget of one metered or free call type is spent;
193
+ always propagates so a rerun resumes from the cache."""
194
+
195
+
196
+ class OsUnreachable(ChannelFootprintError):
197
+ """The request never reached the OS gateway (connection refused, DNS,
198
+ TLS, a connect timeout), so nothing was started or billed. Retried once;
199
+ then it fails the attempt instead of becoming a coverage gap, because it
200
+ says nothing about the source and a rerun resumes from the cache."""
201
+
202
+
203
+ # Consecutive failures of one supplementary source within an attempt after
204
+ # which its remaining domains are skipped instead of started (a provider that
205
+ # is down must not burn the paid actor budget one domain at a time).
206
+ SOURCE_BREAKER_FAILURES = 2
207
+ # Hosts that are not a competitor's destination but the platform itself or a
208
+ # store/link hub: the advertiser name decides for those rows.
209
+ NEUTRAL_DESTINATION_HOSTS = {
210
+ # the platforms themselves and their messaging / short hosts
211
+ "facebook.com", "instagram.com", "fb.me", "fb.com", "fb.watch", "fb.gg", "m.me", "ig.me", "messenger.com",
212
+ "whatsapp.com", "wa.me", "t.me", "reddit.com", "redd.it", "linkedin.com", "lnkd.in", "tiktok.com", "vm.tiktok.com",
213
+ "youtu.be", "youtube.com", "t.co", "x.com", "twitter.com",
214
+ # stores and link hubs
215
+ "apps.apple.com", "itunes.apple.com", "play.google.com", "linktr.ee", "lnk.to", "lnk.bio",
216
+ # shorteners and attribution/deep-link hosts (the real destination is hidden)
217
+ "bit.ly", "tinyurl.com", "ow.ly", "buff.ly", "hubs.ly", "hubs.la", "amzn.to", "rebrand.ly", "cutt.ly",
218
+ "onelink.me", "app.link", "adjust.com", "adj.st", "go.link", "smart.link", "page.link",
219
+ }
220
+ # Click-tracking wrappers whose `u` query parameter carries the real address.
221
+ REDIRECT_WRAPPER_HOSTS = {"l.facebook.com", "lm.facebook.com", "l.instagram.com", "l.messenger.com", "google.com", "www.google.com"}
222
+
223
+
224
+ def _load_script(filename: str, module_name: str) -> ModuleType:
225
+ path = Path(__file__).resolve().parent / filename
226
+ spec = importlib.util.spec_from_file_location(module_name, path)
227
+ if spec is None or spec.loader is None:
228
+ raise ChannelFootprintError(f"cannot load bundled script {filename}")
229
+ module = importlib.util.module_from_spec(spec)
230
+ sys.modules[spec.name] = module
231
+ previous = sys.dont_write_bytecode
232
+ sys.dont_write_bytecode = True
233
+ try:
234
+ spec.loader.exec_module(module)
235
+ finally:
236
+ sys.dont_write_bytecode = previous
237
+ return module
238
+
239
+
240
+ # Cache entries are stored gzip-compressed; plain entries of older runs replay too.
241
+ IO = _load_script("research-artifact-io.py", "pmr_channel_footprint_io")
242
+ _KEYWORD_ECONOMICS: ModuleType | None = None
243
+
244
+
245
+ def _keyword_economics() -> ModuleType:
246
+ global _KEYWORD_ECONOMICS
247
+ if _KEYWORD_ECONOMICS is None:
248
+ _KEYWORD_ECONOMICS = _load_script("run-keyword-economics.py", "pmr_keyword_economics_for_channel_footprint")
249
+ return _KEYWORD_ECONOMICS
250
+
251
+
252
+ def registrable_domain(value: str) -> str:
253
+ return _keyword_economics().registrable_domain(value)
254
+
255
+
256
+ def brand_token(domain: str) -> str:
257
+ return _keyword_economics().brand_token(domain)
258
+
259
+
260
+ def load_sources_config(path: Path | None = None) -> dict[str, Any]:
261
+ config_path = path or (Path(__file__).resolve().parent.parent / SOURCES_CONFIG_REL)
262
+ try:
263
+ raw = json.loads(config_path.read_text(encoding="utf-8"))
264
+ except (OSError, json.JSONDecodeError) as exc:
265
+ raise ChannelFootprintError(f"cannot read the sources config {config_path.name}: {exc}") from exc
266
+ if raw.get("schema_version") != 1 or not isinstance(raw.get("actors"), dict):
267
+ raise ChannelFootprintError("sources config must have schema_version 1 and an actors object")
268
+ for key, spec in raw["actors"].items():
269
+ if not isinstance(spec, dict) or not str(spec.get("actor") or "").strip():
270
+ raise ChannelFootprintError(f"sources config actor `{key}` has no actor id")
271
+ registry = raw.get("channels") or {}
272
+ if not isinstance(registry, dict):
273
+ raise ChannelFootprintError("sources config channels must be an object of channel labels")
274
+ global CHANNELS
275
+ for channel, label in registry.items():
276
+ if not re.fullmatch(r"[a-z][a-z0-9_]+", channel) or not isinstance(label, str) or not label:
277
+ raise ChannelFootprintError("invalid channel registry entry")
278
+ CHANNEL_LABELS[channel] = label
279
+ CHANNELS = tuple(dict.fromkeys((*CHANNELS, *registry)))
280
+ return raw
281
+
282
+
283
+ # ── OS client: metered actors, Meta library, web search; free fetches ──────
284
+
285
+ @dataclass
286
+ class OsClient:
287
+ os_url: str
288
+ api_key: str
289
+ cache_dir: Path
290
+ max_actor_runs: int = MAX_ACTOR_RUNS_PER_ATTEMPT
291
+ max_meta_searches: int = MAX_META_SEARCHES_PER_ATTEMPT
292
+ max_search_calls: int = MAX_SEARCH_CALLS_PER_ATTEMPT
293
+ max_fetches: int = MAX_FETCHES_PER_ATTEMPT
294
+ max_cdx_calls: int = MAX_CDX_CALLS_PER_ATTEMPT
295
+ max_spyfu_calls: int = MAX_SPYFU_CALLS_PER_ATTEMPT
296
+ actor_runs: int = 0
297
+ meta_searches: int = 0
298
+ search_calls: int = 0
299
+ fetches: int = 0
300
+ cdx_calls: int = 0
301
+ spyfu_calls: int = 0
302
+ cache_hits: int = 0
303
+ cache_writes: int = 0
304
+ touched: set[str] = field(default_factory=set)
305
+ # Consecutive live non-budget failures per supplementary source in this attempt.
306
+ source_failures: dict[str, int] = field(default_factory=dict)
307
+ request: Callable[..., dict[str, Any]] | None = None
308
+ fetch: Callable[[str], tuple[int, str]] | None = None
309
+ max_image_fetches: int = MAX_IMAGE_FETCHES_PER_ATTEMPT
310
+ image_fetches: int = 0
311
+ max_relevance_calls: int = MAX_RELEVANCE_CALLS_PER_ATTEMPT
312
+ relevance_calls: int = 0
313
+ fetch_bytes: Callable[[str], tuple[int, bytes]] | None = None
314
+ # The cutoff this client collects for: spend is reported per cutoff, and
315
+ # answers reused from an earlier cutoff are counted apart.
316
+ cutoff: str = ""
317
+ sleep: Callable[[float], None] | None = None
318
+ # Prefix of the wire Idempotency-Key. Sections that send byte-identical
319
+ # payloads (Creative Patterns reuses these actor inputs, and at every new
320
+ # cutoff) set their own scope so the gateway never merges their starts.
321
+ key_scope: str = ""
322
+ max_total_paid_starts: int | None = None
323
+ cumulative_budget_path: Path | None = None
324
+
325
+ def _http(self, path: str, payload: dict[str, Any], timeout: float, *, idempotency_key: str | None = None) -> dict[str, Any]:
326
+ url = f"{self.os_url.rstrip('/')}{path}"
327
+ headers = {"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json", "Accept": "application/json"}
328
+ if idempotency_key:
329
+ headers["Idempotency-Key"] = idempotency_key
330
+ req = urllib.request.Request(url, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST")
331
+ try:
332
+ for _name, _value in _connection_headers(req.full_url).items():
333
+ req.add_unredirected_header(_name, _value)
334
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
335
+ envelope = json.loads(resp.read().decode("utf-8"))
336
+ except urllib.error.HTTPError as exc:
337
+ detail = exc.read().decode("utf-8", errors="replace")[:500]
338
+ raise ChannelFootprintError(f"OS {path} failed: HTTP {exc.code} {detail}") from exc
339
+ except urllib.error.URLError as exc:
340
+ raise OsUnreachable(f"OS {path} unreachable: {exc.reason}") from exc
341
+ except (OSError, http.client.HTTPException, ValueError) as exc:
342
+ # A read timeout or a dropped connection after the request was
343
+ # sent surfaces outside URLError; it is a provider failure too.
344
+ raise ChannelFootprintError(f"OS {path} failed in transit: {type(exc).__name__}: {str(exc)[:200]}") from exc
345
+ data = envelope.get("data")
346
+ if envelope.get("code") not in (None, 200) or not isinstance(data, dict):
347
+ raise ChannelFootprintError(f"OS {path} returned no data: {str(envelope)[:300]}")
348
+ return data
349
+
350
+ def _fetch_url(self, url: str) -> tuple[int, str]:
351
+ """Plain GET of a public page; the status and a bounded body."""
352
+ req = urllib.request.Request(url, headers={
353
+ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
354
+ "Accept": "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8",
355
+ "Accept-Language": "en-US,en;q=0.9",
356
+ })
357
+ timeout = CDX_TIMEOUT_SECONDS if url.startswith(WAYBACK_CDX_URL) else FETCH_TIMEOUT_SECONDS
358
+ try:
359
+ for _name, _value in _connection_headers(req.full_url).items():
360
+ req.add_unredirected_header(_name, _value)
361
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
362
+ body = resp.read(FETCH_MAX_BYTES + 1)
363
+ status = int(getattr(resp, "status", 200) or 200)
364
+ except urllib.error.HTTPError as exc:
365
+ return exc.code, ""
366
+ except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
367
+ raise ChannelFootprintError(f"fetch of {url} failed: {getattr(exc, 'reason', exc)}") from exc
368
+ return status, body[:FETCH_MAX_BYTES].decode("utf-8", errors="replace")
369
+
370
+ def _cache_read(self, cache_file: Path) -> dict[str, Any] | None:
371
+ try:
372
+ return IO.read_cache_entry(cache_file)
373
+ except (OSError, ValueError, EOFError):
374
+ IO.drop_cache_entry(cache_file)
375
+ return None
376
+
377
+ def _cache_write(self, cache_file: Path, entry: dict[str, Any]) -> None:
378
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
379
+ IO.write_cache_entry(cache_file, entry)
380
+ self.cache_writes += 1
381
+
382
+ def _journal_start(self, digest: str) -> None:
383
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
384
+ with (self.cache_dir / "starts.log").open("a", encoding="utf-8") as handle:
385
+ handle.write(f"{date.today().isoformat()} {digest}\n")
386
+
387
+ def _prior_start_count(self, digest: str) -> int:
388
+ try:
389
+ lines = (self.cache_dir / "starts.log").read_text(encoding="utf-8").splitlines()
390
+ except OSError:
391
+ return 0
392
+ return sum(1 for line in lines if line.endswith(f" {digest}"))
393
+
394
+ def _reserve_paid_start(self, kind: str, digest: str) -> None:
395
+ """A durable total cap survives every retry, failed call and restart."""
396
+ if self.max_total_paid_starts is None:
397
+ return
398
+ path = self.cumulative_budget_path or self.cache_dir.parent / "budget.json"
399
+ try:
400
+ budget = json.loads(path.read_text())
401
+ except FileNotFoundError:
402
+ try:
403
+ legacy = [line for line in (self.cache_dir / "starts.log").read_text().splitlines() if line]
404
+ except OSError:
405
+ legacy = []
406
+ budget = {"schema_version": 1, "limit": self.max_total_paid_starts, "paid_starts": [{"kind": "legacy_actor", "digest": line.split()[-1], "started_on": line.split()[0]} for line in legacy]}
407
+ _write_atomic(path, json.dumps(budget, indent=1))
408
+ except (OSError, json.JSONDecodeError) as exc:
409
+ raise ChannelFootprintError(f"invalid cumulative budget ledger: {exc}") from exc
410
+ if budget.get("schema_version") != 1 or type(budget.get("limit")) is not int or budget["limit"] < 0 or not isinstance(budget.get("paid_starts"), list):
411
+ raise ChannelFootprintError("invalid cumulative budget ledger shape")
412
+ # A new invocation cannot silently raise the previously accepted cap.
413
+ maximum = min(int(budget["limit"]), self.max_total_paid_starts)
414
+ starts = budget.get("paid_starts", [])
415
+ if len(starts) >= maximum:
416
+ raise BudgetExhausted(f"cumulative paid-start budget exhausted ({maximum}); remaining checks are budget_deferred")
417
+ starts.append({"kind": kind, "digest": digest, "started_on": date.today().isoformat()})
418
+ budget.update(limit=maximum, paid_starts=starts, spent_starts=len(starts), money_estimate=None)
419
+ _write_atomic(path, json.dumps(budget, indent=1))
420
+
421
+ def _metered(self, kind: str, path: str, payload: dict[str, Any], timeout: float, *, budget_attr: str, budget_max: int, label: str, paid: bool, retry_delay: float,
422
+ cache_payload: dict[str, Any] | None = None) -> tuple[dict[str, Any], str]:
423
+ cache_payload = payload if cache_payload is None else cache_payload
424
+ key_raw = json.dumps({"kind": kind, "path": path, "payload": cache_payload}, sort_keys=True)
425
+ digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
426
+ cache_file = self.cache_dir / f"{digest}.json"
427
+ self.touched.add(digest)
428
+ entry = self._cache_read(cache_file)
429
+ if entry is not None:
430
+ self.cache_hits += 1
431
+ response = entry.get("response")
432
+ return (response if isinstance(response, dict) else {}), entry.get("retrieved_on") or date.today().isoformat()
433
+
434
+ transport = self.request or self._http
435
+ attempts = 0
436
+ wire_key = None
437
+ while True:
438
+ if getattr(self, budget_attr) >= budget_max:
439
+ progress = (
440
+ f"{self.cache_writes} new answer(s) cached in this attempt; rerun to continue from the cache"
441
+ if self.cache_writes
442
+ else "nothing new was cached in this attempt: a failing per-domain source costs up to "
443
+ f"{2 * SOURCE_BREAKER_FAILURES} starts before it is skipped, so raise the budget or narrow --sources"
444
+ )
445
+ raise BudgetExhausted(f"{label} budget exhausted ({budget_max} per attempt); {progress}")
446
+ self._reserve_paid_start(kind, digest)
447
+ setattr(self, budget_attr, getattr(self, budget_attr) + 1)
448
+ if paid:
449
+ # Paid starts count BEFORE the transport call (fail closed)
450
+ # and are journaled durably: a failed start may still bill.
451
+ # Every start gets its own key, including an in-attempt retry:
452
+ # the gateway released the failed start's wallet hold, and a
453
+ # reused key would run the actor without being able to settle.
454
+ wire_key = f"{self.key_scope}{digest}-{self._prior_start_count(digest)}"
455
+ self._journal_start(digest)
456
+ try:
457
+ response = transport(path, payload, timeout, idempotency_key=wire_key)
458
+ break
459
+ except ChannelFootprintError as exc:
460
+ if (kind == "actor" and re.search(r"HTTP 400\b", str(exc))) or (
461
+ kind == "spyfu" and re.search(r"HTTP 4(?:00|22)\b", str(exc))
462
+ ):
463
+ # The provider rejected this input (observed live: an
464
+ # actor URL list without schemes; SpyFu answers 400 for an
465
+ # unsupported market); cache the rejection so replays stay
466
+ # byte-stable and never pay for the same refusal again.
467
+ response = {"unavailable": str(exc)[:400]}
468
+ break
469
+ unreachable = isinstance(exc, OsUnreachable)
470
+ if attempts >= 1 or not (unreachable or re.search(r"HTTP 5\d\d", str(exc))):
471
+ raise
472
+ attempts += 1
473
+ (self.sleep or time.sleep)(max(retry_delay, OS_UNREACHABLE_RETRY_DELAY_SECONDS) if unreachable else retry_delay)
474
+ retrieved_on = date.today().isoformat()
475
+ self._cache_write(cache_file, {"kind": kind, "path": path, "payload": cache_payload, "response": response, "retrieved_on": retrieved_on})
476
+ return (response if isinstance(response, dict) else {}), retrieved_on
477
+
478
+ def _shared(self) -> Path:
479
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
480
+ return self.cache_dir
481
+
482
+ def _download(self, url: str) -> tuple[int, bytes]:
483
+ req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36", "Accept": "image/avif,image/webp,image/png,image/jpeg,*/*;q=0.8"})
484
+ try:
485
+ with urllib.request.urlopen(req, timeout=IMAGE_TIMEOUT_SECONDS) as resp:
486
+ return int(getattr(resp, "status", 200) or 200), resp.read(MAX_IMAGE_BYTES + 1)
487
+ except urllib.error.HTTPError as exc:
488
+ return exc.code, b""
489
+ except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
490
+ raise ChannelFootprintError(f"image download failed: {getattr(exc, 'reason', exc)}") from exc
491
+
492
+ def fetch_image(self, url: str) -> dict[str, Any] | None:
493
+ """Free, cached download of one ad image. Bytes are kept under the
494
+ shared cache by content hash, so a rerun can classify again after the
495
+ platform's signed URL has expired. Returns the image, None when the
496
+ image is definitively unusable (not a JPEG, PNG or WebP, too large,
497
+ HTTP 4xx; cached as such), or {"transient": ...} for a rate limit,
498
+ server error or network failure (not cached; retried next attempt)."""
499
+ digest = hashlib.sha256(json.dumps({"kind": "image", "url": url}, sort_keys=True).encode("utf-8")).hexdigest()
500
+ cache_file = self._shared() / f"{digest}.json"
501
+ self.touched.add(digest)
502
+ entry = self._cache_read(cache_file)
503
+ if entry is not None:
504
+ response = entry.get("response") if isinstance(entry.get("response"), dict) else {}
505
+ blob = self._shared() / "images" / str(response.get("file") or "")
506
+ if not response.get("sha256") or blob.is_file():
507
+ self.cache_hits += 1
508
+ return response if response.get("sha256") else None
509
+ if self.image_fetches >= self.max_image_fetches:
510
+ raise BudgetExhausted(f"image download budget exhausted ({self.max_image_fetches} per attempt); rerun to continue from the cache")
511
+ self.image_fetches += 1
512
+ try:
513
+ status, content = (self.fetch_bytes or self._download)(url)
514
+ except ChannelFootprintError as exc:
515
+ return {"transient": str(exc)[:200]}
516
+ if _transient_status(status):
517
+ return {"transient": f"HTTP {status}"}
518
+ media_type = sniff_image(content) if status == 200 and 0 < len(content) <= MAX_IMAGE_BYTES else None
519
+ response: dict[str, Any] = {"status": status}
520
+ if media_type:
521
+ sha = hashlib.sha256(content).hexdigest()
522
+ ext = media_type.split("/", 1)[1]
523
+ (self._shared() / "images").mkdir(parents=True, exist_ok=True)
524
+ # Atomic: the same bytes may be hard-linked into published assets.
525
+ tmp = self._shared() / "images" / f".{sha}.{ext}.tmp"
526
+ tmp.write_bytes(content)
527
+ tmp.replace(self._shared() / "images" / f"{sha}.{ext}")
528
+ response.update(sha256=sha, media_type=media_type, file=f"{sha}.{ext}", bytes=len(content))
529
+ else:
530
+ response["unusable"] = "not a JPEG, PNG or WebP image under 4 MB" if status == 200 else f"HTTP {status}"
531
+ self._cache_write(cache_file, {"kind": "image", "url": url, "response": response, "cutoff": self.cutoff,
532
+ "retrieved_on": date.today().isoformat()})
533
+ return response if media_type else None
534
+
535
+ def image_bytes(self, image: dict[str, Any]) -> bytes:
536
+ return (self._shared() / "images" / image["file"]).read_bytes()
537
+
538
+ def classify_relevance(self, payload: dict[str, Any]) -> dict[str, Any]:
539
+ """One cached classifier call (an AI upstream call), reserved in the
540
+ durable budget like a provider start. The cache key covers frame
541
+ hashes, never the image bytes."""
542
+ key = {**payload, "frames": [{k: v for k, v in frame.items() if k != "content_base64"} for frame in payload.get("frames", [])]}
543
+ response, _ = self._metered("relevance", RELEVANCE_PATH, payload, RELEVANCE_TIMEOUT_SECONDS, budget_attr="relevance_calls",
544
+ budget_max=self.max_relevance_calls, label="relevance classification", paid=False,
545
+ retry_delay=FREE_RETRY_DELAY_SECONDS, cache_payload=key)
546
+ return response
547
+
548
+ def run_actor(self, actor: str, actor_input: dict[str, Any], limit: int) -> tuple[list[dict[str, Any]], str]:
549
+ payload = {"actor": actor, "input": actor_input, "sync": True, "limit": limit}
550
+ response, retrieved_on = self._metered(
551
+ "actor", "/scrape/apify/run", payload, 330.0,
552
+ budget_attr="actor_runs", budget_max=self.max_actor_runs, label="actor-run", paid=True, retry_delay=ACTOR_RETRY_DELAY_SECONDS,
553
+ )
554
+ if response.get("unavailable"):
555
+ raise SourceUnavailable(f"actor {actor} rejected the request: {response['unavailable']}")
556
+ rows = response.get("rows")
557
+ return ([r for r in rows if isinstance(r, dict)] if isinstance(rows, list) else []), retrieved_on
558
+
559
+ def meta_library_search(self, payload: dict[str, Any]) -> tuple[list[dict[str, Any]], str]:
560
+ response, retrieved_on = self._metered(
561
+ "meta-ad-library", "/tools/meta-ad-library/search", payload, 90.0,
562
+ budget_attr="meta_searches", budget_max=self.max_meta_searches, label="meta-ad-library", paid=False, retry_delay=FREE_RETRY_DELAY_SECONDS,
563
+ )
564
+ rows = response.get("data")
565
+ return ([r for r in rows if isinstance(r, dict)] if isinstance(rows, list) else []), retrieved_on
566
+
567
+ def spyfu_domain_stats(self, domain: str, country: str, months: int) -> tuple[dict[str, Any], str]:
568
+ """SpyFu's modeled Google Ads budget, paid clicks and paid keywords per
569
+ month. Billed per returned row (one per month) to the shared account;
570
+ the gateway's daily `spyfu:rows` limit bounds it per claw."""
571
+ response, retrieved_on = self._metered(
572
+ "spyfu", "/tools/spyfu/domain-stats", {"domain": domain, "country_code": country, "months": months}, 90.0,
573
+ budget_attr="spyfu_calls", budget_max=self.max_spyfu_calls, label="SpyFu lookup", paid=False, retry_delay=FREE_RETRY_DELAY_SECONDS,
574
+ )
575
+ if response.get("unavailable"):
576
+ raise SourceUnavailable(f"SpyFu rejected the request: {response['unavailable']}")
577
+ return response, retrieved_on
578
+
579
+ def brave(self, query: str, count: int, country: str) -> tuple[dict[str, Any], str]:
580
+ return self._metered(
581
+ "search", "/search/web/brave", {"query": query, "count": count, "country": country}, 60.0,
582
+ budget_attr="search_calls", budget_max=self.max_search_calls, label="web search", paid=False, retry_delay=FREE_RETRY_DELAY_SECONDS,
583
+ )
584
+
585
+ def fetch_page(self, url: str, *, budget_attr: str = "fetches", budget_max: int | None = None, label: str = "page fetch") -> tuple[dict[str, Any], str]:
586
+ """Free, cached GET of a public page or API; failures are cached as a
587
+ status so a blocked site does not get hammered on every rerun."""
588
+ kind = "cdx" if budget_attr == "cdx_calls" else "fetch"
589
+ key_raw = json.dumps({"kind": kind, "url": url}, sort_keys=True)
590
+ digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
591
+ cache_file = self.cache_dir / f"{digest}.json"
592
+ self.touched.add(digest)
593
+ entry = self._cache_read(cache_file)
594
+ if entry is not None:
595
+ cached = entry.get("response") if isinstance(entry.get("response"), dict) else {}
596
+ if _transient_status(cached.get("status")):
597
+ # An older runner cached rate limits and server errors; such
598
+ # an entry is a miss, never a replayable answer.
599
+ IO.drop_cache_entry(cache_file)
600
+ else:
601
+ self.cache_hits += 1
602
+ return cached, entry.get("retrieved_on") or date.today().isoformat()
603
+ maximum = self.max_fetches if budget_max is None else budget_max
604
+ if getattr(self, budget_attr) >= maximum:
605
+ raise BudgetExhausted(f"{label} budget exhausted ({maximum} per attempt); {self.cache_writes} new answer(s) cached in this attempt; rerun to continue from the cache")
606
+ setattr(self, budget_attr, getattr(self, budget_attr) + 1)
607
+ transport = self.fetch or self._fetch_url
608
+ retrieved_on = date.today().isoformat()
609
+ try:
610
+ status, body = transport(url)
611
+ response: dict[str, Any] = {"status": status, "body": body}
612
+ except ChannelFootprintError as exc:
613
+ # A timeout or connection error is transient: report it for this
614
+ # attempt but do not cache it, so a rerun tries again (bounded by
615
+ # the per-attempt budget).
616
+ return {"status": None, "body": "", "error": str(exc)[:200]}, retrieved_on
617
+ if _transient_status(status):
618
+ # Rate limits and server errors are transient as well: not cached,
619
+ # not an observation.
620
+ return {**response, "error": f"HTTP {status}"}, retrieved_on
621
+ self._cache_write(cache_file, {"kind": kind, "url": url, "response": response, "retrieved_on": retrieved_on})
622
+ return response, retrieved_on
623
+
624
+ def ledger_totals(self) -> dict[str, Any]:
625
+ """Durable provenance over entries THIS execution touched. A cached
626
+ actor rejection is a journaled start, not a successful observation."""
627
+ counts = {"actor": 0, "meta-ad-library": 0, "search": 0, "spyfu": 0, "fetch": 0, "cdx": 0, "relevance": 0}
628
+ rejected = 0
629
+ rejected_spyfu = 0
630
+ dates: list[str] = []
631
+ if self.cache_dir.is_dir():
632
+ for digest in sorted(self.touched):
633
+ try:
634
+ entry = IO.read_cache_entry(self.cache_dir / f"{digest}.json")
635
+ except (OSError, ValueError, EOFError):
636
+ continue
637
+ if not isinstance(entry, dict):
638
+ continue
639
+ kind = entry.get("kind")
640
+ response = entry.get("response") if isinstance(entry.get("response"), dict) else {}
641
+ if response.get("unavailable"):
642
+ if kind == "spyfu":
643
+ rejected_spyfu += 1
644
+ else:
645
+ rejected += 1
646
+ elif kind in ("fetch", "cdx") and _transient_status(response.get("status")):
647
+ continue # an older cache entry of a rate limit or server error is not an observation
648
+ elif kind in counts:
649
+ counts[kind] += 1
650
+ if isinstance(entry.get("retrieved_on"), str):
651
+ dates.append(entry["retrieved_on"])
652
+ today = date.today().isoformat()
653
+ try:
654
+ starts_recorded = sum(1 for line in (self.cache_dir / "starts.log").read_text(encoding="utf-8").splitlines() if line)
655
+ except OSError:
656
+ starts_recorded = 0
657
+ return {
658
+ "actor_runs": counts["actor"],
659
+ "meta_searches": counts["meta-ad-library"],
660
+ "search_calls": counts["search"],
661
+ "fetches": counts["fetch"],
662
+ "cdx_calls": counts["cdx"],
663
+ "spyfu_calls": counts["spyfu"],
664
+ "relevance_calls": counts["relevance"],
665
+ "provider_calls": counts["actor"] + counts["meta-ad-library"] + counts["search"] + counts["spyfu"],
666
+ "rejected_actor_runs": rejected,
667
+ "rejected_spyfu_calls": rejected_spyfu,
668
+ # Every successful observation, metered or free: the completion
669
+ # proof for selections that use only free sources.
670
+ "observations": counts["actor"] + counts["meta-ad-library"] + counts["search"] + counts["spyfu"] + counts["fetch"] + counts["cdx"],
671
+ "actor_starts_recorded": starts_recorded,
672
+ "first_retrieved_on": min(dates) if dates else today,
673
+ "last_retrieved_on": max(dates) if dates else today,
674
+ }
675
+
676
+
677
+ # ── inputs ──────────────────────────────────────────────────────────────────
678
+
679
+ def parse_domains(raw: str, cap: int) -> list[str]:
680
+ domains: list[str] = []
681
+ for term in (raw or "").split(","):
682
+ domain = registrable_domain(term)
683
+ if domain and _DOMAIN_RE.match(domain) and domain not in domains:
684
+ domains.append(domain)
685
+ return domains[:cap]
686
+
687
+
688
+ DOMAIN_SOURCES = frozenset({"pixels", "history", "traffic", "linkedin", "reddit"}) # one check per domain, not per country
689
+ EXPANDABLE_SOURCES = frozenset({"google", "meta", "tiktok", "linkedin"}) # a direct sample may be expanded once
690
+
691
+
692
+ def check_starts(source: str, country: str | None, pixel_actor_fallback: bool = False) -> dict[str, int]:
693
+ """Paid provider starts of one check, by provider kind."""
694
+ if source == "partners":
695
+ return {"search": 3}
696
+ if source == "spyfu":
697
+ return {"spyfu": 1}
698
+ if source == "meta":
699
+ return {"actor": 1, "meta-ad-library": 1} if country in EU_COUNTRIES else {"actor": 1}
700
+ if source == "tiktok":
701
+ return {"actor": 1} if country in EU_COUNTRIES else {}
702
+ if source == "pixels":
703
+ return {"actor": 1} if pixel_actor_fallback else {}
704
+ return {"actor": 1} if source in {"traffic", "google", "linkedin", "reddit"} else {}
705
+
706
+
707
+ def collection_plan(competitors: list[tuple[str, str]], sources: list[str], target_countries: list[str], *,
708
+ pixel_actor_fallback: bool = False) -> list[dict[str, Any]]:
709
+ """Every (domain, tier) x source x country check with its upper bound of
710
+ paid provider starts. Every scoped domain is collected regardless of role."""
711
+ plan = []
712
+ for domain, role in competitors:
713
+ for source in sources:
714
+ for country in ([None] if source in DOMAIN_SOURCES else target_countries):
715
+ starts = check_starts(source, country, pixel_actor_fallback)
716
+ plan.append({"domain": domain, "competitor_tier": role, "source": source, "country": country,
717
+ "max_expansion_factor": 2 if source in EXPANDABLE_SOURCES else 1,
718
+ "provider_starts": starts, "provider_starts_upper_bound": sum(starts.values())})
719
+ return plan
720
+
721
+
722
+ def sniff_image(content: bytes) -> str | None:
723
+ if content[:3] == b"\xff\xd8\xff":
724
+ return "image/jpeg"
725
+ if content[:8] == b"\x89PNG\r\n\x1a\n":
726
+ return "image/png"
727
+ if content[:4] == b"RIFF" and content[8:12] == b"WEBP":
728
+ return "image/webp"
729
+ return None
730
+
731
+
732
+ def typed_intake(workspace: Path) -> dict[str, Any]:
733
+ """Use the existing typed-input validator; no prose or project discovery."""
734
+ loader = _load_script("run-competitor-set.py", "pmr_typed_intake_for_channels")
735
+ try:
736
+ return loader.load_intake(workspace / _artifact_path("egbe-docs/research/market/intake.json")) or {}
737
+ except loader.CompetitorSetError as exc:
738
+ raise ValueError(str(exc)) from exc
739
+
740
+
741
+ def relevance_context(workspace: Path, args: argparse.Namespace) -> tuple[str, list[str]]:
742
+ """The offer the classifier judges ads against, and the audience languages."""
743
+ parts = [IO.context_text(workspace, SECTION_ID)]
744
+ intake = typed_intake(workspace)
745
+ offer = {key: value for key, value in (intake.get("offer") or {}).items()
746
+ if value not in (None, "", [])}
747
+ if offer:
748
+ parts.append("Offer card: " + json.dumps(offer, ensure_ascii=False))
749
+ text = "\n\n".join(part for part in parts if part)
750
+ if len(text) > 24000:
751
+ raise ChannelFootprintError("relevance context exceeds 24000 characters; supply bounded context")
752
+ explicit = str(_arg(args, "languages", "") or "")
753
+ languages = ([code.strip().lower() for code in explicit.split(",") if code.strip()]
754
+ if explicit else list(intake.get("languages") or []))
755
+ _load_script("creative-semantics.py", "pmr_relevance_context_validator").build_semantic_context(text, languages)
756
+ return text, languages
757
+
758
+
759
+ def model_relevance(client: "OsClient", corpus: list[dict[str, Any]], pending: list[str], decisions: dict[str, Any], context_hash: str,
760
+ product_context: str, languages: list[str]) -> dict[str, Any] | None:
761
+ """Decide pending ads by the classifier's relevance verdict from their text
762
+ or their downloaded image; an ad with neither stays pending for review.
763
+ None: nothing new was decided (nothing to judge, or the budget stopped)."""
764
+ platforms = set(_taxonomy()["platforms"])
765
+ judged = []
766
+ for ad in (ad for ad in corpus if ad["ad_id"] in pending and str(ad["platform"]).removesuffix("_ads") in platforms):
767
+ text = (ad.get("ad_text") or "").strip() # a CTA label alone is no copy: the offer is in the picture
768
+ image = client.fetch_image(ad["image_url"]) if not text and ad.get("image_url") else None
769
+ if text or (image and image.get("sha256")):
770
+ judged.append((ad, image if image and image.get("sha256") else None))
771
+ if not judged:
772
+ return None
773
+ semantic = _load_script("creative-semantics.py", "pmr_semantics_for_footprint").build_semantic_context(product_context, languages)
774
+ new: list[dict[str, Any]] = []
775
+ for start in range(0, len(judged), RELEVANCE_BATCH):
776
+ batch = {ad["ad_id"]: ad for ad, _ in judged[start:start + RELEVANCE_BATCH]}
777
+ frames = [{"ref": f"{ad['ad_id']}/image", "timestamp_seconds": 0, "media_type": image["media_type"], "sha256": image["sha256"],
778
+ "content_base64": base64.b64encode(client.image_bytes(image)).decode("ascii")} for ad, image in judged[start:start + RELEVANCE_BATCH] if image]
779
+ creatives = [{"ad_ref": ad_id, "platform": str(ad["platform"]).removesuffix("_ads"), "advertiser": str(ad.get("advertiser") or ad["domain"])[:200],
780
+ "body": (ad.get("ad_text") or None) and str(ad["ad_text"])[:4000], "cta_label": (ad.get("cta") or None) and str(ad["cta"])[:100],
781
+ "landing_url": (ad.get("landing_url") or None) and str(ad["landing_url"])[:2000],
782
+ "frame_refs": [frame["ref"] for frame in frames if frame["ref"] == f"{ad_id}/image"]} for ad_id, ad in batch.items()]
783
+ try:
784
+ data = client.classify_relevance({"schema_version": 1, "taxonomy_version": _taxonomy()["taxonomy_version"], "semantic_context": semantic,
785
+ "creatives": creatives, "frames": frames})
786
+ except BudgetExhausted:
787
+ break # the rest stays pending; a rerun continues from the cache
788
+ for item in data.get("classifications") or []:
789
+ ad = batch.get(item.get("ad_ref"))
790
+ if ad is None:
791
+ continue
792
+ verdict = (item.get("semantics") or {}).get("relevance") or {}
793
+ status = verdict.get("status") if verdict.get("status") in {"relevant", "irrelevant"} else "unknown"
794
+ excerpt = f' Excerpt: "{verdict["excerpt"]}".' if verdict.get("excerpt") else ""
795
+ source = "ad_text" if (ad.get("ad_text") or "").strip() else "image"
796
+ new.append({"ad_id": ad["ad_id"], "candidate_sha256": ad["candidate_sha256"], "relevance": status,
797
+ "reason": f"{verdict.get('reason') or 'The classifier gave no reason.'}{excerpt} (classifier role {data.get('model_role')})",
798
+ "evidence_refs": [f"{ad['ad_id']}:{source}", *([ad["landing_url"]] if ad.get("landing_url") else [])], "decided_by": "model"})
799
+ if not new:
800
+ return None
801
+ decided = {d["ad_id"] for d in new}
802
+ return {"schema_version": 1, "product_context_sha256": context_hash,
803
+ "decisions": [d for d in decisions.get("decisions", []) if d.get("ad_id") not in decided] + new}
804
+
805
+
806
+ def _taxonomy() -> dict[str, Any]:
807
+ return json.loads((Path(__file__).resolve().parent.parent / "references/config/creative-taxonomy.v1.json").read_text(encoding="utf-8"))
808
+
809
+
810
+ def collection_budget(plan: list[dict[str, Any]], config: dict[str, Any]) -> dict[str, Any]:
811
+ """Persistent shared cap, including expansion and the configured retry reserve.
812
+
813
+ Starts are not allocated to provider kinds in the ledger. Failed uncached
814
+ requests can consume the pool across resumed attempts, so each active kind's
815
+ conservative upper bound is the whole pool, rather than a promised share.
816
+ """
817
+ reserve = int(config.get("budget", {}).get("retry_reserve_factor", 1))
818
+ if reserve < 0:
819
+ raise ChannelFootprintError("retry reserve factor must be nonnegative")
820
+ base, expanded = {}, {}
821
+ for check in plan:
822
+ for kind, starts in check["provider_starts"].items():
823
+ base[kind] = base.get(kind, 0) + starts
824
+ expanded[kind] = expanded.get(kind, 0) + starts * check["max_expansion_factor"]
825
+ limit = max(1, sum(expanded.values()) * (1 + reserve))
826
+ return {"base": base, "expanded": expanded, "upper": {kind: limit for kind in expanded},
827
+ "retry_reserve_factor": reserve, "cumulative_limit": limit}
828
+
829
+
830
+ def paid_start_budget(plan: list[dict[str, Any]], config: dict[str, Any]) -> dict[str, int]:
831
+ """Aggregate runner budget projected from the shared provider calculation."""
832
+ budget = collection_budget(plan, config)
833
+ return {"base": sum(budget["base"].values()), "bounded": sum(budget["expanded"].values()),
834
+ "retry_reserve_factor": budget["retry_reserve_factor"], "cumulative_limit": budget["cumulative_limit"]}
835
+
836
+
837
+ def parse_sources(raw: str) -> list[str]:
838
+ if not (raw or "").strip():
839
+ return list(ALL_SOURCES)
840
+ chosen = [s.strip().lower() for s in raw.split(",") if s.strip()]
841
+ unknown = [s for s in chosen if s not in ALL_SOURCES]
842
+ if unknown:
843
+ raise ChannelFootprintError(f"unknown sources: {', '.join(unknown)} (known: {', '.join(ALL_SOURCES)})")
844
+ return [s for s in ALL_SOURCES if s in chosen]
845
+
846
+
847
+ def domains_from_competitor_set(workspace: Path, cap: int) -> list[str]:
848
+ evidence = _load_script("research-evidence.py", "pmr_evidence_for_channel_footprint")
849
+ return evidence.competitor_set_inputs(workspace, consumer=SECTION_ID, cap=cap)
850
+
851
+
852
+ def domains_from_keyword_evidence(workspace: Path, cap: int, exclude=frozenset()) -> list[str]:
853
+ evidence = _load_script("research-evidence.py", "pmr_evidence_for_channel_footprint_kwe")
854
+ return evidence.keyword_inputs(workspace, consumer=SECTION_ID, row_type="competitor_domain", field="domain", cap=cap, exclude=exclude)
855
+
856
+
857
+ def domains_excluded_by_competitor_set(workspace: Path) -> set[str]:
858
+ """Domains the completed Competitor Set rejected; never refilled automatically."""
859
+ evidence = _load_script("research-evidence.py", "pmr_evidence_for_channel_footprint_excl")
860
+ return evidence.competitor_set_excluded_domains(workspace)
861
+
862
+
863
+ @dataclass
864
+ class RunInputs:
865
+ domains: list[str]
866
+ domain_sources: dict[str, str]
867
+ product_domain: str
868
+ sources: list[str]
869
+ country: str
870
+ eu_country: str
871
+ pixel_actor_fallback: bool
872
+ target_countries: list[str] = field(default_factory=list)
873
+ competitor_context: dict[str, dict[str, Any]] = field(default_factory=dict)
874
+ product_context_sha256: str = ""
875
+ effective_policy: dict[str, Any] = field(default_factory=dict)
876
+ relevance_decisions: dict[str, Any] = field(default_factory=dict)
877
+ stage: str = "all"
878
+ # Sample expansions recorded by the previous compatible collect.
879
+ prior_expansions: list[dict[str, Any]] = field(default_factory=list)
880
+
881
+
882
+ # ── helpers ─────────────────────────────────────────────────────────────────
883
+
884
+ def _days_between(start_iso: str | None, end_iso: str | None) -> int | None:
885
+ try:
886
+ start = date.fromisoformat(str(start_iso)[:10])
887
+ end = date.fromisoformat(str(end_iso)[:10])
888
+ except (ValueError, TypeError):
889
+ return None
890
+ return max(0, (end - start).days)
891
+
892
+
893
+ def _iso_from_epoch(value: Any) -> str | None:
894
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
895
+ if isinstance(value, str) and value.strip():
896
+ digits = value.strip()
897
+ if digits.isdigit():
898
+ value = int(digits)
899
+ else:
900
+ return digits[:10]
901
+ else:
902
+ return None
903
+ try:
904
+ seconds = float(value)
905
+ if seconds > 10_000_000_000:
906
+ seconds /= 1000.0
907
+ return datetime.fromtimestamp(seconds, tz=timezone.utc).date().isoformat()
908
+ except (OverflowError, OSError, ValueError):
909
+ return None
910
+
911
+
912
+ _GENERIC_NAME_TOKENS = {
913
+ "inc", "llc", "ltd", "co", "corp", "corporation", "gmbh", "app", "apps", "software", "hq", "official",
914
+ "team", "teams", "io", "ai", "com", "labs", "technologies", "technology", "tech", "the", "limited", "plc", "sas", "srl",
915
+ # product-tier pages a brand runs under its own name
916
+ "pro", "plus", "premium", "business", "enterprise", "for", "education", "edu", "learn", "academy",
917
+ }
918
+ # Handle prefixes platforms put in front of an advertiser name.
919
+ _NAME_PREFIX_RE = re.compile(r"^\s*(?:u/|r/|@)+", re.IGNORECASE)
920
+
921
+
922
+ def _name_matches(name: Any, brand: str) -> bool:
923
+ """Advertiser identity: the name IS the brand (plus generic corporate or
924
+ product-tier suffixes), not merely a name that contains the brand token:
925
+ 'Mural Painting Co' is a namesake of mural.co, 'Lucid Software' and
926
+ 'Codecademy Pro' are the brands."""
927
+ if not brand:
928
+ return False
929
+ tokens = [t for t in re.split(r"[^a-z0-9]+", _NAME_PREFIX_RE.sub("", str(name or "")).lower()) if t]
930
+ if not tokens:
931
+ return False
932
+ joined = "".join(tokens)
933
+ if joined == brand:
934
+ return True
935
+ for size in range(1, len(tokens) + 1):
936
+ if "".join(tokens[:size]) == brand:
937
+ return all(t in _GENERIC_NAME_TOKENS for t in tokens[size:])
938
+ if len("".join(tokens[:size])) > len(brand):
939
+ break
940
+ return False
941
+
942
+
943
+ def _mentions(text: Any, brand: str) -> bool:
944
+ """Free text (a search result title or address) that names the brand."""
945
+ return bool(brand) and brand in re.sub(r"[^a-z0-9]", "", str(text or "").lower())
946
+
947
+
948
+ def _chunks(items: list[str], size: int) -> list[list[str]]:
949
+ size = max(1, int(size))
950
+ return [items[i:i + size] for i in range(0, len(items), size)]
951
+
952
+
953
+ def _transient_status(status: Any) -> bool:
954
+ return isinstance(status, int) and not isinstance(status, bool) and (status == 429 or status >= 500)
955
+
956
+
957
+ def _provider_detail(exc: BaseException) -> str:
958
+ """The provider's message without the per-attempt counter that
959
+ `_call_with_context` appends, so evidence stays byte-stable on replay.
960
+ Only our own wrapper is unwrapped; a transport error's cause (urllib)
961
+ would lose the gateway's message."""
962
+ inner = exc.__cause__ if isinstance(exc.__cause__, ChannelFootprintError) else exc
963
+ return str(inner).split(" — this attempt had already started", 1)[0][:240]
964
+
965
+
966
+ def _coverage_gap(coverage: list[dict[str, Any]], source: str, domains: list[str], detail: str, status: str = "unavailable") -> None:
967
+ coverage.extend({"type": "coverage", "domain": d, "source": source, "status": status, "detail": detail[:240], "tier": "C"} for d in domains)
968
+
969
+
970
+ def _attempt(client: OsClient, coverage: list[dict[str, Any]], source: str, domains: list[str], fn: Callable[[], Any]) -> Any:
971
+ """One supplementary provider call. A failure other than budget
972
+ exhaustion becomes a coverage gap for THESE domains only; results already
973
+ collected for other domains stay and later domains are still attempted.
974
+ After SOURCE_BREAKER_FAILURES consecutive failures of the same source the
975
+ remaining domains are skipped without starting more paid runs. Budget
976
+ exhaustion and an unreachable OS always propagate so a rerun resumes from
977
+ the cache."""
978
+ failures = client.source_failures.get(source, 0)
979
+ if failures >= SOURCE_BREAKER_FAILURES:
980
+ _coverage_gap(coverage, source, domains, f"skipped after {failures} consecutive {source} failures in this attempt; rerun to retry")
981
+ return None
982
+ hits_before = client.cache_hits
983
+ try:
984
+ result = fn()
985
+ except BudgetExhausted as exc:
986
+ if "cumulative" not in str(exc):
987
+ raise
988
+ _coverage_gap(coverage, source, domains, str(exc), status="budget_deferred")
989
+ return None
990
+ except OsUnreachable:
991
+ raise
992
+ except ChannelFootprintError as exc:
993
+ # Only a LIVE failure feeds the breaker: a rejection replayed from the
994
+ # cache costs nothing, and counting it would skip the remaining
995
+ # domains on every rerun forever.
996
+ if client.cache_hits == hits_before:
997
+ client.source_failures[source] = failures + 1
998
+ _coverage_gap(coverage, source, domains, _provider_detail(exc))
999
+ return None
1000
+ client.source_failures[source] = 0
1001
+ return result
1002
+
1003
+
1004
+ def _call_with_context(client: OsClient, subject: str, label: str, fn: Callable[[], Any]) -> Any:
1005
+ try:
1006
+ return fn()
1007
+ except ChannelFootprintError as exc:
1008
+ # Keep the subclass: a cached provider rejection stays recognisable.
1009
+ raise type(exc)(
1010
+ f"{label} for `{subject}` failed: {exc} — this attempt had already started "
1011
+ f"{client.actor_runs}/{client.max_actor_runs} actor run(s); see GET /me/limits?resource=apify:runs "
1012
+ "for remaining daily quota."
1013
+ ) from exc
1014
+
1015
+
1016
+ def _cumulative_answer(coverage: list[dict[str, Any]] | None, source: str, domains: list[str], country: str | None, fn: Callable[[], Any]) -> Any:
1017
+ """Catch total-cap exhaustion at the smallest provider request unit.
1018
+
1019
+ Completed chunks stay in the collector's result. Attempt exhaustion
1020
+ still stops and resumes from the provider cache.
1021
+ """
1022
+ try:
1023
+ return fn()
1024
+ except BudgetExhausted as exc:
1025
+ if "cumulative" not in str(exc):
1026
+ raise
1027
+ if coverage is not None:
1028
+ coverage.extend({"type": "coverage", "domain": d, "source": source, "country": country, "status": "budget_deferred", "detail": str(exc), "tier": "C"} for d in domains)
1029
+ return None
1030
+
1031
+
1032
+ def usable_destination(value: Any, _depth: int = 0) -> str:
1033
+ """The registrable domain of an ad destination when the value is really
1034
+ an address (not display text such as 'Learn more') and not the platform
1035
+ itself, a messaging or short host, a store, a link hub or a shortener;
1036
+ otherwise '' so the advertiser name decides. Click-tracking wrappers
1037
+ (l.facebook.com/l.php?u=...) are unwrapped to the address they carry."""
1038
+ text = str(value or "").strip()
1039
+ if not text or " " in text or "." not in text:
1040
+ return ""
1041
+ lowered = text.lower()
1042
+ host = lowered
1043
+ query = ""
1044
+ if "//" in lowered:
1045
+ parsed = urllib.parse.urlparse(lowered)
1046
+ host = parsed.hostname or ""
1047
+ query = parsed.query
1048
+ host = host.split("/", 1)[0].split("?", 1)[0].removeprefix("www.").rstrip(".")
1049
+ if host.split(":", 1)[0] in REDIRECT_WRAPPER_HOSTS or ("www." + host) in REDIRECT_WRAPPER_HOSTS:
1050
+ wrapped = urllib.parse.parse_qs(query).get("u") or urllib.parse.parse_qs(query).get("q") or []
1051
+ if wrapped and _depth < 2:
1052
+ return usable_destination(urllib.parse.unquote(wrapped[0]), _depth + 1)
1053
+ return ""
1054
+ domain = registrable_domain(text)
1055
+ if not domain or not _DOMAIN_RE.match(domain):
1056
+ return ""
1057
+ if domain in NEUTRAL_DESTINATION_HOSTS or host.split(":", 1)[0] in NEUTRAL_DESTINATION_HOSTS:
1058
+ return ""
1059
+ return domain
1060
+
1061
+
1062
+ def meta_search_url(brand: str, country: str, active_status: str = "all") -> str:
1063
+ """The public Ad Library page search. `all` also lists stopped ads that
1064
+ Meta still keeps (observed 2026-10-02: 58 of 71 US results for four
1065
+ global brands were inactive), so a sample of 30 is mostly old ads;
1066
+ `active` lists running ads only."""
1067
+ if active_status not in ("all", "active"):
1068
+ raise ValueError(f"unsupported Meta active_status {active_status!r}")
1069
+ return (
1070
+ f"https://www.facebook.com/ads/library/?active_status={active_status}&ad_type=all"
1071
+ f"&country={urllib.parse.quote(country)}&q=%22{urllib.parse.quote(brand)}%22&search_type=keyword_unordered&media_type=all"
1072
+ )
1073
+
1074
+
1075
+ # ── pixels ──────────────────────────────────────────────────────────────────
1076
+
1077
+ def detect_tags(blobs: list[str]) -> list[dict[str, str]]:
1078
+ """Match the versioned signature list over page HTML and container JS."""
1079
+ text = "\n".join(html_lib.unescape(b or "") for b in blobs)
1080
+ found: list[dict[str, str]] = []
1081
+ for tag, channel, pattern in TAG_SIGNATURES:
1082
+ if re.search(pattern, text, re.IGNORECASE):
1083
+ found.append({"tag": tag, "channel": channel})
1084
+ return found
1085
+
1086
+
1087
+ def detect_tag_managers(blobs: list[str]) -> list[str]:
1088
+ text = "\n".join(html_lib.unescape(b or "") for b in blobs)
1089
+ return [name for name, pattern in TAG_MANAGER_SIGNATURES if re.search(pattern, text, re.IGNORECASE)]
1090
+
1091
+
1092
+ def linkedin_company_slug(body: str) -> str | None:
1093
+ """The competitor's own LinkedIn page link (footer, schema markup)."""
1094
+ for match in LINKEDIN_COMPANY_RE.finditer(html_lib.unescape(body or "")):
1095
+ slug = match.group(1).strip("/").lower()
1096
+ if slug and slug not in {"company", "school", "showcase", "login", "signup"}:
1097
+ return slug
1098
+ return None
1099
+
1100
+
1101
+ def scan_site(client: OsClient, domain: str) -> dict[str, Any]:
1102
+ """Home page plus up to three Google Tag Manager containers."""
1103
+ page, _ = client.fetch_page(f"https://{domain}/")
1104
+ status = page.get("status")
1105
+ body = str(page.get("body") or "")
1106
+ if status is None:
1107
+ fetch_status = "error"
1108
+ elif int(status) >= 400 or (int(status) < 300 and not body.strip()):
1109
+ fetch_status = "blocked" if int(status) in (401, 403, 429, 503) else "error"
1110
+ else:
1111
+ fetch_status = "ok"
1112
+ containers = list(dict.fromkeys(GTM_ID_RE.findall(body)))[:GTM_CONTAINERS_PER_SITE] if fetch_status == "ok" else []
1113
+ blobs = [body]
1114
+ for container in containers:
1115
+ gtm, _ = client.fetch_page(f"https://www.googletagmanager.com/gtm.js?id={container}")
1116
+ blobs.append(str(gtm.get("body") or ""))
1117
+ tags = detect_tags(blobs) if fetch_status == "ok" else []
1118
+ managers = detect_tag_managers(blobs) if fetch_status == "ok" else []
1119
+ return {
1120
+ "type": "pixel_scan",
1121
+ "domain": domain,
1122
+ "fetch_status": fetch_status,
1123
+ "http_status": status,
1124
+ "via": "html",
1125
+ "gtm_containers": containers,
1126
+ "tag_managers": managers,
1127
+ "tags_detected": [t["tag"] for t in tags],
1128
+ "channels_detected": sorted({t["channel"] for t in tags}),
1129
+ "linkedin_company": linkedin_company_slug(body) if fetch_status == "ok" else None,
1130
+ "tier": "A" if fetch_status == "ok" else "C",
1131
+ }
1132
+
1133
+
1134
+ def scan_sites_with_actor(client: OsClient, actor: str, domains: list[str]) -> dict[str, dict[str, Any]]:
1135
+ """Headless fallback for sites whose HTML cannot be fetched directly."""
1136
+ rows, _ = _call_with_context(client, ", ".join(domains), "pixel detector actor", lambda: client.run_actor(
1137
+ actor, {"startUrls": [f"https://{d}/" for d in domains], "crawlPages": False, "maxPages": 1}, limit=len(domains) * 2,
1138
+ ))
1139
+ scans: dict[str, dict[str, Any]] = {}
1140
+ for row in rows:
1141
+ if row.get("type") not in (None, "page"):
1142
+ continue
1143
+ domain = registrable_domain(str(row.get("url") or ""))
1144
+ if domain not in domains:
1145
+ continue
1146
+ tools = [t for t in (row.get("tools") or []) if isinstance(t, dict) and t.get("detected", True)]
1147
+ names = [str(t.get("tool") or "") for t in tools]
1148
+ tags = detect_tags([" ".join(names + [json.dumps(t.get("details") or {}) for t in tools])])
1149
+ # Tool names from the detector are mapped through the same channel
1150
+ # vocabulary by keyword; unknown tools are kept as names only.
1151
+ for name in names:
1152
+ lowered = name.lower()
1153
+ for keyword, tag, channel in (
1154
+ ("facebook pixel", "meta_pixel", "meta_ads"), ("meta pixel", "meta_pixel", "meta_ads"),
1155
+ ("google ads", "google_ads_tag", "google_ads"), ("reddit", "reddit_pixel", "reddit_ads"),
1156
+ ("linkedin insight", "linkedin_insight", "linkedin_ads"), ("tiktok", "tiktok_pixel", "tiktok_ads"),
1157
+ ("bing", "microsoft_uet", "microsoft_ads"), ("microsoft", "microsoft_uet", "microsoft_ads"),
1158
+ ("twitter", "x_pixel", "x_ads"), ("pinterest", "pinterest_tag", "pinterest_ads"),
1159
+ ("snap", "snap_pixel", "snap_ads"), ("quora", "quora_pixel", "quora_ads"),
1160
+ ("taboola", "taboola", "display_native"), ("outbrain", "outbrain", "display_native"),
1161
+ ("criteo", "criteo", "display_native"), ("capterra", "capterra_tracker", "review_marketplace"),
1162
+ ("partnerstack", "partnerstack", "affiliate"), ("impact", "impact", "affiliate"),
1163
+ ("rewardful", "rewardful", "affiliate"), ("firstpromoter", "firstpromoter", "affiliate"),
1164
+ ):
1165
+ if keyword in lowered and tag not in {t["tag"] for t in tags}:
1166
+ tags.append({"tag": tag, "channel": channel})
1167
+ scans[domain] = {
1168
+ "type": "pixel_scan",
1169
+ "domain": domain,
1170
+ "fetch_status": "ok",
1171
+ "http_status": row.get("httpStatus"),
1172
+ "via": "actor",
1173
+ "gtm_containers": [],
1174
+ "tag_managers": [n.lower().replace(" ", "_") for n in names if any(k in n.lower() for k in ("tag manager", "segment", "tealium", "adobe launch"))][:4],
1175
+ "tags_detected": [t["tag"] for t in tags],
1176
+ "channels_detected": sorted({t["channel"] for t in tags}),
1177
+ "linkedin_company": None,
1178
+ "tools_reported": names[:40],
1179
+ "tier": "A",
1180
+ }
1181
+ return scans
1182
+
1183
+
1184
+ # ── page history (Wayback CDX) ──────────────────────────────────────────────
1185
+
1186
+ def _cdx_rows(response: dict[str, Any]) -> list[list[str]]:
1187
+ try:
1188
+ value = json.loads(str(response.get("body") or "[]"))
1189
+ except json.JSONDecodeError:
1190
+ return []
1191
+ return [r for r in value[1:] if isinstance(r, list)] if isinstance(value, list) and value else []
1192
+
1193
+
1194
+ def collect_history(client: OsClient, domain: str, today: date) -> dict[str, Any]:
1195
+ """Distinct home-page versions in the window (CDX `collapse=digest`,
1196
+ so a changed page counts once and an unchanged monthly crawl does not)
1197
+ plus the first archived date. A home page that only redirects to `www`
1198
+ has no 200 rows of its own, so `www.` is tried when the bare host is empty."""
1199
+ since = (today - timedelta(days=30 * HISTORY_MONTHS)).strftime("%Y%m")
1200
+
1201
+ def fetch(host: str, *, recent: bool) -> dict[str, Any]:
1202
+ target = urllib.parse.quote(host + "/", safe="")
1203
+ url = (
1204
+ f"{WAYBACK_CDX_URL}?url={target}&output=json&fl=timestamp,digest&filter=statuscode:200"
1205
+ f"&collapse=digest&from={since}&limit={CDX_ROW_LIMIT}"
1206
+ if recent
1207
+ else f"{WAYBACK_CDX_URL}?url={target}&output=json&fl=timestamp&filter=statuscode:200&limit=1"
1208
+ )
1209
+ before = client.cache_hits
1210
+ response, _ = client.fetch_page(url, budget_attr="cdx_calls", budget_max=client.max_cdx_calls, label="Wayback CDX")
1211
+ if client.cache_hits == before:
1212
+ # The archive rate-limits bursts; space out live calls.
1213
+ (client.sleep or time.sleep)(CDX_PAUSE_SECONDS)
1214
+ return response
1215
+
1216
+ host = domain
1217
+ recent = fetch(host, recent=True)
1218
+ if recent.get("status") == 200 and not _cdx_rows(recent):
1219
+ host = f"www.{domain}"
1220
+ recent = fetch(host, recent=True)
1221
+ first = fetch(host, recent=False)
1222
+ rows = _cdx_rows(recent)
1223
+ versions = {str(r[1]) for r in rows if len(r) > 1}
1224
+ months = {str(r[0])[:6] for r in rows if r}
1225
+ first_rows = _cdx_rows(first)
1226
+ first_ts = str(first_rows[0][0]) if first_rows and first_rows[0] else None
1227
+ available = recent.get("status") == 200 and first.get("status") == 200
1228
+ return {
1229
+ "type": "page_history",
1230
+ "domain": domain,
1231
+ "source": "wayback",
1232
+ "status": "ok" if available else "unavailable",
1233
+ "detail": "" if available else str(recent.get("error") or first.get("error") or f"HTTP {recent.get('status')}/{first.get('status')}")[:160],
1234
+ "archived_host": host,
1235
+ "first_archived": f"{first_ts[:4]}-{first_ts[4:6]}-{first_ts[6:8]}" if first_ts and len(first_ts) >= 8 else None,
1236
+ "months_archived_24m": len(months) if available else None,
1237
+ # Dynamic pages change their digest on every crawl; at the row cap
1238
+ # the count is a floor and the page is simply "dynamic".
1239
+ "distinct_versions_24m": len(versions) if available else None,
1240
+ "versions_capped": available and len(rows) >= CDX_ROW_LIMIT,
1241
+ "window_months": HISTORY_MONTHS,
1242
+ "tier": "B" if available else "C",
1243
+ }
1244
+
1245
+
1246
+ # ── traffic mix ─────────────────────────────────────────────────────────────
1247
+
1248
+ def _share_rows(domain: str, visits: Any, shares: Any, source: str) -> list[dict[str, Any]]:
1249
+ rows: list[dict[str, Any]] = []
1250
+ if not isinstance(shares, dict):
1251
+ return rows
1252
+ visits_value = None
1253
+ if isinstance(visits, (int, float)) and not isinstance(visits, bool):
1254
+ visits_value = int(visits)
1255
+ elif isinstance(visits, list) and visits:
1256
+ last = visits[-1]
1257
+ if isinstance(last, dict):
1258
+ last = last.get("visits") or last.get("value")
1259
+ if isinstance(last, (int, float)) and not isinstance(last, bool):
1260
+ visits_value = int(last)
1261
+ for key, value in shares.items():
1262
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
1263
+ continue
1264
+ share = float(value) * (100.0 if float(value) <= 1.0 else 1.0)
1265
+ channel_key = re.sub(r"[^a-z]", "_", str(key).lower()).strip("_")
1266
+ rows.append({
1267
+ "type": "traffic_mix",
1268
+ "domain": domain,
1269
+ "channel": channel_key,
1270
+ "paid_channel": TRAFFIC_CHANNEL_MAP.get(channel_key),
1271
+ "share_pct": round(share, 2),
1272
+ "share_pct_raw": share,
1273
+ "visits_monthly": visits_value,
1274
+ "source": source,
1275
+ "market_scope": "global",
1276
+ "tier": "B",
1277
+ })
1278
+ return rows
1279
+
1280
+
1281
+ def collect_traffic(client: OsClient, spec: dict[str, Any], domains: list[str], coverage: list[dict[str, Any]]) -> list[dict[str, Any]]:
1282
+ traffic: list[dict[str, Any]] = []
1283
+ for chunk in _chunks(domains, int(spec.get("batch_domains") or 10)):
1284
+ outcome = _attempt(client, coverage, "traffic", chunk, lambda: _call_with_context(client, ", ".join(chunk), "Similarweb actor", lambda: client.run_actor(
1285
+ spec["actor"], {"startUrls": [{"url": f"https://{d}"} for d in chunk]}, limit=len(chunk) * 2,
1286
+ )))
1287
+ if outcome is None:
1288
+ continue
1289
+ rows, _ = outcome
1290
+ traffic.extend(_traffic_rows(rows, chunk, coverage))
1291
+ return traffic
1292
+
1293
+
1294
+ def _traffic_rows(rows: list[dict[str, Any]], domains: list[str], coverage: list[dict[str, Any]]) -> list[dict[str, Any]]:
1295
+ traffic: list[dict[str, Any]] = []
1296
+ seen: set[str] = set()
1297
+ for row in rows:
1298
+ # Live shape (2026-10-02): {meta: {website, request_parameter},
1299
+ # traffic: {history: [{month, visits}], sources_share: {...}}};
1300
+ # flat {domain, visits, sources_share} is accepted as well.
1301
+ meta = row.get("meta") if isinstance(row.get("meta"), dict) else {}
1302
+ traffic_block = row.get("traffic") if isinstance(row.get("traffic"), dict) else {}
1303
+ domain = registrable_domain(str(meta.get("website") or meta.get("request_parameter") or row.get("domain") or row.get("url") or row.get("site") or ""))
1304
+ if domain not in domains or domain in seen:
1305
+ continue
1306
+ seen.add(domain)
1307
+ shares = traffic_block.get("sources_share") or row.get("sources_share") or row.get("traffic_sources") or row.get("sourcesShare") or {}
1308
+ visits = traffic_block.get("history") or row.get("visits")
1309
+ traffic.extend(_share_rows(domain, visits, shares, "similarweb_actor"))
1310
+ coverage.extend(
1311
+ {"type": "coverage", "domain": d, "source": "traffic", "status": "ok" if d in seen else "no_data", "detail": "" if d in seen else "no public Similarweb estimate (small or unlisted site)", "tier": "C"}
1312
+ for d in domains
1313
+ )
1314
+ return traffic
1315
+
1316
+
1317
+ # ── ad libraries ────────────────────────────────────────────────────────────
1318
+
1319
+ def _model() -> ModuleType:
1320
+ return _load_script("channel-footprint-model.py", "pmr_channel_footprint_model")
1321
+
1322
+
1323
+ def _presence(domain: str, platform: str, source: str, ads: list[dict[str, Any]], *, market_scope: str, verified_by: str, today: date, note: str = "", country: str | None = None, request_limit: int | None = None, raw_count: int | None = None, native_date_scope: str = "global", native_activity_scope: str = "global", retrieved_on: str | None = None) -> dict[str, Any]:
1324
+ """Raw source observation; eligibility is decided after product relevance."""
1325
+ model = _model()
1326
+ policy, _ = _load_script("research-policy.py", "pmr_channel_policy").load_policy()
1327
+ observed = min(today.isoformat(), retrieved_on) if retrieved_on else today.isoformat()
1328
+ observations = [model.observation(a, source, country, market_scope, date.fromisoformat(observed), policy, native_date_scope, native_activity_scope) for a in ads]
1329
+ durations = [o["served_days"] for o in observations if o["served_days"] is not None]
1330
+ formats: dict[str, int] = {}
1331
+ formats_recent: dict[str, int] = {}
1332
+ for ad, obs in zip(ads, observations):
1333
+ fmt = str(ad.get("format") or "unknown").lower()
1334
+ formats[fmt] = formats.get(fmt, 0) + 1
1335
+ if obs["status"] in {"active", "recent"}:
1336
+ formats_recent[fmt] = formats_recent.get(fmt, 0) + 1
1337
+ received = len(ads) if raw_count is None else raw_count
1338
+ capped = bool(request_limit and received >= request_limit)
1339
+ return {"type": "library_presence", "domain": domain, "platform": platform, "source": source,
1340
+ "matched_ads": len(ads), "active_ads": sum(o["status"] in {"active", "recent"} for o in observations),
1341
+ "oldest_days": max(durations) if durations else None,
1342
+ "latest_seen": max((o["last_shown"] for o in observations if o["last_shown"]), default=None),
1343
+ "formats": formats, "formats_recent": formats_recent, "present": bool(ads), "market_scope": market_scope,
1344
+ "verified_by": verified_by, "note": note, "tier": "A" if ads else "B",
1345
+ "country": country, "countries": [country] if country else [], "native_date_scope": native_date_scope,
1346
+ "native_activity_scope": native_activity_scope, "observed_on": observed, "source_retrieved_on": retrieved_on or observed, "ads": ads,
1347
+ "status": "observed" if ads else ("not_verifiable" if source == "reddit_inspiration_actor" else "no_matches_in_sample"),
1348
+ "sampling": {"request_limit": request_limit, "raw_received": received, "identity_matched": len(ads),
1349
+ "limit_reached": capped, "complete": False if capped else None, "pagination_complete": None,
1350
+ "total_ads": None, "total_lower_bound": received if capped else None}}
1351
+
1352
+
1353
+ def collect_google(client: OsClient, spec: dict[str, Any], domains: list[str], today: date, country: str | None = None, coverage: list[dict[str, Any]] | None = None) -> list[dict[str, Any]]:
1354
+ per_domain = int(spec.get("ads_per_domain") or 40)
1355
+ results: list[dict[str, Any]] = []
1356
+ for chunk in _chunks(domains, int(spec.get("batch_domains") or 5)):
1357
+ outcome = _cumulative_answer(coverage, "google", chunk, country, lambda: _call_with_context(client, ", ".join(chunk), "Google Ads Transparency", lambda: client.run_actor(
1358
+ spec["actor"], {"domains": chunk, "resultType": "ads", "region": country or "ANYWHERE", "maxAdsPerSearch": per_domain}, limit=per_domain * len(chunk),
1359
+ )))
1360
+ if outcome is None:
1361
+ continue
1362
+ rows, retrieved_on = outcome
1363
+ by_domain: dict[str, list[dict[str, Any]]] = {d: [] for d in chunk}
1364
+ for row in rows:
1365
+ destination = registrable_domain(str(row.get("domain") or ""))
1366
+ if destination not in by_domain:
1367
+ continue
1368
+ first = str(row.get("firstShown") or "")[:10] or None
1369
+ last = str(row.get("lastShown") or "")[:10] or None
1370
+ by_domain[destination].append({"first_seen": first, "last_seen": last, "is_active": None, "format": row.get("format"), "raw": row})
1371
+ for domain in chunk:
1372
+ results.append(_presence(domain, "google_ads", "google_transparency_actor", by_domain[domain], market_scope="target" if country else "global", verified_by="destination", today=today, country=country, request_limit=per_domain, raw_count=sum(registrable_domain(str(r.get("domain") or "")) == domain for r in rows), retrieved_on=retrieved_on))
1373
+ return results
1374
+
1375
+
1376
+ def collect_meta(client: OsClient, spec: dict[str, Any], domains: list[str], today: date, country: str, eu_country: str, coverage: list[dict[str, Any]] | None = None) -> list[dict[str, Any]]:
1377
+ per_domain = int(spec.get("ads_per_domain") or 30)
1378
+ results: list[dict[str, Any]] = []
1379
+ for chunk in _chunks(domains, int(spec.get("batch_domains") or 5)):
1380
+ brands = {d: brand_token(d) for d in chunk}
1381
+ outcome = _cumulative_answer(coverage, "meta", chunk, country, lambda: _call_with_context(client, ", ".join(chunk), "Meta Ads Library (page search actor)", lambda: client.run_actor(
1382
+ spec["actor"], {"urls": [{"url": meta_search_url(brands[d], country)} for d in chunk], "count": per_domain}, limit=per_domain * len(chunk),
1383
+ )))
1384
+ if outcome is None:
1385
+ continue
1386
+ rows, retrieved_on = outcome
1387
+ by_domain: dict[str, list[dict[str, Any]]] = {d: [] for d in chunk}
1388
+ for row in rows:
1389
+ page = str(row.get("page_name") or row.get("pageName") or "")
1390
+ snapshot = row.get("snapshot") or {}
1391
+ # The display caption is the second chance when the click link is
1392
+ # a wrapper or short host that hides the destination.
1393
+ link_domain = (usable_destination(snapshot.get("link_url")) or usable_destination(snapshot.get("caption"))) if isinstance(snapshot, dict) else ""
1394
+ # Destination first: a foreign destination excludes the ad even
1395
+ # when the page name resembles the brand (namesake policy of the
1396
+ # Competitor Ads runner); without a usable destination the name
1397
+ # decides.
1398
+ if link_domain:
1399
+ target = next((d for d in chunk if link_domain == d), None)
1400
+ else:
1401
+ target = next((d for d in chunk if _name_matches(page, brands[d])), None)
1402
+ if target is None:
1403
+ continue
1404
+ first = _iso_from_epoch(row.get("start_date") or row.get("startDate"))
1405
+ last = _iso_from_epoch(row.get("end_date") or row.get("endDate"))
1406
+ is_active = row.get("is_active")
1407
+ by_domain[target].append({"first_seen": first, "last_seen": last, "is_active": bool(is_active) if is_active is not None else None, "format": None, "raw": row})
1408
+ for domain in chunk:
1409
+ results.append(_presence(domain, "meta_ads", "meta_library_actor", by_domain[domain], market_scope="target", verified_by="destination_or_page_name", today=today, note=f"page search country {country}; provider delivery dates are global unless native scope is explicit", country=country, request_limit=per_domain, raw_count=len(rows), native_activity_scope="global", retrieved_on=retrieved_on))
1410
+ if not eu_country:
1411
+ return results
1412
+ for domain in domains:
1413
+ brand = brand_token(domain)
1414
+ outcome = _cumulative_answer(coverage, "meta", [domain], eu_country, lambda: _call_with_context(client, domain, "Meta Ads Library (official EU API)", lambda: client.meta_library_search(
1415
+ {"query": brand, "countries": [eu_country], "activeStatus": "ALL", "limit": per_domain},
1416
+ )))
1417
+ if outcome is None:
1418
+ continue
1419
+ rows, retrieved_on = outcome
1420
+ ads: list[dict[str, Any]] = []
1421
+ for row in rows:
1422
+ page = str(row.get("page_name") or "")
1423
+ links = row.get("ad_creative_link_captions") or []
1424
+ if isinstance(links, str):
1425
+ links = [links]
1426
+ # Captions are display text ('Learn more', 'MURAL.CO | ...') as
1427
+ # often as addresses: only address-shaped captions count as a
1428
+ # destination. Destination first: a known foreign destination
1429
+ # excludes the ad whatever the page is called; the name decides
1430
+ # only when the row carries no usable destination.
1431
+ link_domains = {usable_destination(x) for x in links if x} - {""} if isinstance(links, list) else set()
1432
+ if link_domains:
1433
+ if domain not in link_domains:
1434
+ continue
1435
+ elif not _name_matches(page, brand):
1436
+ continue
1437
+ first = str(row.get("ad_delivery_start_time") or "")[:10] or None
1438
+ last = str(row.get("ad_delivery_stop_time") or "")[:10] or None
1439
+ # No stop time = delivering now; a stop time only dates the ad,
1440
+ # the recency window decides whether it still counts as recent.
1441
+ ads.append({"first_seen": first, "last_seen": last, "is_active": True if (first and last is None) else (False if last else None), "format": None, "raw": row})
1442
+ results.append(_presence(domain, "meta_ads", "meta_library_official_eu", ads, market_scope="target" if eu_country == country else "eu_proxy", verified_by="destination_or_page_name", today=today, note=f"EU delivery country {eu_country}; ad-level dates have global scope unless provider supplies country dates", country=eu_country, request_limit=per_domain, raw_count=len(rows), native_activity_scope="global", retrieved_on=retrieved_on))
1443
+ return results
1444
+
1445
+
1446
+ def collect_linkedin(client: OsClient, spec: dict[str, Any], domains: list[str], today: date, company_slugs: dict[str, str] | None, coverage: list[dict[str, Any]]) -> list[dict[str, Any]]:
1447
+ """Advertiser search by LinkedIn company page URL. A keyword query
1448
+ returns ads whose copy mentions the word (observed live: 'coddy' matched
1449
+ 'Caddi'), so the query is the company page, resolved from the
1450
+ competitor's own site when the tag scan saw it and guessed from the
1451
+ brand token otherwise. Rows are attributed by the query they came from
1452
+ and checked against the paying advertiser's name."""
1453
+ per_domain = int(spec.get("ads_per_domain") or 30)
1454
+ slugs = company_slugs or {}
1455
+ results: list[dict[str, Any]] = []
1456
+ for chunk in _chunks(domains, int(spec.get("batch_domains") or 5)):
1457
+ brands = {d: brand_token(d) for d in chunk}
1458
+ queries = {d: f"https://www.linkedin.com/company/{slugs.get(d) or brands[d]}" for d in chunk}
1459
+ outcome = _attempt(client, coverage, "linkedin", chunk, lambda: _call_with_context(client, ", ".join(chunk), "LinkedIn Ad Library", lambda: client.run_actor(
1460
+ spec["actor"], {"queries": [queries[d] for d in chunk], "maxResults": per_domain * len(chunk), "enrich": True}, limit=per_domain * len(chunk),
1461
+ )))
1462
+ if outcome is None:
1463
+ continue
1464
+ rows, retrieved_on = outcome
1465
+ by_domain: dict[str, list[dict[str, Any]]] = {d: [] for d in chunk}
1466
+ for row in rows:
1467
+ advertiser = row.get("advertiser_name") or row.get("advertiserName") or ""
1468
+ query = str(row.get("query") or "").strip().lower()
1469
+ target = next((d for d in chunk if query and query == queries[d].lower()), None)
1470
+ if target is None:
1471
+ target = next((d for d in chunk if _name_matches(advertiser, brands[d])), None)
1472
+ if target is None:
1473
+ continue
1474
+ else:
1475
+ # Query attribution still needs the paying advertiser to BE
1476
+ # the brand (or the company page read from its own site):
1477
+ # a guessed slug can land on a namesake company page.
1478
+ slug_brand = re.sub(r"[^a-z0-9]", "", slugs.get(target, "").lower())
1479
+ if not (_name_matches(advertiser, brands[target]) or (slug_brand and _name_matches(advertiser, slug_brand))):
1480
+ continue
1481
+ first = str(row.get("_first_seen") or row.get("first_seen") or "")[:10] or None
1482
+ last = str(row.get("_last_seen") or row.get("last_seen") or "")[:10] or None
1483
+ by_domain[target].append({"first_seen": first, "last_seen": last, "is_active": None, "format": row.get("creative_format"), "raw": row})
1484
+ for domain in chunk:
1485
+ source_note = f"company page {queries[domain]}" + ("" if slugs.get(domain) else " (slug guessed from the brand token)")
1486
+ results.append(_presence(domain, "linkedin_ads", "linkedin_library_actor", by_domain[domain], market_scope="global", verified_by="company_page_query", today=today, note=f"{source_note}; LinkedIn publishes no delivery dates; sightings are listing history", request_limit=per_domain, raw_count=len(rows), native_date_scope="unknown", retrieved_on=retrieved_on))
1487
+ return results
1488
+
1489
+
1490
+ def collect_tiktok(client: OsClient, spec: dict[str, Any], domains: list[str], today: date, eu_country: str, coverage: list[dict[str, Any]]) -> list[dict[str, Any]]:
1491
+ per_domain = int(spec.get("ads_per_domain") or 30)
1492
+ results: list[dict[str, Any]] = []
1493
+ for domain in domains:
1494
+ brand = brand_token(domain)
1495
+ outcome = _attempt(client, coverage, "tiktok", [domain], lambda: _call_with_context(client, domain, "TikTok Ad Library", lambda: client.run_actor(
1496
+ spec["actor"], {"scrapeMode": "ad_library", "searchQuery": brand, "country": eu_country, "quickSearch": True, "maxResults": per_domain}, limit=per_domain,
1497
+ )))
1498
+ if outcome is None:
1499
+ continue
1500
+ rows, retrieved_on = outcome
1501
+ ads = []
1502
+ for row in rows:
1503
+ if not _name_matches(row.get("advertiserName"), brand):
1504
+ continue
1505
+ first = str(row.get("firstShownDate") or "")[:10] or None
1506
+ last = str(row.get("lastShownDate") or "")[:10] or None
1507
+ status = str(row.get("status") or "").lower()
1508
+ ads.append({"first_seen": first, "last_seen": last, "is_active": True if status == "active" else (False if status == "inactive" else None), "format": row.get("adFormat"), "raw": row})
1509
+ results.append(_presence(domain, "tiktok_ads", "tiktok_library_actor", ads, market_scope="target", verified_by="advertiser_name", today=today, note=f"EU library country {eu_country}; native date scope remains global unless explicitly returned", country=eu_country, request_limit=per_domain, raw_count=len(rows), native_activity_scope="global", retrieved_on=retrieved_on))
1510
+ return results
1511
+
1512
+
1513
+ def collect_reddit(client: OsClient, spec: dict[str, Any], domains: list[str], today: date, coverage: list[dict[str, Any]]) -> list[dict[str, Any]]:
1514
+ per_domain = int(spec.get("ads_per_domain") or 30)
1515
+ results: list[dict[str, Any]] = []
1516
+ for domain in domains:
1517
+ brand = brand_token(domain)
1518
+ outcome = _attempt(client, coverage, "reddit", [domain], lambda: _call_with_context(client, domain, "Reddit ads library", lambda: client.run_actor(
1519
+ spec["actor"], {"query": brand, "maxItems": per_domain}, limit=per_domain,
1520
+ )))
1521
+ if outcome is None:
1522
+ continue
1523
+ rows, retrieved_on = outcome
1524
+ ads = []
1525
+ for row in rows:
1526
+ advertiser = row.get("advertiserName") or row.get("profile_name") or ""
1527
+ # Destination first, name only when the row has no usable
1528
+ # destination (a promoted post links to reddit.com itself).
1529
+ destination = usable_destination(row.get("destinationUrl") or row.get("destination_url") or "")
1530
+ if destination:
1531
+ if destination != domain:
1532
+ continue
1533
+ elif not _name_matches(advertiser, brand):
1534
+ continue
1535
+ created = _iso_from_epoch(row.get("createdAt") or row.get("created_at"))
1536
+ ads.append({"first_seen": created, "last_seen": created, "is_active": None, "format": str(row.get("creativeType") or row.get("type") or "unknown").lower(), "budget": row.get("budgetCategory") or row.get("budget_category"), "placements": row.get("placements"), "raw": row})
1537
+ presence = _presence(domain, "reddit_ads", "reddit_inspiration_actor", ads, market_scope="global", verified_by="destination_or_advertiser_name", today=today, note="inspiration examples; current campaigns are not verifiable; dates are creation dates, not run spans", request_limit=per_domain, raw_count=len(rows), native_date_scope="unknown", retrieved_on=retrieved_on)
1538
+ budgets = sorted({str(a.get("budget")) for a in ads if a.get("budget")})
1539
+ if budgets:
1540
+ presence["note"] += f"; budget tiers {', '.join(budgets)}"
1541
+ results.append(presence)
1542
+ return results
1543
+
1544
+
1545
+ # ── partner signals ─────────────────────────────────────────────────────────
1546
+
1547
+ def _paid_search_row(domain: str, country: str, data: dict[str, Any], retrieved_on: str) -> dict[str, Any]:
1548
+ """Per competitor: SpyFu's modeled Google Ads spend over the complete
1549
+ months checked. `found: false` means SpyFu has no data for the domain,
1550
+ which is not the same as zero spend."""
1551
+ months = [m for m in (data.get("rows") or []) if isinstance(m, dict)]
1552
+ complete = [m for m in months if not m.get("partial_month")]
1553
+ spending = [m for m in complete if float(m.get("monthly_budget_usd") or 0) > 0]
1554
+ latest = complete[0] if complete else None # the gateway returns the newest month first
1555
+ budgets = [float(m["monthly_budget_usd"]) for m in spending]
1556
+
1557
+ def latest_value(key: str) -> Any:
1558
+ # SpyFu models clicks as floats (64759.99999999999); evidence keeps whole numbers.
1559
+ value = latest.get(key) if latest else None
1560
+ return int(round(float(value))) if isinstance(value, (int, float)) and not isinstance(value, bool) else None
1561
+
1562
+ return {
1563
+ "type": "paid_search_stats",
1564
+ "domain": domain,
1565
+ "platform": "google_ads",
1566
+ "source": "spyfu",
1567
+ "country": country,
1568
+ "found": bool(data.get("found")),
1569
+ "months_checked": len(complete),
1570
+ "months_with_spend": len(spending),
1571
+ "spending_now": bool(latest and float(latest.get("monthly_budget_usd") or 0) > 0),
1572
+ "latest_month": f"{int(latest['year']):04d}-{int(latest['month']):02d}" if latest and latest.get("year") and latest.get("month") else None,
1573
+ "latest_monthly_budget_usd": latest_value("monthly_budget_usd"),
1574
+ "latest_monthly_paid_clicks": latest_value("monthly_paid_clicks"),
1575
+ "latest_paid_keywords": latest_value("paid_keywords"),
1576
+ "avg_monthly_budget_usd": round(sum(budgets) / len(budgets)) if budgets else None,
1577
+ "max_monthly_budget_usd": round(max(budgets)) if budgets else None,
1578
+ "retrieved_on": retrieved_on,
1579
+ "market_scope": "target",
1580
+ "tier": "B",
1581
+ }
1582
+
1583
+
1584
+ def collect_spyfu(client: OsClient, domains: list[str], country: str, coverage: list[dict[str, Any]]) -> list[dict[str, Any]]:
1585
+ rows: list[dict[str, Any]] = []
1586
+ for domain in domains:
1587
+ answer = _attempt(client, coverage, "spyfu", [domain], lambda d=domain: client.spyfu_domain_stats(d, country, SPYFU_MONTHS))
1588
+ if answer is None:
1589
+ continue
1590
+ data, retrieved_on = answer
1591
+ row = _paid_search_row(domain, country, data, retrieved_on)
1592
+ rows.append(row)
1593
+ coverage.append({"type": "coverage", "domain": domain, "source": "spyfu", "status": "ok" if row["found"] else "no_data",
1594
+ "detail": "" if row["found"] else "SpyFu has no Google search data for this domain", "tier": "C"})
1595
+ return rows
1596
+
1597
+
1598
+ def _partner_hit(kind: str, domain: str, brand: str, url: str, title: str) -> bool:
1599
+ """Precision over recall: a hit must be the competitor's own program
1600
+ page, a known affiliate network or coupon site, or a page whose own
1601
+ address names the brand together with the signal word. Generic
1602
+ listicles that merely mention the brand do not count."""
1603
+ host = registrable_domain(url)
1604
+ path = urllib.parse.urlparse(url).path.lower()
1605
+ own = host == domain
1606
+ if kind == "affiliate_program":
1607
+ if own:
1608
+ return bool(re.search(r"affiliate|partner|referral|ambassador", path))
1609
+ return host in AFFILIATE_NETWORK_DOMAINS and _mentions(f"{url} {title}", brand)
1610
+ if kind == "coupon_listing":
1611
+ if own:
1612
+ return False
1613
+ return (host in COUPON_DOMAINS and _mentions(f"{url} {title}", brand)) or (
1614
+ bool(re.search(r"coupon|promo|discount", path)) and _mentions(path, brand)
1615
+ )
1616
+ if kind == "sponsorship":
1617
+ if own:
1618
+ return False
1619
+ return bool(re.search(r"sponsor", f"{url} {title}", re.IGNORECASE)) and _mentions(f"{url} {title}", brand)
1620
+ return False
1621
+
1622
+
1623
+ def collect_partners(client: OsClient, domain: str, country: str, coverage: list[dict[str, Any]] | None = None) -> list[dict[str, Any]]:
1624
+ brand = brand_token(domain)
1625
+ rows: list[dict[str, Any]] = []
1626
+ for kind, template in PARTNER_KINDS:
1627
+ query = template.format(brand=brand)
1628
+ try:
1629
+ response, _ = client.brave(query, BRAVE_COUNT, country.lower())
1630
+ except BudgetExhausted as exc:
1631
+ if "cumulative" not in str(exc):
1632
+ raise
1633
+ if coverage is not None:
1634
+ coverage.append({"type": "coverage", "domain": domain, "source": "partners", "country": country, "status": "budget_deferred", "detail": f"{kind}: {exc}", "tier": "C"})
1635
+ rows.append({"type": "partner_signal", "domain": domain, "country": country, "kind": kind, "query": query, "hits": None, "urls": [], "source": "brave", "status": "budget_deferred", "tier": "C"})
1636
+ continue
1637
+ except OsUnreachable:
1638
+ raise
1639
+ except ChannelFootprintError as exc:
1640
+ rows.append({"type": "partner_signal", "domain": domain, "kind": kind, "query": query, "hits": None, "urls": [], "unavailable": _provider_detail(exc)[:200], "source": "brave", "tier": "C"})
1641
+ continue
1642
+ hits = 0
1643
+ urls: list[str] = []
1644
+ for item in response.get("results") or []:
1645
+ if not isinstance(item, dict):
1646
+ continue
1647
+ url = str(item.get("url") or "")
1648
+ title = str(item.get("title") or "")
1649
+ if _partner_hit(kind, domain, brand, url, title):
1650
+ hits += 1
1651
+ if len(urls) < PARTNER_URLS_PER_KIND:
1652
+ urls.append(url)
1653
+ rows.append({"type": "partner_signal", "domain": domain, "kind": kind, "query": query, "hits": hits, "urls": urls, "source": "brave", "tier": "B" if hits else "C"})
1654
+ return rows
1655
+
1656
+
1657
+ # ── aggregation ─────────────────────────────────────────────────────────────
1658
+
1659
+ def observed_channels(presences=(), scans=(), traffic=(), paid_search=()) -> tuple[str, ...]:
1660
+ names = set(CHANNELS)
1661
+ names.update(p.get("platform") for p in presences if p.get("platform"))
1662
+ names.update(channel for scan in scans for channel in scan.get("channels_detected", []))
1663
+ names.update(row.get("paid_channel") for row in traffic if row.get("paid_channel"))
1664
+ names.update(row.get("platform") for row in paid_search if row.get("platform"))
1665
+ return tuple(sorted(name for name in names if isinstance(name, str) and re.fullmatch(r"[a-z][a-z0-9_]+", name)))
1666
+
1667
+
1668
+ def channel_label(channel: str) -> str:
1669
+ return CHANNEL_LABELS.get(channel, channel)
1670
+
1671
+
1672
+ def rank_channels(domains: list[str], presences: list[dict[str, Any]], scans: list[dict[str, Any]], traffic: list[dict[str, Any]], partners: list[dict[str, Any]], paid_search: list[dict[str, Any]] = (), competitor_context: dict[str, dict[str, Any]] | None = None) -> list[dict[str, Any]]:
1673
+ """Describe direct-competitor observations; do not rank channel merit.
1674
+
1675
+ Evidence kinds are preserved because tags, listings and modeled spend are
1676
+ different observations. Counts never imply acquisition success.
1677
+ """
1678
+ context = competitor_context or {}
1679
+ direct = {d for d in domains if context.get(d, {}).get("competitor_tier") == "direct"}
1680
+ rows = []
1681
+ for channel in observed_channels(presences, scans, traffic, paid_search):
1682
+ evidence: dict[str, set[str]] = {}
1683
+ active_ids, proven_domains, durations = set(), set(), []
1684
+ for p in presences:
1685
+ if p.get("domain") not in direct or p.get("platform") != channel:
1686
+ continue
1687
+ if p.get("present") and p.get("relevance") == "relevant":
1688
+ evidence.setdefault(p["domain"], set()).add("library" if p.get("active_ads") else "listing")
1689
+ active_ids.update(p.get("current_ad_ids") or [])
1690
+ if p.get("current_presence"):
1691
+ proven_domains.add(p["domain"])
1692
+ if p.get("oldest_days") is not None:
1693
+ durations.append(p["oldest_days"])
1694
+ for scan in scans:
1695
+ if scan["domain"] in direct and channel in scan.get("channels_detected", []):
1696
+ evidence.setdefault(scan["domain"], set()).add("pixel")
1697
+ samples = [t for t in traffic if t["domain"] in direct and t.get("paid_channel") == channel and t.get("share_pct") is not None]
1698
+ for row in samples:
1699
+ if float(row.get("share_pct_raw", row["share_pct"])) > PRESENT_SHARE_PCT:
1700
+ evidence.setdefault(row["domain"], set()).add("modeled_traffic")
1701
+ if channel == "affiliate":
1702
+ for row in partners:
1703
+ if row["domain"] in direct and row.get("kind") in {"affiliate_program", "coupon_listing"} and row.get("hits", 0):
1704
+ evidence.setdefault(row["domain"], set()).add("partner_signal")
1705
+ budgets = [r for r in paid_search if r["domain"] in direct and r.get("platform") == channel and r.get("spending_now") and r.get("latest_monthly_budget_usd") is not None]
1706
+ for row in budgets:
1707
+ evidence.setdefault(row["domain"], set()).add("modeled_spend")
1708
+ count = len(evidence)
1709
+ rows.append({"type": "channel_observation", "channel": channel, "label": channel_label(channel),
1710
+ "competitor_tier": "direct", "competitors_present": count, "competitors_total": len(direct), "present_domains": sorted(evidence),
1711
+ "evidence_kinds": {d: sorted(kinds) for d, kinds in sorted(evidence.items())}, "active_ads_total": len(active_ids),
1712
+ "active_competitors": len(proven_domains), "max_oldest_days": max(durations) if durations else None,
1713
+ "avg_share_pct": round(sum(float(r.get("share_pct_raw", r["share_pct"])) for r in samples) / len(samples), 2) if samples else None,
1714
+ "share_sample_n": len(samples), "share_sample_domains": sorted({r["domain"] for r in samples}), "share_scope": "modeled_domain_global",
1715
+ "est_monthly_budget_usd": sum(float(r["latest_monthly_budget_usd"]) for r in budgets) if budgets else None,
1716
+ "why": f"{count} of {len(direct)} direct competitor(s) with a positive observation; evidence kinds remain separate", "tier": "B" if count else "C",
1717
+ "campaign_data_status": "activity_observed" if proven_domains else "activity_unknown"})
1718
+ rows.sort(key=lambda r: r["channel"])
1719
+ return rows
1720
+
1721
+
1722
+ # ── rendering ───────────────────────────────────────────────────────────────
1723
+
1724
+ def _cell(value: Any, cap: int = 120) -> str:
1725
+ text = " ".join(str(value if value is not None else "—").split()).replace("|", "\\|")
1726
+ return text if len(text) <= cap else text[: cap - 1] + "…"
1727
+
1728
+
1729
+ # ── orchestration ───────────────────────────────────────────────────────────
1730
+
1731
+ EU_COUNTRIES = frozenset("AT BE BG HR CY CZ DE DK EE ES FI FR GR HU IE IT LT LU LV MT NL PL PT RO SE SI SK".split())
1732
+
1733
+
1734
+ def _effective_policy(config: dict[str, Any]) -> tuple[dict[str, Any], str]:
1735
+ module = _load_script("research-policy.py", "pmr_channel_policy")
1736
+ taxonomy_path = Path(__file__).resolve().parent.parent / "references/config/creative-taxonomy.v1.json"
1737
+ return module.policy_from_taxonomy(json.loads(taxonomy_path.read_text()))
1738
+
1739
+
1740
+ def _coverage_grid(inputs: RunInputs, presences: list[dict[str, Any]], coverage: list[dict[str, Any]], raw: dict[str, Any] | None = None) -> None:
1741
+ for domain in inputs.domains:
1742
+ for source in inputs.sources:
1743
+ for country in inputs.target_countries:
1744
+ existing = [r for r in coverage if r.get("domain") == domain and r.get("source") == source and r.get("country") in {country, None}]
1745
+ if existing:
1746
+ continue
1747
+ matching = [p for p in presences if p["domain"] == domain and p["source"].startswith({"google": "google_", "meta": "meta_", "linkedin": "linkedin_", "reddit": "reddit_", "tiktok": "tiktok_"}.get(source, source)) and p.get("country") in {country, None}]
1748
+ global_only = source in {"pixels", "history", "traffic", "linkedin", "reddit"}
1749
+ unsupported = source == "tiktok" and country not in EU_COUNTRIES
1750
+ measurements = [r for r in (raw or {}).get({"history": "history", "partners": "partners", "spyfu": "paid_search", "traffic": "traffic", "pixels": "scans"}.get(source, ""), []) if r.get("domain") == domain and r.get("country") in {country, None}]
1751
+ if unsupported:
1752
+ status = "unsupported_market"
1753
+ elif source in {"linkedin", "reddit"} and matching:
1754
+ status = "not_verifiable"
1755
+ elif matching:
1756
+ status = "ok" if any(p.get("ads") for p in matching) else "no_matches_in_sample"
1757
+ elif measurements:
1758
+ if source == "partners":
1759
+ status = "ok" if any(r.get("hits", 0) for r in measurements) else "no_matches_in_sample"
1760
+ else:
1761
+ status = next((r.get("status") for r in measurements if r.get("status")), "ok")
1762
+ else:
1763
+ status = "not_checked"
1764
+ coverage.append({"type": "coverage", "domain": domain, "source": source, "country": country,
1765
+ "market_scope": "global" if global_only else "target", "status": status,
1766
+ "detail": "source cannot verify current campaigns or native delivery duration" if source in {"linkedin", "reddit"} else ("source does not support this target country" if unsupported else ""), "tier": "C"})
1767
+
1768
+
1769
+ def finalize_pipeline(raw: dict[str, Any], inputs: RunInputs, config: dict[str, Any], today: date, stale_decisions: str = "raise") -> dict[str, Any]:
1770
+ model = _model()
1771
+ policy, policy_hash = _effective_policy(config)
1772
+ raw_presences = raw["presences"]
1773
+ corpus = model.build_corpus(raw_presences, inputs.product_context_sha256, policy)
1774
+ for ad in corpus:
1775
+ ad["competitor_tier"] = inputs.competitor_context.get(ad["domain"], {}).get("competitor_tier", "unknown")
1776
+ ad["activity_status"] = "unknown" # use native observation status for decisions
1777
+ ad["countries"] = sorted({o["country"] for o in ad["observations"] if o.get("country")})
1778
+ try:
1779
+ pending = model.apply_decisions(corpus, inputs.relevance_decisions, inputs.product_context_sha256, stale=stale_decisions)
1780
+ except ValueError as exc:
1781
+ raise ChannelFootprintError(str(exc)) from exc
1782
+ presences = model.summarize_presences(raw_presences, corpus, inputs.competitor_context, policy, inputs.product_context_sha256)
1783
+ # A native placement creates a separately scoped observation. A video
1784
+ # format alone stays Google and never asserts YouTube delivery.
1785
+ for placement, channel in (("youtube", "youtube_ads"), ("display", "display_native")):
1786
+ sub_raw = []
1787
+ for presence in raw_presences:
1788
+ if presence["platform"] != "google_ads":
1789
+ continue
1790
+ ids = {a["ad_id"] for a in corpus if any(o.get("placement") == placement and o.get("country") == presence.get("country") for o in a["observations"])}
1791
+ selected_ids = [x for x in presence.get("ad_ids", []) if x in ids]
1792
+ if selected_ids:
1793
+ sub_raw.append({**presence, "platform": channel, "ad_ids": selected_ids})
1794
+ if sub_raw:
1795
+ presences.extend(model.summarize_presences(sub_raw, corpus, inputs.competitor_context, policy, inputs.product_context_sha256, placement_filter=placement))
1796
+ for row in raw["scans"] + raw["traffic"] + raw["paid_search"] + raw["partners"] + raw["history"]:
1797
+ row["competitor_tier"] = inputs.competitor_context.get(row["domain"], {}).get("competitor_tier", "unknown")
1798
+ row.setdefault("market_scope", "global")
1799
+ country_rows = model.country_rows(presences, raw["scans"], inputs.target_countries, inputs.competitor_context, policy, observed_channels(presences, raw["scans"], raw["traffic"], raw["paid_search"]))
1800
+ ranks = rank_channels(inputs.domains, presences, raw["scans"], raw["traffic"], raw["partners"], raw["paid_search"], inputs.competitor_context)
1801
+ coverage = [dict(row) for row in raw["coverage"]]
1802
+ _coverage_grid(inputs, raw_presences, coverage, raw)
1803
+ totals = raw["totals"]
1804
+ retrieved_on = totals["first_retrieved_on"] if totals["first_retrieved_on"] == totals["last_retrieved_on"] else f"{totals['first_retrieved_on']}..{totals['last_retrieved_on']}"
1805
+ evidence = {"schema_version": SCHEMA_VERSION, "section_id": SECTION_ID, "generated_by": Path(__file__).name, "retrieved_on": retrieved_on,
1806
+ "inputs": {"domains": inputs.domains, "domain_sources": inputs.domain_sources, "competitor_context": inputs.competitor_context,
1807
+ "product_domain": inputs.product_domain, "product_context_sha256": inputs.product_context_sha256,
1808
+ "sources": inputs.sources, "country": inputs.country, "eu_country": inputs.eu_country, "target_countries": inputs.target_countries,
1809
+ "present_share_pct": PRESENT_SHARE_PCT, "effective_policy": policy, "policy_sha256": policy_hash},
1810
+ "rows": ranks + country_rows + presences + corpus + raw["paid_search"] + raw["scans"] + raw["traffic"] + raw["partners"] + raw["history"] + coverage,
1811
+ "totals": {k: totals[k] for k in ("actor_runs", "meta_searches", "search_calls", "spyfu_calls", "fetches", "cdx_calls", "provider_calls", "rejected_actor_runs", "rejected_spyfu_calls", "observations")}}
1812
+ section = render_section_v2(inputs, presences, corpus, ranks, country_rows, {**raw, "coverage": coverage}, retrieved_on)
1813
+ return {"evidence": evidence, "section_markdown": section, "totals": totals, "attempt": raw["attempt"], "raw": raw,
1814
+ "pending_relevance": pending, "corpus": corpus, "collection_complete": raw.get("collection_complete", True)}
1815
+
1816
+
1817
+ def run_pipeline(client: OsClient, inputs: RunInputs, config: dict[str, Any], today: date) -> dict[str, Any]:
1818
+ actors = config["actors"]
1819
+ policy, _ = _effective_policy(config)
1820
+ inputs.effective_policy = policy
1821
+ inputs.target_countries = _model().countries(inputs.target_countries or [inputs.country])
1822
+ inputs.country = inputs.target_countries[0]
1823
+ raw = {"scans": [], "history": [], "traffic": [], "presences": [], "partners": [], "coverage": [], "paid_search": []}
1824
+ primary = list(inputs.domains) # All explicitly scoped competitors; roles describe comparability, not eligibility.
1825
+
1826
+ def collect(domains: list[str], deficits: set[tuple[str, str]] | None = None) -> None:
1827
+ if not domains:
1828
+ return
1829
+ def call(source: str, country: str | None, fn: Callable[[], Any], target: str):
1830
+ coverage_before = len(raw["coverage"])
1831
+ try:
1832
+ value = fn()
1833
+ except BudgetExhausted as exc:
1834
+ if "cumulative" not in str(exc):
1835
+ raise
1836
+ raw["coverage"].extend({"type": "coverage", "domain": d, "source": source, "country": country, "status": "budget_deferred", "detail": str(exc), "tier": "C"} for d in domains)
1837
+ return
1838
+ for row in raw["coverage"][coverage_before:]:
1839
+ row.setdefault("country", country)
1840
+ if value:
1841
+ if country and target in {"paid_search", "partners"}:
1842
+ for row in value:
1843
+ row.setdefault("country", country)
1844
+ raw[target].extend(value)
1845
+ if "pixels" in inputs.sources:
1846
+ call("pixels", None, lambda: [scan_site(client, d) for d in domains], "scans")
1847
+ opaque = [s["domain"] for s in raw["scans"] if s["domain"] in domains and (s["fetch_status"] != "ok" or not s.get("tags_detected"))]
1848
+ if opaque and inputs.pixel_actor_fallback:
1849
+ for chunk in _chunks(opaque, int(actors["pixel_detector"].get("batch_domains") or 5)):
1850
+ try:
1851
+ replacements = _attempt(client, raw["coverage"], "pixels_actor", chunk, lambda: scan_sites_with_actor(client, actors["pixel_detector"]["actor"], chunk))
1852
+ except BudgetExhausted as exc:
1853
+ if "cumulative" not in str(exc):
1854
+ raise
1855
+ replacements = None
1856
+ raw["coverage"].extend({"type": "coverage", "domain": d, "source": "pixels_actor", "status": "budget_deferred", "detail": str(exc), "tier": "C"} for d in chunk)
1857
+ if replacements:
1858
+ for scan in raw["scans"]:
1859
+ replacement = replacements.get(scan["domain"])
1860
+ if replacement is not None:
1861
+ replacement["linkedin_company"] = scan.get("linkedin_company")
1862
+ replacement["gtm_containers"] = scan.get("gtm_containers") or []
1863
+ raw["scans"] = [replacements.get(s["domain"], s) for s in raw["scans"]]
1864
+ for scan in raw["scans"]:
1865
+ if scan["domain"] in domains:
1866
+ raw["coverage"].append({"type": "coverage", "domain": scan["domain"], "source": "pixels", "market_scope": "global", "status": "ok" if scan["fetch_status"] == "ok" else scan["fetch_status"], "detail": "domain tracking signal; campaign details unavailable" if scan.get("channels_detected") else "", "tier": "C"})
1867
+ if "history" in inputs.sources:
1868
+ call("history", None, lambda: [collect_history(client, d, today) for d in domains], "history")
1869
+ if "traffic" in inputs.sources:
1870
+ call("traffic", None, lambda: collect_traffic(client, actors["traffic_mix"], domains, raw["coverage"]), "traffic")
1871
+ for country in inputs.target_countries:
1872
+ if "google" in inputs.sources and (deficits is None or ("google_ads", country) in deficits):
1873
+ call("google", country, lambda: collect_google(client, actors["google_ads"], domains, today, country, raw["coverage"]), "presences")
1874
+ if "spyfu" in inputs.sources and (deficits is None or ("google_ads", country) in deficits):
1875
+ call("spyfu", country, lambda: collect_spyfu(client, domains, country, raw["coverage"]), "paid_search")
1876
+ if "meta" in inputs.sources and (deficits is None or ("meta_ads", country) in deficits):
1877
+ call("meta", country, lambda: collect_meta(client, actors["meta_ads"], domains, today, country, country if country in EU_COUNTRIES else "", raw["coverage"]), "presences")
1878
+ if "partners" in inputs.sources and (deficits is None or ("affiliate", country) in deficits):
1879
+ call("partners", country, lambda: [r for d in domains for r in collect_partners(client, d, country, raw["coverage"])], "partners")
1880
+ if "tiktok" in inputs.sources and country in EU_COUNTRIES and (deficits is None or ("tiktok_ads", country) in deficits):
1881
+ call("tiktok", country, lambda: collect_tiktok(client, actors["tiktok_ads"], domains, today, country, raw["coverage"]), "presences")
1882
+ if "linkedin" in inputs.sources and (deficits is None or any(ch == "linkedin_ads" for ch, _ in deficits)):
1883
+ slugs = {s["domain"]: s["linkedin_company"] for s in raw["scans"] if s.get("linkedin_company")}
1884
+ call("linkedin", None, lambda: collect_linkedin(client, actors["linkedin_ads"], domains, today, slugs, raw["coverage"]), "presences")
1885
+ if "reddit" in inputs.sources and (deficits is None or any(ch == "reddit_ads" for ch, _ in deficits)):
1886
+ call("reddit", None, lambda: collect_reddit(client, actors["reddit_ads"], domains, today, raw["coverage"]), "presences")
1887
+ collect(primary)
1888
+ raw["totals"] = client.ledger_totals()
1889
+ raw["attempt"] = {"actor_runs": client.actor_runs, "meta_searches": client.meta_searches, "search_calls": client.search_calls, "spyfu_calls": client.spyfu_calls, "fetches": client.fetches, "cdx_calls": client.cdx_calls, "relevance_calls": client.relevance_calls, "cache_hits": client.cache_hits}
1890
+ # The base sample lacks fields that the expansion may fill in an ad
1891
+ # decided after it, so this gate takes decisions by ad_id; the passes on
1892
+ # the expanded corpus send such a changed ad back to review instead.
1893
+ preliminary = finalize_pipeline(raw, inputs, config, today, stale_decisions="accept")
1894
+ # A reviewed direct sample without comparable ads may be expanded once.
1895
+ # The enlarged payload is cached; repeated collect never adds another step.
1896
+ # A collected expansion is replayed whatever a later review says: an ad it
1897
+ # enriched and the founder then found relevant keeps its expanded sample.
1898
+ reviewed = not preliminary["pending_relevance"]
1899
+ expanded_before = {(e.get("domain"), e.get("source"), e.get("country")) for e in inputs.prior_expansions}
1900
+ raw["expansions"] = []
1901
+ for presence in list(raw["presences"]):
1902
+ if not presence.get("sampling", {}).get("limit_reached"):
1903
+ continue
1904
+ if (presence["domain"], presence["source"], presence.get("country")) not in expanded_before:
1905
+ if not reviewed:
1906
+ continue
1907
+ comparable = next((r for r in preliminary["evidence"]["rows"] if r["type"] == "library_presence" and r["domain"] == presence["domain"] and r["platform"] == presence["platform"] and r.get("country") == presence.get("country")), {})
1908
+ if comparable.get("relevant_ads"):
1909
+ continue
1910
+ source_key = {"google_transparency_actor": "google_ads", "meta_library_actor": "meta_ads", "linkedin_library_actor": "linkedin_ads", "tiktok_library_actor": "tiktok_ads"}.get(presence["source"])
1911
+ if not source_key:
1912
+ continue
1913
+ expanded_spec = dict(actors[source_key])
1914
+ base = int(expanded_spec.get("ads_per_domain") or 30)
1915
+ maximum = int(expanded_spec.get("max_ads_per_domain") or base * 2)
1916
+ if maximum <= base:
1917
+ continue
1918
+ expanded_spec["ads_per_domain"] = maximum
1919
+ domain, country = presence["domain"], presence.get("country")
1920
+ try:
1921
+ if source_key == "google_ads":
1922
+ added = collect_google(client, expanded_spec, [domain], today, country, raw["coverage"])
1923
+ elif source_key == "meta_ads":
1924
+ added = collect_meta(client, expanded_spec, [domain], today, country, country if country in EU_COUNTRIES else "", raw["coverage"])
1925
+ elif source_key == "linkedin_ads":
1926
+ slugs = {s["domain"]: s["linkedin_company"] for s in raw["scans"] if s.get("linkedin_company")}
1927
+ added = collect_linkedin(client, expanded_spec, [domain], today, slugs, raw["coverage"])
1928
+ else:
1929
+ added = collect_tiktok(client, expanded_spec, [domain], today, country, raw["coverage"])
1930
+ raw["presences"].extend(added)
1931
+ if added:
1932
+ raw["expansions"].append({"domain": domain, "source": presence["source"], "country": country})
1933
+ except BudgetExhausted as exc:
1934
+ if "cumulative" not in str(exc):
1935
+ raise
1936
+ raw["coverage"].append({"type": "coverage", "domain": domain, "source": source_key, "country": country, "status": "budget_deferred", "detail": str(exc), "tier": "C"})
1937
+ if reviewed or raw["expansions"]:
1938
+ preliminary = finalize_pipeline(raw, inputs, config, today, stale_decisions="pending")
1939
+ raw["collection_complete"] = True
1940
+ raw["totals"] = client.ledger_totals()
1941
+ raw["attempt"] = {"actor_runs": client.actor_runs, "meta_searches": client.meta_searches, "search_calls": client.search_calls, "spyfu_calls": client.spyfu_calls, "fetches": client.fetches, "cdx_calls": client.cdx_calls, "relevance_calls": client.relevance_calls, "cache_hits": client.cache_hits}
1942
+ return finalize_pipeline(raw, inputs, config, today, stale_decisions="pending")
1943
+
1944
+
1945
+ def render_section_v2(inputs, presences, corpus, ranks, country_rows, raw, retrieved_on):
1946
+ lines = [f"Observed: {retrieved_on}. Target countries: {', '.join(inputs.target_countries)}.", "",
1947
+ "Counts describe the bounded source sample, not channel priority or success. Tracking tags, modeled traffic and ad-library observations remain distinct; a tag does not describe campaigns or country delivery.", ""]
1948
+ for role, label in (("direct", "Direct competitors"), ("indirect", "Indirect competitors"), ("unknown", "Unclassified domains")):
1949
+ domains = [d for d in inputs.domains if inputs.competitor_context.get(d, {}).get("competitor_tier", "unknown") == role]
1950
+ if not domains:
1951
+ continue
1952
+ lines += [f"### {label}: channel footprint matrix", "", "| Competitor | Channel | Country / scope | Observations | Campaign data |", "| --- | --- | --- | --- | --- |"]
1953
+ for domain in domains:
1954
+ for channel in (r["channel"] for r in ranks):
1955
+ pixels = any(s["domain"] == domain and channel in s.get("channels_detected", []) for s in raw["scans"])
1956
+ rows = [r for r in presences if r["domain"] == domain and r["platform"] == channel]
1957
+ if not rows and not pixels:
1958
+ continue
1959
+ if not rows:
1960
+ lines.append(f"| {domain} | {channel_label(channel)} | global / unknown | pixel found; campaign data unavailable | needs_campaign_data |")
1961
+ for row in rows:
1962
+ parts = ["pixel found (global / unknown)"] if pixels else []
1963
+ parts += [f"comparable ads {row['relevant_ads']}; active {row['active_ads']} ({', '.join(row['activity_scopes']) or 'unknown'} scope); country current {len(row['current_ad_ids'])}; stopped {row['stopped_ads']}; undated {row['unknown_activity_ads']}"]
1964
+ if row["unknown_relevance_ads"]:
1965
+ parts.append(f"relevance unknown {row['unknown_relevance_ads']}")
1966
+ parts.append(row["status"])
1967
+ lines.append(f"| {domain} | {channel_label(channel)} | {_cell(row.get('country') or row['market_scope'])} | {_cell('; '.join(parts), 300)} | {'activity_observed' if row['current_presence'] else 'activity_unknown'} |")
1968
+ lines += ["", "### Channel observations (direct competitors)", "", "| Channel | Direct positive observations | Active competitors | Activity coverage |", "| --- | --- | --- | --- |"]
1969
+ for row in ranks:
1970
+ lines.append(f"| {row['label']} | {row['competitors_present']}/{row['competitors_total']} | {row['active_competitors']} | {row['campaign_data_status']} |")
1971
+ lines += ["", "### Country observations", "", "| Channel | Country | Direct observed | Indirect observed | Active domains | Coverage |", "| --- | --- | --- | --- | --- | --- |"]
1972
+ for row in country_rows:
1973
+ lines.append(f"| {channel_label(row['channel'])} | {row['country']} | {len(row['observed_domains_by_role']['direct'])} | {len(row['observed_domains_by_role']['indirect'])} | {len(row['active_domains'])} | {row['status']} |")
1974
+ lines += ["", "### Source samples", "", "| Competitor | Source | Country / scope | Received before relevance | Comparable | Limit / completeness | Max observed delivery duration |", "| --- | --- | --- | --- | --- | --- | --- |"]
1975
+ for row in presences:
1976
+ samples = row.get("sampling", [])
1977
+ cap = any(x.get("limit_reached") for x in samples)
1978
+ received = max((x.get("raw_received", 0) for x in samples), default=0)
1979
+ lines.append(f"| {row['domain']} | {_cell(', '.join(row['sources']))} | {_cell(row.get('country') or row['market_scope'])} | {'≥' if cap else ''}{received} | {row['relevant_ads']} found | {'limit reached; total unknown' if cap else 'completeness unknown'} | {_cell(row['oldest_days'])} days ({row['duration_scope']}) |")
1980
+ lines += ["", "### Coverage", "", "| Competitor | Source | Country / scope | Status | Detail |", "| --- | --- | --- | --- | --- |"]
1981
+ for row in raw["coverage"]:
1982
+ lines.append(f"| {row['domain']} | {row['source']} | {_cell(row.get('country') or row.get('market_scope') or 'global / unknown')} | {row['status']} | {_cell(row.get('detail', ''), 300)} |")
1983
+ lines += ["", "### Source measurements", "", "| Competitor | Source / type | Market | Measurement |", "| --- | --- | --- | --- |"]
1984
+ for row in raw["traffic"] + raw["paid_search"] + raw["partners"] + raw["history"]:
1985
+ measurement = {k: v for k, v in row.items() if k not in {"type", "domain", "competitor_tier", "tier", "market_scope"}}
1986
+ lines.append(f"| {row['domain']} | {_cell(row.get('source') or row['type'])} | {_cell(row.get('country') or row.get('market_scope') or 'unknown')} | {_cell(json.dumps(measurement, ensure_ascii=False), 500)} |")
1987
+ lines += ["", "Legend: pixel found; active native status; stopped; undated listing; no matches in sample; not_checked; not_verifiable; unsupported_market; budget_deferred. Repeated listing observations are presence only. Empty samples never assert channel absence.", ""]
1988
+ return "\n".join(lines)
1989
+
1990
+
1991
+ def _collection_matches(stored: dict[str, Any], context_hash: str, policy_hash: str, sources: list[str], config: dict[str, Any]) -> bool:
1992
+ model = _model()
1993
+ return (stored.get("input_context_sha256") == context_hash and stored.get("policy_sha256") == policy_hash and stored.get("sources") == sources
1994
+ and stored.get("sources_config_sha256") == model.digest(config) and stored.get("raw_sha256") == model.digest(stored.get("raw")))
1995
+
1996
+
1997
+ def _write_atomic(path: Path, content: str) -> None:
1998
+ path.parent.mkdir(parents=True, exist_ok=True)
1999
+ tmp = path.with_suffix(path.suffix + ".tmp")
2000
+ tmp.write_text(content, encoding="utf-8")
2001
+ tmp.replace(path)
2002
+
2003
+
2004
+ def _arg(args: argparse.Namespace, name: str, default: Any) -> Any:
2005
+ value = getattr(args, name, None)
2006
+ return default if value is None else value
2007
+
2008
+
2009
+ def run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
2010
+ workspace = workspace.resolve()
2011
+ io = _load_script("research-artifact-io.py", "pmr_artifact_io")
2012
+ with io.section_attempt(workspace, SECTION_ID, context_file=getattr(args, "context_file", None)):
2013
+ return _run(workspace, args)
2014
+
2015
+
2016
+ def _run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
2017
+ workspace = workspace.resolve()
2018
+ runtime = workspace / RUNTIME_REL
2019
+ cache_dir = runtime / "cache"
2020
+ runtime.mkdir(parents=True, exist_ok=True)
2021
+
2022
+ config = load_sources_config(Path(_arg(args, "sources_config", "")) if _arg(args, "sources_config", "") else None)
2023
+ sources = parse_sources(str(_arg(args, "sources", "") or ""))
2024
+ stage = str(_arg(args, "stage", "all"))
2025
+ if stage not in {"all", "collect", "finalize"}:
2026
+ raise ChannelFootprintError("stage must be collect, finalize or all")
2027
+ policy, policy_hash = _effective_policy(config)
2028
+ max_competitors = max(1, int(_arg(args, "max_competitors", MAX_COMPETITORS)))
2029
+ product_domain = registrable_domain(str(_arg(args, "product_domain", "") or ""))
2030
+ explicit = [d for d in parse_domains(_arg(args, "domains", ""), MAX_EXPLICIT_DOMAINS) if d != product_domain]
2031
+ tiered = domains_from_competitor_set(workspace, MAX_COMPETITORS)
2032
+ rejected = domains_excluded_by_competitor_set(workspace) - set(explicit)
2033
+ discovered = domains_from_keyword_evidence(workspace, MAX_COMPETITORS, exclude=rejected)
2034
+ domains, domain_sources = [], {}
2035
+ for source, candidates in (("explicit", explicit), ("competitor_set", tiered), ("keyword_economics", discovered)):
2036
+ for domain in candidates:
2037
+ if len(domains) >= max_competitors:
2038
+ break
2039
+ if domain and domain != product_domain and domain not in domains:
2040
+ domains.append(domain)
2041
+ domain_sources[domain] = source
2042
+ if not domains:
2043
+ raise ChannelFootprintError("no competitor domains available: pass --domains, or complete the Competitor Set (or Keyword Economics) section first")
2044
+ try:
2045
+ provenance = json.loads((runtime / "competitor-set-inputs.json").read_text())
2046
+ except (OSError, json.JSONDecodeError):
2047
+ provenance = {}
2048
+ competitor_context = {r["domain"]: {**r, "source_sha256": provenance.get("source_sha256"), "source_path": _artifact_path("egbe-docs/research/market/evidence/competitors.json")} for r in provenance.get("inputs", []) if r.get("domain") in domains}
2049
+ for domain in domains:
2050
+ competitor_context.setdefault(domain, {"domain": domain, "competitor_tier": "unknown", "reason": "no approved Competitor Set role", "source_path": None, "source_sha256": None})
2051
+ intake_path = workspace / _artifact_path("egbe-docs/research/market/intake.json")
2052
+ intake = typed_intake(workspace)
2053
+ intake_hash = IO.sha256(intake_path) if intake_path.is_file() else None
2054
+ research_context = IO.context_binding(workspace, SECTION_ID)
2055
+ context = IO.context_text(workspace, SECTION_ID)
2056
+ geography = (_arg(args, "countries", "") or _arg(args, "country", "")
2057
+ or intake.get("markets") or [DEFAULT_COUNTRY])
2058
+ target_countries = _model().countries(geography)
2059
+ context_hash = _model().digest({"context_sha256": research_context["sha256"], "intake_sha256": intake_hash, "competitor_context": competitor_context, "product_domain": product_domain, "target_countries": target_countries})
2060
+ relevance_inputs = relevance_context(workspace, args) if _arg(args, "relevance_by_model", False) else None
2061
+ decision_path = Path(_arg(args, "relevance_decisions", "") or runtime / "relevance-decisions.json")
2062
+ try:
2063
+ decisions = json.loads(decision_path.read_text()) if decision_path.is_file() else {}
2064
+ except (OSError, json.JSONDecodeError) as exc:
2065
+ raise ChannelFootprintError(f"invalid relevance decisions: {exc}") from exc
2066
+ if decisions and decisions.get("product_context_sha256") != context_hash:
2067
+ if stage == "finalize":
2068
+ raise ChannelFootprintError("relevance decisions belong to a different product/intake context; collect the new candidates")
2069
+ decisions = {} # retain the old input on disk; expose new context-bound candidates
2070
+ inputs = RunInputs(domains=domains, domain_sources=domain_sources, product_domain=product_domain, sources=sources,
2071
+ country=target_countries[0], eu_country=str(_arg(args, "eu_country", "") or ""), pixel_actor_fallback=bool(_arg(args, "pixel_actor_fallback", False)),
2072
+ target_countries=target_countries, competitor_context=competitor_context, product_context_sha256=context_hash,
2073
+ relevance_decisions=decisions, effective_policy=policy, stage=stage)
2074
+ plan = collection_plan([(domain, competitor_context[domain]["competitor_tier"]) for domain in domains], sources, target_countries,
2075
+ pixel_actor_fallback=inputs.pixel_actor_fallback)
2076
+ budget = paid_start_budget(plan, config)
2077
+ max_total = int(_arg(args, "max_total_paid_starts", budget["cumulative_limit"]))
2078
+ if max_total < 0:
2079
+ raise ChannelFootprintError("total paid-start budget must be nonnegative")
2080
+ plan_doc = {"schema_version": 1, "input_context_sha256": context_hash, "policy_sha256": policy_hash,
2081
+ "base_paid_starts_upper_bound": budget["base"], "bounded_paid_starts_upper_bound": budget["bounded"], "retry_reserve_factor": budget["retry_reserve_factor"], "cumulative_limit": max_total, "money_estimate": None, "checks": plan}
2082
+ _write_atomic(runtime / "collection-plan.json", json.dumps(plan_doc, indent=1))
2083
+ collected_path = runtime / "collected.json"
2084
+ if stage == "finalize":
2085
+ try:
2086
+ stored = json.loads(collected_path.read_text())
2087
+ except (OSError, json.JSONDecodeError) as exc:
2088
+ raise ChannelFootprintError(f"collect the section before finalize: {exc}") from exc
2089
+ if not _collection_matches(stored, context_hash, policy_hash, sources, config):
2090
+ raise ChannelFootprintError("collected data context or policy changed; rerun collect using compatible provider cache")
2091
+ outcome = finalize_pipeline(stored["raw"], inputs, config, date.fromisoformat(stored["captured_on"]))
2092
+ else:
2093
+ api_key = (os.environ.get("OS_API_KEY") or "").strip()
2094
+ if not api_key:
2095
+ raise ChannelFootprintError("OS_API_KEY is required for collection")
2096
+ os_url = (os.environ.get("OS_URL") or DEFAULT_OS_URL).strip() or DEFAULT_OS_URL
2097
+ client = OsClient(
2098
+ os_url=os_url,
2099
+ api_key=api_key,
2100
+ cache_dir=cache_dir,
2101
+ max_actor_runs=int(_arg(args, "max_actor_runs", MAX_ACTOR_RUNS_PER_ATTEMPT)),
2102
+ max_meta_searches=int(_arg(args, "max_meta_searches", MAX_META_SEARCHES_PER_ATTEMPT)),
2103
+ max_search_calls=int(_arg(args, "max_search_calls", MAX_SEARCH_CALLS_PER_ATTEMPT)),
2104
+ max_fetches=int(_arg(args, "max_fetches", MAX_FETCHES_PER_ATTEMPT)),
2105
+ max_cdx_calls=int(_arg(args, "max_cdx_calls", MAX_CDX_CALLS_PER_ATTEMPT)),
2106
+ max_spyfu_calls=int(_arg(args, "max_spyfu_calls", MAX_SPYFU_CALLS_PER_ATTEMPT)),
2107
+ max_total_paid_starts=max_total, cumulative_budget_path=runtime / "budget.json",
2108
+ )
2109
+
2110
+ try:
2111
+ previous = json.loads(collected_path.read_text())
2112
+ except (OSError, json.JSONDecodeError):
2113
+ previous = {}
2114
+ if _collection_matches(previous, context_hash, policy_hash, sources, config):
2115
+ inputs.prior_expansions = (previous.get("raw") or {}).get("expansions") or []
2116
+ outcome = run_pipeline(client, inputs, config, date.today())
2117
+ if _arg(args, "relevance_by_model", False):
2118
+ product_context, languages = relevance_inputs
2119
+ for _ in range(MODEL_RELEVANCE_ROUNDS):
2120
+ decided = model_relevance(client, outcome["corpus"], outcome["pending_relevance"], decisions, context_hash, product_context, languages)
2121
+ if decided is None:
2122
+ break
2123
+ decisions = inputs.relevance_decisions = decided
2124
+ _write_atomic(decision_path, json.dumps(decided, indent=1, ensure_ascii=False))
2125
+ # A collected expansion is replayed in the next pass, as on a rerun.
2126
+ inputs.prior_expansions = outcome["raw"]["expansions"] + [e for e in inputs.prior_expansions if e not in outcome["raw"]["expansions"]]
2127
+ outcome = run_pipeline(client, inputs, config, date.today())
2128
+ _write_atomic(collected_path, json.dumps({"schema_version": 2, "input_context_sha256": context_hash, "policy_sha256": policy_hash, "sources": sources, "sources_config_sha256": _model().digest(config), "captured_on": date.today().isoformat(), "raw_sha256": _model().digest(outcome["raw"]), "raw": outcome["raw"]}, indent=1))
2129
+ no_observations_budget_deferred = outcome["totals"]["observations"] == 0 and any(r.get("status") == "budget_deferred" for r in outcome["raw"]["coverage"])
2130
+ if outcome["totals"]["observations"] == 0 and not no_observations_budget_deferred:
2131
+ raise ChannelFootprintError(
2132
+ "no source answered for any competitor (every provider call was rejected or failed); the section "
2133
+ "cannot complete without at least one successful observation. Rerun later or change --sources."
2134
+ )
2135
+
2136
+ candidates = {"schema_version": 1, "product_context_sha256": context_hash, "policy_sha256": policy_hash,
2137
+ "product_context": {"context": context, "intake": intake, "product_domain": product_domain, "competitor_context": competitor_context},
2138
+ "candidates": outcome["corpus"], "pending_ad_ids": outcome["pending_relevance"]}
2139
+ _write_atomic(runtime / "relevance-candidates.json", json.dumps(candidates, indent=1, ensure_ascii=False))
2140
+ if outcome["pending_relevance"] or not outcome["collection_complete"] or stage == "collect" or no_observations_budget_deferred:
2141
+ status = "budget_deferred" if no_observations_budget_deferred else "awaiting_relevance" if outcome["pending_relevance"] else ("awaiting_collection" if not outcome["collection_complete"] else "awaiting_finalize")
2142
+ state = {"schema_version": 1, "section_id": SECTION_ID, "status": status, "domains": domains, "target_countries": target_countries,
2143
+ "research_context": research_context,
2144
+ "input_context_sha256": context_hash, "policy_sha256": policy_hash, "pending_ad_ids": outcome["pending_relevance"], "coverage": outcome["raw"]["coverage"],
2145
+ "candidates_path": (RUNTIME_REL / "relevance-candidates.json").as_posix(), "collected_path": (RUNTIME_REL / "collected.json").as_posix(), "error": None}
2146
+ _write_atomic(runtime / "request.json", json.dumps(state, indent=1))
2147
+ return state
2148
+
2149
+ evidence_path = workspace / EVIDENCE_REL
2150
+ evidence_json = json.dumps(outcome["evidence"], indent=1, ensure_ascii=False)
2151
+ _write_atomic(evidence_path, evidence_json)
2152
+ section_file = runtime / "result.md"
2153
+ _write_atomic(section_file, outcome["section_markdown"])
2154
+
2155
+ upsert = _load_script("upsert-markdown-section.py", "pmr_upsert_markdown_section")
2156
+ report_path = workspace / REPORT_REL
2157
+ report_path.parent.mkdir(parents=True, exist_ok=True)
2158
+ upsert.upsert_markdown_section(report_path, SECTION_HEADING, outcome["section_markdown"])
2159
+
2160
+ persist = _load_script("persist-research-report.py", "pmr_persist_research_report")
2161
+ persistence = persist.persist_research_report(
2162
+ workspace,
2163
+ commit_message=f"research: update {SECTION_HEADING.lower()} section",
2164
+ extra_paths=(Path(_artifact_path("egbe-docs/research/market/evidence/channel-footprint.json")),),
2165
+ )
2166
+
2167
+ state = {
2168
+ "schema_version": 1,
2169
+ "section_id": SECTION_ID,
2170
+ "status": "completed",
2171
+ "domains": inputs.domains,
2172
+ "domain_sources": inputs.domain_sources,
2173
+ "sources": inputs.sources,
2174
+ "country": inputs.country,
2175
+ "eu_country": inputs.eu_country,
2176
+ "target_countries": inputs.target_countries,
2177
+ "input_context_sha256": context_hash,
2178
+ "intake_sha256": intake_hash,
2179
+ "competitor_set_sha256": provenance.get("source_sha256"),
2180
+ "policy_sha256": policy_hash,
2181
+ "effective_policy": policy,
2182
+ "relevance_decisions_sha256": _model().digest(decisions),
2183
+ "sources_config_sha256": _model().digest(config),
2184
+ "evidence_path": EVIDENCE_REL.as_posix(),
2185
+ "evidence_sha256": hashlib.sha256(evidence_json.encode("utf-8")).hexdigest(),
2186
+ "result_path": (RUNTIME_REL / "result.md").as_posix(),
2187
+ "result_sha256": hashlib.sha256(outcome["section_markdown"].encode("utf-8")).hexdigest(),
2188
+ "provider_calls": outcome["totals"]["provider_calls"],
2189
+ "actor_runs": outcome["totals"]["actor_runs"],
2190
+ "meta_searches": outcome["totals"]["meta_searches"],
2191
+ "search_calls": outcome["totals"]["search_calls"],
2192
+ "fetches": outcome["totals"]["fetches"],
2193
+ "cdx_calls": outcome["totals"]["cdx_calls"],
2194
+ "spyfu_calls": outcome["totals"]["spyfu_calls"],
2195
+ "relevance_calls": outcome["totals"].get("relevance_calls", 0),
2196
+ "rejected_actor_runs": outcome["totals"]["rejected_actor_runs"],
2197
+ "rejected_spyfu_calls": outcome["totals"]["rejected_spyfu_calls"],
2198
+ "observations": outcome["totals"]["observations"],
2199
+ "actor_starts_recorded": outcome["totals"]["actor_starts_recorded"],
2200
+ "attempt_actor_runs": outcome["attempt"]["actor_runs"],
2201
+ "attempt_meta_searches": outcome["attempt"]["meta_searches"],
2202
+ "attempt_search_calls": outcome["attempt"]["search_calls"],
2203
+ "attempt_fetches": outcome["attempt"]["fetches"],
2204
+ "attempt_cdx_calls": outcome["attempt"]["cdx_calls"],
2205
+ "attempt_cache_hits": outcome["attempt"]["cache_hits"],
2206
+ "report_commit_sha": persistence.commit_sha,
2207
+ "publication_mode": persistence.publication_mode,
2208
+ "research_context": _load_script("research-artifact-io.py", "pmr_input_io").context_binding(workspace, SECTION_ID),
2209
+ "error": None,
2210
+ }
2211
+ _write_atomic(runtime / "request.json", json.dumps(state, indent=1))
2212
+ return state
2213
+
2214
+
2215
+ def main() -> int:
2216
+ parser = argparse.ArgumentParser(description=__doc__)
2217
+ parser.add_argument("--context-file", type=Path, help="Explicit context for this study; no project files are discovered implicitly.")
2218
+ parser.add_argument("--workspace", type=Path, required=True)
2219
+ parser.add_argument("--domains", default="", help="Comma-separated competitor domains; merged ahead of the Competitor Set and Keyword Economics domains.")
2220
+ parser.add_argument("--product-domain", default="", help="The product's own domain; never scanned as a competitor.")
2221
+ parser.add_argument("--sources", default="", help=f"Comma-separated subset of {', '.join(ALL_SOURCES)} (default: all).")
2222
+ parser.add_argument("--sources-config", default="", help="Path to an alternative sources config JSON (actor ids and batch sizes).")
2223
+ parser.add_argument("--max-competitors", type=int, default=MAX_COMPETITORS)
2224
+ parser.add_argument("--country", default="", help="Explicit single target country; otherwise typed intake markets, then US.")
2225
+ parser.add_argument("--eu-country", default="", help="Deprecated compatibility argument; target markets come from intake/--countries.")
2226
+ parser.add_argument("--countries", "--target-countries", default="", help="Explicit ISO target countries; otherwise --country or typed intake markets.")
2227
+ parser.add_argument("--languages", default="", help="Explicit accepted language codes for relevance classification; otherwise typed intake languages.")
2228
+ parser.add_argument("--stage", choices=("all", "collect", "finalize"), default="all")
2229
+ parser.add_argument("--relevance-decisions", default="", help="Context-bound decisions JSON (default runtime/relevance-decisions.json).")
2230
+ parser.add_argument("--relevance-by-model", action="store_true", help="Decide ads with text by the creative classifier's relevance verdict; the rest stay for review.")
2231
+ parser.add_argument("--max-total-paid-starts", type=int, default=None, help="Durable section cap across every retry and rerun.")
2232
+ parser.add_argument("--pixel-actor-fallback", action="store_true", help="Use the headless tag-detector actor for sites whose HTML cannot be fetched directly (one actor run per batch).")
2233
+ parser.add_argument("--max-actor-runs", type=int, default=MAX_ACTOR_RUNS_PER_ATTEMPT)
2234
+ parser.add_argument("--max-meta-searches", type=int, default=MAX_META_SEARCHES_PER_ATTEMPT)
2235
+ parser.add_argument("--max-search-calls", type=int, default=MAX_SEARCH_CALLS_PER_ATTEMPT)
2236
+ parser.add_argument("--max-fetches", type=int, default=MAX_FETCHES_PER_ATTEMPT)
2237
+ parser.add_argument("--max-cdx-calls", type=int, default=MAX_CDX_CALLS_PER_ATTEMPT)
2238
+ parser.add_argument("--max-spyfu-calls", type=int, default=MAX_SPYFU_CALLS_PER_ATTEMPT, help="SpyFu domain-stats lookups per attempt (one per competitor).")
2239
+ args = parser.parse_args()
2240
+
2241
+ try:
2242
+ state = run(args.workspace, args)
2243
+ except (ChannelFootprintError, ValueError) as exc:
2244
+ print(f"{DISPLAY_NAME}: {exc}", file=sys.stderr)
2245
+ return 1
2246
+ if state["status"] != "completed":
2247
+ print(f"{DISPLAY_NAME}: {state['status']} — {state.get('candidates_path', '')}; review decisions, then collect/finalize")
2248
+ return 2
2249
+ print(
2250
+ f"{DISPLAY_NAME}: completed — {len(state['domains'])} competitor(s), {state['actor_runs']} actor run(s), "
2251
+ f"{state['meta_searches']} Meta library search(es), {state['search_calls']} web search(es), evidence at "
2252
+ f"{state['evidence_path']}, report commit {(state['report_commit_sha'] or '')[:12] or 'n/a'}"
2253
+ )
2254
+ return 0
2255
+
2256
+
2257
+ if __name__ == "__main__":
2258
+ raise SystemExit(main())