@egbe-ai/opencode 0.0.0-stage → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (479) hide show
  1. package/LICENSE +4 -0
  2. package/README.md +124 -2
  3. package/index.js +5 -0
  4. package/package.json +55 -4
  5. package/plugin.js +230 -0
  6. package/session-log.js +315 -0
  7. package/skills/.egbe-catalog/manifest.json +1 -0
  8. package/skills/egbe-os-analytics/SKILL.md +499 -0
  9. package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
  10. package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
  11. package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
  12. package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
  13. package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
  14. package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
  15. package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
  16. package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
  17. package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
  18. package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
  19. package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
  20. package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
  21. package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
  22. package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
  23. package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
  24. package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
  25. package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
  26. package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
  27. package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
  28. package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
  29. package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
  30. package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
  31. package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
  32. package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
  33. package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
  34. package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
  35. package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
  36. package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
  37. package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
  38. package/skills/egbe-os-db/SKILL.md +174 -0
  39. package/skills/egbe-os-deploy/SKILL.md +412 -0
  40. package/skills/egbe-os-design/SKILL.md +182 -0
  41. package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
  42. package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
  43. package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
  44. package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
  45. package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
  46. package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
  47. package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
  48. package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
  49. package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
  50. package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
  51. package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
  52. package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
  53. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
  54. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
  55. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
  56. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
  57. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
  58. package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
  59. package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
  60. package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
  61. package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
  62. package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
  63. package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
  64. package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
  65. package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
  66. package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
  67. package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
  68. package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
  69. package/skills/egbe-os-destination-readiness/references/report.md +116 -0
  70. package/skills/egbe-os-domains/SKILL.md +189 -0
  71. package/skills/egbe-os-errors/SKILL.md +259 -0
  72. package/skills/egbe-os-git/SKILL.md +275 -0
  73. package/skills/egbe-os-git/references/issues.md +33 -0
  74. package/skills/egbe-os-google-ads/SKILL.md +163 -0
  75. package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
  76. package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
  77. package/skills/egbe-os-google-ads/references/conversions.md +153 -0
  78. package/skills/egbe-os-google-ads/references/delivery.md +128 -0
  79. package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
  80. package/skills/egbe-os-google-ads/references/reporting.md +105 -0
  81. package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
  82. package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
  83. package/skills/egbe-os-google-auth/SKILL.md +285 -0
  84. package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
  85. package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
  86. package/skills/egbe-os-images/SKILL.md +163 -0
  87. package/skills/egbe-os-images/references/stock-images.md +21 -0
  88. package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
  89. package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
  90. package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
  91. package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
  92. package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
  93. package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
  94. package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
  95. package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
  96. package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
  97. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
  98. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
  99. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
  100. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
  101. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
  102. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
  103. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
  104. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
  105. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
  106. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
  107. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
  108. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
  109. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
  110. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
  111. package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
  112. package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
  113. package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
  114. package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
  115. package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
  116. package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
  117. package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
  118. package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
  119. package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
  120. package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
  121. package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
  122. package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
  123. package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
  124. package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
  125. package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
  126. package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
  127. package/skills/egbe-os-llm/SKILL.md +160 -0
  128. package/skills/egbe-os-mail/SKILL.md +180 -0
  129. package/skills/egbe-os-me/SKILL.md +119 -0
  130. package/skills/egbe-os-meta-ads/SKILL.md +114 -0
  131. package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
  132. package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
  133. package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
  134. package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
  135. package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
  136. package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
  137. package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
  138. package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
  139. package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
  140. package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
  141. package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
  142. package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
  143. package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
  144. package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
  145. package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
  146. package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
  147. package/skills/egbe-os-notifications/SKILL.md +78 -0
  148. package/skills/egbe-os-openai-ads/SKILL.md +149 -0
  149. package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
  150. package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
  151. package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
  152. package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
  153. package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
  154. package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
  155. package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
  156. package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
  157. package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
  158. package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
  159. package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
  160. package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
  161. package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
  162. package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
  163. package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
  164. package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
  165. package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
  166. package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
  167. package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
  168. package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
  169. package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
  170. package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
  171. package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
  172. package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
  173. package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
  174. package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
  175. package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
  176. package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
  177. package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
  178. package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
  179. package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
  180. package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
  181. package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
  182. package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
  183. package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
  184. package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
  185. package/skills/egbe-os-payments/SKILL.md +215 -0
  186. package/skills/egbe-os-payments/agents/openai.yaml +4 -0
  187. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
  188. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
  189. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
  190. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
  191. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
  192. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
  193. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
  194. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
  195. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
  196. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
  197. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
  198. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
  199. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
  200. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
  201. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
  202. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
  203. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
  204. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
  205. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
  206. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
  207. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
  208. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
  209. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
  210. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
  211. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
  212. package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
  213. package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
  214. package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
  215. package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
  216. package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
  217. package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
  218. package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
  219. package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
  220. package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
  221. package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
  222. package/skills/egbe-os-playbook/SKILL.md +79 -0
  223. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
  224. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
  225. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
  226. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
  227. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
  228. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
  229. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
  230. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
  231. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
  232. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
  233. package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
  234. package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
  235. package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
  236. package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
  237. package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
  238. package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
  239. package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
  240. package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
  241. package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
  242. package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
  243. package/skills/egbe-os-playbook/playbooks/product.md +67 -0
  244. package/skills/egbe-os-playbook/references/catalog.md +127 -0
  245. package/skills/egbe-os-playbook/references/changes.md +167 -0
  246. package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
  247. package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
  248. package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
  249. package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
  250. package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
  251. package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
  252. package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
  253. package/skills/egbe-os-product-auth/SKILL.md +151 -0
  254. package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
  255. package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
  256. package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
  257. package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
  258. package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
  259. package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
  260. package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
  261. package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
  262. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
  263. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
  264. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
  265. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
  266. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
  267. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
  268. package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
  269. package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
  270. package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
  271. package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
  272. package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
  273. package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
  274. package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
  275. package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
  276. package/skills/egbe-os-product-build/SKILL.md +111 -0
  277. package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
  278. package/skills/egbe-os-product-build/references/capabilities.md +25 -0
  279. package/skills/egbe-os-product-build/references/engineering.md +72 -0
  280. package/skills/egbe-os-product-build/references/execution.md +55 -0
  281. package/skills/egbe-os-product-build/references/foundations.md +93 -0
  282. package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
  283. package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
  284. package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
  285. package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
  286. package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
  287. package/skills/egbe-os-product-market-research/SKILL.md +159 -0
  288. package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
  289. package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
  290. package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
  291. package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
  292. package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
  293. package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
  294. package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
  295. package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
  296. package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
  297. package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
  298. package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
  299. package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
  300. package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
  301. package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
  302. package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
  303. package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
  304. package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
  305. package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
  306. package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
  307. package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
  308. package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
  309. package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
  310. package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
  311. package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
  312. package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
  313. package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
  314. package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
  315. package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
  316. package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
  317. package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
  318. package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
  319. package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
  320. package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
  321. package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
  322. package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
  323. package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
  324. package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
  325. package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
  326. package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
  327. package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
  328. package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
  329. package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
  330. package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
  331. package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
  332. package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
  333. package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
  334. package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
  335. package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
  336. package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
  337. package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
  338. package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
  339. package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
  340. package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
  341. package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
  342. package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
  343. package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
  344. package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
  345. package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
  346. package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
  347. package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
  348. package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
  349. package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
  350. package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
  351. package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
  352. package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
  353. package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
  354. package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
  355. package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
  356. package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
  357. package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
  358. package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
  359. package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
  360. package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
  361. package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
  362. package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
  363. package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
  364. package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
  365. package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
  366. package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
  367. package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
  368. package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
  369. package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
  370. package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
  371. package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
  372. package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
  373. package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
  374. package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
  375. package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
  376. package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
  377. package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
  378. package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
  379. package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
  380. package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
  381. package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
  382. package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
  383. package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
  384. package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
  385. package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
  386. package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
  387. package/skills/egbe-os-product-report/SKILL.md +150 -0
  388. package/skills/egbe-os-product-report/references/artifacts.md +84 -0
  389. package/skills/egbe-os-product-report/references/brief.md +34 -0
  390. package/skills/egbe-os-product-report/references/daily.md +76 -0
  391. package/skills/egbe-os-product-report/references/sources.md +68 -0
  392. package/skills/egbe-os-product-routines/SKILL.md +166 -0
  393. package/skills/egbe-os-product-routines/references/catalog.md +22 -0
  394. package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
  395. package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
  396. package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
  397. package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
  398. package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
  399. package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
  400. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
  401. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
  402. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
  403. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
  404. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
  405. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
  406. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
  407. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
  408. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
  409. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
  410. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
  411. package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
  412. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
  413. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
  414. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
  415. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
  416. package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
  417. package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
  418. package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
  419. package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
  420. package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
  421. package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
  422. package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
  423. package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
  424. package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
  425. package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
  426. package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
  427. package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
  428. package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
  429. package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
  430. package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
  431. package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
  432. package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
  433. package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
  434. package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
  435. package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
  436. package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
  437. package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
  438. package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
  439. package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
  440. package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
  441. package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
  442. package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
  443. package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
  444. package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
  445. package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
  446. package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
  447. package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
  448. package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
  449. package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
  450. package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
  451. package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
  452. package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
  453. package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
  454. package/skills/egbe-os-report-ui/SKILL.md +133 -0
  455. package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
  456. package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
  457. package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
  458. package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
  459. package/skills/egbe-os-report-ui/references/components.md +209 -0
  460. package/skills/egbe-os-research/SKILL.md +125 -0
  461. package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
  462. package/skills/egbe-os-scrape/SKILL.md +160 -0
  463. package/skills/egbe-os-search/SKILL.md +182 -0
  464. package/skills/egbe-os-search/references/google-places.md +81 -0
  465. package/skills/egbe-os-support/SKILL.md +149 -0
  466. package/skills/egbe-os-telemetry/SKILL.md +344 -0
  467. package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
  468. package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
  469. package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
  470. package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
  471. package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
  472. package/skills/egbe-os-tools/SKILL.md +180 -0
  473. package/skills/egbe-os-tools/references/theona.md +26 -0
  474. package/skills/egbe-os-wallet/SKILL.md +181 -0
  475. package/skills/egbe-setup/EGBE.md +484 -0
  476. package/skills/egbe-setup/SKILL.md +116 -0
  477. package/skills/egbe-setup/references/project.md +80 -0
  478. package/state.js +37 -0
  479. package/update.js +361 -0
@@ -0,0 +1,2042 @@
1
+ #!/usr/bin/env python3
2
+ """Collect individual competitor ads and derive scoped creative patterns.
3
+
4
+ Reuse completed Channel Footprint inputs, restore native text and available
5
+ media, and classify comparable offers through OS. Generic taxonomy dimensions
6
+ and category-specific message meanings remain distinct. Confirmed serving
7
+ history and current examples support patterns within market/channel/type/format;
8
+ listing spans never establish delivery duration or conversion performance.
9
+ Cached provider responses, labels and video analysis survive compatible retries,
10
+ with a durable cumulative resource ceiling and explicit partial coverage.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import sys as _artifact_sys
15
+ from pathlib import Path as _ArtifactPath
16
+ _artifact_sys.path.insert(0, str(_ArtifactPath(__file__).resolve().parent))
17
+ from egbe_artifacts import artifact_path as _artifact_path
18
+
19
+ import argparse
20
+ import base64
21
+ import copy
22
+ import hashlib
23
+ import importlib.util
24
+ import json
25
+ import os
26
+ import re
27
+ import sys
28
+ import time
29
+ import urllib.error
30
+ import urllib.parse
31
+ import urllib.request
32
+ from dataclasses import dataclass, field
33
+ from datetime import date, timedelta
34
+ from pathlib import Path
35
+ from types import ModuleType
36
+ from typing import Any, Callable
37
+
38
+ SCHEMA_VERSION = 1
39
+ EVIDENCE_SCHEMA_VERSION = 2
40
+ SECTION_ID = "creative_patterns"
41
+ SECTION_HEADING = "Creative patterns"
42
+ DEFAULT_OS_URL = "https://os.hz.egbe.dev"
43
+ RUNTIME_REL = Path(_artifact_path("egbe-docs/research/market/work")) / SECTION_ID
44
+ EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/creative-patterns.json"))
45
+ REPORT_REL = Path(_artifact_path("egbe-docs/research/market/full-report.md"))
46
+ TAXONOMY_REL = Path("references/config/creative-taxonomy.v1.json")
47
+ CHANNEL_FOOTPRINT_EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/channel-footprint.json"))
48
+
49
+ MAX_COMPETITORS = 10
50
+ MAX_EXPLICIT_DOMAINS = 10
51
+ # Ten competitors with every source need 32 runs (two Google batches, then
52
+ # Meta, LinkedIn and Reddit per competitor): one attempt covers a default run.
53
+ MAX_ACTOR_RUNS_PER_ATTEMPT = 36
54
+ MAX_META_SEARCHES_PER_ATTEMPT = 12
55
+ MAX_SPYFU_CALLS_PER_ATTEMPT = 24
56
+ MAX_CLASSIFY_CALLS_PER_ATTEMPT = 40
57
+ GOOGLE_ADS_PER_DOMAIN = 40
58
+ # A non-brand advertiser is the competitor's legal entity only when it clearly
59
+ # dominates the domain's rows. Live 2026-10-02: freecodecamp.org runs no
60
+ # Google ads; four unrelated advertisers pointed at it and the largest, with
61
+ # 2 of the 4 rows, was taken for the competitor.
62
+ GOOGLE_DOMINANT_MIN_ROWS = 3
63
+ GOOGLE_DOMINANT_MIN_SHARE = 0.75
64
+ META_ADS_PER_BRAND = 30
65
+ META_EU_ADS_PER_BRAND = 30
66
+ # The LinkedIn library actor works through only the first company page of a
67
+ # multi-page input (observed live), so each competitor gets its own run.
68
+ LINKEDIN_ADS_PER_COMPANY = 30
69
+ REDDIT_ADS_PER_BRAND = 30
70
+ SPYFU_ROWS_PER_WINDOW = 100
71
+ SPYFU_EARLIER_WINDOW = (105, 365)
72
+ TEXT_BATCH = 10
73
+ IMAGE_BATCH = 8
74
+ CLASSIFY_TIMEOUT_SECONDS = 330.0
75
+ # Classification errors that are about the caller, not the batch.
76
+ STOP_CLASSIFYING_RE = re.compile(r"HTTP (?:401|402|403|429)\b|budget_exceeded|Budget has been exceeded|ExceededBudget")
77
+ PRODUCT_CONTEXT_CHARS = 24000
78
+ ALL_SOURCES = ("google_transparency", "meta", "google_search", "linkedin", "reddit")
79
+ # Libraries that keep stopped ads listed (LinkedIn keeps them a year after the
80
+ # last impression): being listed at a cutoff proves neither age nor delivery.
81
+ LISTS_STOPPED_ADS = {"linkedin_library_actor", "reddit_library_actor"}
82
+ # Rows kept per competitor and platform for ads that stopped since an earlier
83
+ # cutoff; they never take the places of ads seen at this cutoff.
84
+ # Which creative `source` values each selectable source produces.
85
+ SOURCE_ROW_SOURCES = {
86
+ "google_transparency": {"google_transparency"},
87
+ "meta": {"meta_library_actor", "meta_library_official_eu"},
88
+ "google_search": {"spyfu_ad_history"},
89
+ "linkedin": {"linkedin_library_actor"},
90
+ "reddit": {"reddit_library_actor"},
91
+ }
92
+ PLATFORM_CHANNELS = {"google": "google_ads", "youtube": "youtube_ads", "meta": "meta_ads", "linkedin": "linkedin_ads", "reddit": "reddit_ads"}
93
+ # A price needs an amount: a currency sign or code next to a number, or a
94
+ # percent discount. Period words alone ("in a month") are not prices.
95
+ _WS_RE = re.compile(r"\s+")
96
+ _TEMPLATE_RE = re.compile(r"\{\{[^}]+\}\}")
97
+ # SpyFu renders sitelinks and callouts as Markdown links to "#".
98
+ _MD_LINK_RE = re.compile(r"\[([^\]]*)\]\(#?\)")
99
+ _SITELINK_SEP_RE = re.compile(r"\s*(?:·|\\n|\n)\s*")
100
+ _DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
101
+
102
+
103
+ def _load_script(filename: str, module_name: str) -> ModuleType:
104
+ path = Path(__file__).resolve().parent / filename
105
+ spec = importlib.util.spec_from_file_location(module_name, path)
106
+ if spec is None or spec.loader is None:
107
+ raise RuntimeError(f"cannot load bundled script {filename}")
108
+ module = importlib.util.module_from_spec(spec)
109
+ sys.modules[spec.name] = module
110
+ previous = sys.dont_write_bytecode
111
+ sys.dont_write_bytecode = True
112
+ try:
113
+ spec.loader.exec_module(module)
114
+ finally:
115
+ sys.dont_write_bytecode = previous
116
+ return module
117
+
118
+
119
+ # The Channel Footprint runner owns the OS client (cache, journaled starts,
120
+ # per-attempt budgets, the per-source breaker) and the identity rules
121
+ # (destination first, strict advertiser names); this runner reuses them.
122
+ CF = _load_script("run-channel-footprint.py", "pmr_channel_footprint_for_creative_patterns")
123
+ POLICY = _load_script("research-policy.py", "pmr_policy_for_creative_patterns")
124
+ MODEL = _load_script("creative-model.py", "pmr_creative_model")
125
+ SEMANTICS = _load_script("creative-semantics.py", "pmr_creative_semantics")
126
+ VIDEO = _load_script("creative-video.py", "pmr_creative_video")
127
+ SPYFU_RECENT_WINDOW_DAYS = 30 # Bounded provider query window; never an activity or success threshold.
128
+ CreativePatternsError = CF.ChannelFootprintError
129
+ BudgetExhausted = CF.BudgetExhausted
130
+ SourceUnavailable = CF.SourceUnavailable
131
+ registrable_domain = CF.registrable_domain
132
+ brand_token = CF.brand_token
133
+ usable_destination = CF.usable_destination
134
+
135
+
136
+ def load_taxonomy(path: Path | None = None) -> tuple[dict[str, Any], str]:
137
+ taxonomy_path = path or (Path(__file__).resolve().parent.parent / TAXONOMY_REL)
138
+ raw = taxonomy_path.read_bytes()
139
+ taxonomy = json.loads(raw)
140
+ if taxonomy.get("schema_version") != 1 or not isinstance(taxonomy.get("dimensions"), dict):
141
+ raise CreativePatternsError("the creative taxonomy file has an unexpected shape")
142
+ return taxonomy, hashlib.sha256(raw).hexdigest()
143
+
144
+
145
+ # ── OS client: the Channel Footprint client plus SpyFu ad history and
146
+ # creative classification
147
+
148
+ @dataclass
149
+ class OsClient(CF.OsClient):
150
+ max_classify_calls: int = MAX_CLASSIFY_CALLS_PER_ATTEMPT
151
+ classify_calls: int = 0
152
+ # Classifications and images depend only on the creative, so they live in
153
+ # one cache shared by every cutoff (a refresh never pays for them twice).
154
+ shared_dir: Path | None = None
155
+ semantic_context: dict[str, Any] | None = None
156
+ media_calls: int = 0
157
+ max_media_calls: int = 100
158
+ budget_gaps: list[dict] = field(default_factory=list)
159
+ # Deployment and shared AI-account failures are scoped to this attempt.
160
+ # Successful content caches stay available; a later client can recover.
161
+ media_endpoint_blocked: dict[str, str] = field(default_factory=dict)
162
+ paid_ai_blocked_reason: str = ""
163
+ # GET /v1/media capability flags, read once per attempt on the first uncached transcribe.
164
+ get: Callable[[str, float], dict[str, Any]] | None = None
165
+ media_capabilities: dict[str, Any] | None = None
166
+
167
+ def _transcription_configured(self) -> bool:
168
+ """False only when GET /v1/media says so; an unreadable answer keeps transcribe."""
169
+ if self.media_capabilities is None:
170
+ try:
171
+ self.media_capabilities = (self.get or self._http_get)("/v1/media", 30.0)
172
+ except Exception:
173
+ self.media_capabilities = {}
174
+ flags = self.media_capabilities.get("transcription")
175
+ return not (isinstance(flags, dict) and flags.get("configured") is False)
176
+
177
+ def _http_get(self, path: str, timeout: float) -> dict[str, Any]:
178
+ req = urllib.request.Request(f"{self.os_url.rstrip('/')}{path}", method="GET",
179
+ headers={"Authorization": f"Bearer {self.api_key}", "Accept": "application/json"})
180
+ for name, value in CF._connection_headers(req.full_url).items():
181
+ req.add_unredirected_header(name, value)
182
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
183
+ envelope = json.loads(resp.read().decode("utf-8"))
184
+ data = envelope.get("data") if isinstance(envelope, dict) else None
185
+ return data if isinstance(data, dict) else {}
186
+
187
+ def media_call(self, path: str, payload: dict[str, Any], *, kind: str, timeout: float = 330.0) -> dict[str, Any]:
188
+ key_payload = {"kind": kind, "path": path, "payload": payload}
189
+ digest = hashlib.sha256(json.dumps(key_payload, sort_keys=True).encode()).hexdigest()
190
+ cache = self._shared() / f"{digest}.json"
191
+ entry = self._cache_read(cache)
192
+ self.touched.add(digest)
193
+ if entry is not None:
194
+ self.cache_hits += 1
195
+ return entry.get("response") or {}
196
+ if path.startswith("/v1/media/") and self.paid_ai_blocked_reason:
197
+ raise CreativePatternsError(self.paid_ai_blocked_reason)
198
+ if path in self.media_endpoint_blocked:
199
+ raise SourceUnavailable(self.media_endpoint_blocked[path])
200
+ if path == "/v1/media/transcribe" and not self._transcription_configured():
201
+ self.media_endpoint_blocked[path] = "GET /v1/media reports transcription.configured is false; no audio was sent"
202
+ raise SourceUnavailable(self.media_endpoint_blocked[path])
203
+ if self.media_calls >= self.max_media_calls:
204
+ raise BudgetExhausted("media call budget exhausted for this attempt; remaining videos are budget_deferred")
205
+ self._reserve_paid_start(kind, digest)
206
+ self.media_calls += 1
207
+ try:
208
+ response = (self.request or self._http)(path, payload, timeout, idempotency_key=None)
209
+ except Exception as exc:
210
+ reason = str(exc)[:500]
211
+ if path.startswith("/v1/media/") and STOP_CLASSIFYING_RE.search(reason):
212
+ self.paid_ai_blocked_reason = reason
213
+ raise CreativePatternsError(reason) from exc
214
+ if re.search(r"HTTP (?:404|405|503)\b", reason):
215
+ self.media_endpoint_blocked[path] = reason
216
+ raise SourceUnavailable(reason) from exc
217
+ raise
218
+ self._cache_write(cache, {"kind": kind, "response": response, "cutoff": self.cutoff, "retrieved_on": date.today().isoformat()})
219
+ return response
220
+
221
+
222
+ def _shared(self) -> Path:
223
+ base = self.shared_dir or self.cache_dir
224
+ base.mkdir(parents=True, exist_ok=True)
225
+ return base
226
+
227
+ def spyfu_ad_history(self, domain: str, country: str, since: str, until: str, max_rows: int) -> tuple[dict[str, Any], str]:
228
+ payload = {"domain": domain, "country_code": country, "max_rows": max_rows, "since": since, "until": until}
229
+ response, retrieved_on = self._metered(
230
+ "spyfu", "/tools/spyfu/ad-history", payload, 90.0,
231
+ budget_attr="spyfu_calls", budget_max=self.max_spyfu_calls, label="SpyFu lookup", paid=False,
232
+ retry_delay=CF.FREE_RETRY_DELAY_SECONDS,
233
+ )
234
+ if response.get("unavailable"):
235
+ raise SourceUnavailable(f"SpyFu rejected the request: {response['unavailable']}")
236
+ if response.get("limited_by_quota"):
237
+ # The gateway cut the page to the claw's remaining daily rows: not
238
+ # the window's answer. Drop it from the cache and retry later.
239
+ digest = hashlib.sha256(json.dumps({"kind": "spyfu", "path": "/tools/spyfu/ad-history", "payload": payload}, sort_keys=True).encode("utf-8")).hexdigest()
240
+ CF.IO.drop_cache_entry(self.cache_dir / f"{digest}.json")
241
+ raise CreativePatternsError("SpyFu answer was cut by the daily spyfu:rows limit; rerun after the limit resets")
242
+ return response, retrieved_on
243
+
244
+ def classify(self, request: dict[str, Any], images: dict[str, dict[str, Any]], taxonomy_sha256: str) -> tuple[dict[str, Any], str, str]:
245
+ """One POST /v1/media/classify-creative call. The cache key covers the
246
+ request with frame hashes (never the image bytes) and the local
247
+ taxonomy file's hash, so an edited taxonomy is never served old answers."""
248
+ key_payload = {**request, "taxonomy_sha256": taxonomy_sha256, "classification_contract": 4,
249
+ "frames": [{"ref": ref, "sha256": images[ref]["sha256"], "media_type": images[ref]["media_type"]} for ref in sorted(images)]}
250
+ digest = hashlib.sha256(json.dumps({"kind": "classify", "path": "/v1/media/classify-creative", "payload": key_payload}, sort_keys=True).encode("utf-8")).hexdigest()
251
+ cache_file = self._shared() / f"{digest}.json"
252
+ self.touched.add(digest)
253
+ entry = self._cache_read(cache_file)
254
+ if entry is not None:
255
+ self.cache_hits += 1
256
+ response = entry.get("response")
257
+ if isinstance(response, dict) and response.get("unavailable"):
258
+ error = SourceUnavailable(f"classification batch was rejected: {response['unavailable']}")
259
+ error.batch_digest = digest
260
+ raise error
261
+ return (response if isinstance(response, dict) else {}), entry.get("retrieved_on") or date.today().isoformat(), digest
262
+ wire = {**{k: v for k, v in request.items()}, "frames": [
263
+ {"ref": ref, "timestamp_seconds": 0, "media_type": images[ref]["media_type"], "sha256": images[ref]["sha256"],
264
+ "content_base64": base64.b64encode(self.image_bytes(images[ref])).decode("ascii")}
265
+ for ref in sorted(images)
266
+ ]}
267
+ if self.paid_ai_blocked_reason:
268
+ raise CreativePatternsError(self.paid_ai_blocked_reason)
269
+ transport = self.request or self._http
270
+ attempts = 0
271
+ while True:
272
+ if self.classify_calls >= self.max_classify_calls:
273
+ raise BudgetExhausted(f"classification budget exhausted ({self.max_classify_calls} calls per attempt); {self.cache_writes} new answer(s) cached in this attempt; rerun to continue from the cache")
274
+ self._reserve_paid_start("classify", digest)
275
+ self.classify_calls += 1
276
+ try:
277
+ response = transport("/v1/media/classify-creative", wire, CLASSIFY_TIMEOUT_SECONDS, idempotency_key=None)
278
+ break
279
+ except CreativePatternsError as exc:
280
+ if re.search(r"HTTP 422\b", str(exc)):
281
+ # A rejected request is a fact about this batch: cache it
282
+ # so reruns never send (or pay for) it again.
283
+ self._cache_write(cache_file, {"kind": "classify", "path": "/v1/media/classify-creative", "payload": key_payload,
284
+ "response": {"unavailable": str(exc)[:300]}, "cutoff": self.cutoff,
285
+ "retrieved_on": date.today().isoformat()})
286
+ error = SourceUnavailable(f"classification batch was rejected: {str(exc)[:300]}")
287
+ error.batch_digest = digest
288
+ raise error from exc
289
+ # A server error (502 included: the model's answer may fail
290
+ # validation once) or an unreachable OS gets one retry.
291
+ unreachable = isinstance(exc, CF.OsUnreachable)
292
+ if attempts >= 1 or not (unreachable or re.search(r"HTTP 5(?:00|02|03|04)\b", str(exc))):
293
+ raise
294
+ attempts += 1
295
+ (self.sleep or time.sleep)(CF.OS_UNREACHABLE_RETRY_DELAY_SECONDS if unreachable else CF.FREE_RETRY_DELAY_SECONDS)
296
+ retrieved_on = date.today().isoformat()
297
+ self._cache_write(cache_file, {"kind": "classify", "path": "/v1/media/classify-creative", "payload": key_payload, "response": response,
298
+ "cutoff": self.cutoff, "retrieved_on": retrieved_on})
299
+ return response, retrieved_on, digest
300
+
301
+ def ledger_totals(self) -> dict[str, Any]:
302
+ kinds = ("actor", "meta-ad-library", "spyfu", "classify", "image", "video_metadata", "video_transcription", "video_analysis")
303
+ counts = dict.fromkeys(kinds, 0)
304
+ reused = dict.fromkeys(kinds, 0)
305
+ rejected = 0
306
+ usage = {"classify_prompt_tokens": 0, "classify_completion_tokens": 0,
307
+ "media_prompt_tokens": 0, "media_completion_tokens": 0}
308
+ dates = []
309
+ for digest in sorted(self.touched):
310
+ path = self.cache_dir / f"{digest}.json"
311
+ if CF.IO.cache_entry_path(path) is None and self.shared_dir is not None:
312
+ path = self.shared_dir / f"{digest}.json"
313
+ entry = self._cache_read(path)
314
+ if not entry:
315
+ continue
316
+ kind, response = entry.get("kind"), entry.get("response") or {}
317
+ if response.get("unavailable"):
318
+ rejected += 1
319
+ elif kind in kinds:
320
+ if kind in ("classify", "image", "video_metadata", "video_transcription", "video_analysis") and self.cutoff and entry.get("cutoff") not in (None, self.cutoff):
321
+ reused[kind] += 1
322
+ else:
323
+ counts[kind] += 1
324
+ token_usage = response.get("usage") or {}
325
+ prefix = "classify" if kind == "classify" else "media"
326
+ if kind != "image":
327
+ for field in ("prompt_tokens", "completion_tokens"):
328
+ key = prefix + "_" + field
329
+ value = token_usage.get(field)
330
+ if key in usage and type(value) is int:
331
+ usage[key] += value
332
+ if isinstance(entry.get("retrieved_on"), str):
333
+ dates.append(entry["retrieved_on"])
334
+ media_calls = sum(counts[k] for k in ("video_metadata", "video_transcription", "video_analysis"))
335
+ provider = sum(counts[k] for k in kinds if k != "image")
336
+ budget = self._cache_read(self.cumulative_budget_path) if self.cumulative_budget_path else None
337
+ return {"actor_runs": counts["actor"], "meta_searches": counts["meta-ad-library"], "spyfu_calls": counts["spyfu"],
338
+ "classify_calls": counts["classify"], "reused_classify_calls": reused["classify"], "image_fetches": counts["image"],
339
+ "media_calls": media_calls, "video_metadata_calls": counts["video_metadata"],
340
+ "video_transcription_calls": counts["video_transcription"], "video_analysis_calls": counts["video_analysis"],
341
+ "reused_media_calls": sum(reused[k] for k in ("video_metadata", "video_transcription", "video_analysis")),
342
+ "provider_calls": provider, "rejected_calls": rejected, "observations": provider + counts["image"], **usage,
343
+ "cumulative_paid_starts": len((budget or {}).get("paid_starts", [])),
344
+ "cumulative_paid_start_limit": (budget or {}).get("limit", self.max_total_paid_starts),
345
+ "first_retrieved_on": min(dates) if dates else date.today().isoformat(),
346
+ "last_retrieved_on": max(dates) if dates else date.today().isoformat()}
347
+
348
+
349
+ # ── normalization helpers ───────────────────────────────────────────────────
350
+
351
+ def _clean_text(value: Any, cap: int | None = 600) -> str | None:
352
+ if not isinstance(value, str):
353
+ return None
354
+ if "](#" in value or "]()" in value:
355
+ value = _MD_LINK_RE.sub(lambda m: "; " + m.group(1).strip() if m.group(1).strip() else "", value)
356
+ value = _SITELINK_SEP_RE.sub(" ", value).replace(" ;", ";").strip(" ;")
357
+ text = _WS_RE.sub(" ", value).strip()
358
+ if not text or _TEMPLATE_RE.fullmatch(text) or text.startswith("{{"):
359
+ return None
360
+ return text[:cap]
361
+
362
+
363
+ def _norm(value: str | None) -> str:
364
+ return _WS_RE.sub(" ", (value or "").casefold()).strip()
365
+
366
+
367
+ def fingerprint(headline: str | None, body: str | None, media: str | None = None, ad_key: str | None = None) -> str:
368
+ """Similarity key for copy/visual variants; never an ad identity or age proof."""
369
+ if _norm(headline) or _norm(body):
370
+ text = "copy:" + _norm(headline) + " | " + _norm(body)
371
+ elif media:
372
+ text = "media:" + media
373
+ else:
374
+ text = "ad:" + (ad_key or "")
375
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()[:20]
376
+
377
+
378
+ def destination_url(values: list[Any]) -> str | None:
379
+ """The first full landing URL whose host is usable (redirect wrappers
380
+ unwrapped, platform and link-hub hosts skipped); the host alone is what
381
+ identity checks use, the URL is what the classifier and funnels need."""
382
+ for value in values:
383
+ if not isinstance(value, str) or not value.strip():
384
+ continue
385
+ url = value.strip()
386
+ parsed = urllib.parse.urlsplit(url if "//" in url else "https://" + url)
387
+ host = (parsed.hostname or "").lower()
388
+ if host in CF.REDIRECT_WRAPPER_HOSTS:
389
+ wrapped = urllib.parse.parse_qs(parsed.query).get("u") or urllib.parse.parse_qs(parsed.query).get("q")
390
+ if not wrapped:
391
+ continue
392
+ parsed = urllib.parse.urlsplit(urllib.parse.unquote(wrapped[0]))
393
+ if parsed.scheme not in ("http", "https") or not usable_destination(urllib.parse.urlunsplit(parsed)):
394
+ continue
395
+ return urllib.parse.urlunsplit(parsed)[:2000]
396
+ return None
397
+
398
+
399
+ def _days(first: str | None, last: str | None) -> int | None:
400
+ return CF._days_between(first, last)
401
+
402
+
403
+ def _alias(taxonomy: dict[str, Any], dimension: str, platform: str, raw: Any) -> str | None:
404
+ if not isinstance(raw, str) or not raw:
405
+ return None
406
+ return (taxonomy.get("aliases", {}).get(dimension) or {}).get(f"{platform}:{raw}")
407
+
408
+
409
+ def _creative(*, domain: str, platform: str, source: str, ad_id: str, advertiser: str | None, headline: str | None,
410
+ body: str | None, extra_texts: list[str] | None = None, cta_label: str | None = None, cta_code: str | None = None,
411
+ landing_url: str | None = None, format_raw: str | None = None, image_url: str | None = None,
412
+ first_seen: str | None, last_seen: str | None, served_days: int | None, running: bool | None,
413
+ age_basis: str, market_scope: str, verified_by: str, link: str | None, extra: dict[str, Any] | None = None,
414
+ taxonomy: dict[str, Any]) -> dict[str, Any]:
415
+ extra_texts = [t for t in (extra_texts or []) if t]
416
+ row = {
417
+ "type": "creative",
418
+ "creative_id": f"{platform}:{ad_id}",
419
+ "domain": domain,
420
+ "platform": platform,
421
+ "channel": PLATFORM_CHANNELS.get(platform, platform),
422
+ "source": source,
423
+ "ad_id": ad_id,
424
+ "competitor_tier": "unknown", "relevance": "unknown", "country": None, "countries": [],
425
+ "duration_scope": "global" if age_basis == "platform_dates" else "unknown",
426
+ "advertiser": advertiser,
427
+ "headline": headline,
428
+ "body": body,
429
+ "extra_texts": extra_texts,
430
+ "cta_label": cta_label,
431
+ "cta_code": cta_code,
432
+ "cta_intent": _alias(taxonomy, "cta_intent", platform, cta_code),
433
+ "landing_url": landing_url,
434
+ "landing_host": registrable_domain(landing_url) if landing_url else None,
435
+ "format_raw": format_raw,
436
+ "format_kind": _alias(taxonomy, "format_kind", platform, format_raw),
437
+ "image_url": image_url,
438
+ "first_seen": first_seen,
439
+ "last_seen": last_seen,
440
+ "served_days": served_days,
441
+ "running_at_capture": running,
442
+ "age_basis": age_basis,
443
+ "market_scope": market_scope,
444
+ "verified_by": verified_by,
445
+ "link": link,
446
+ "fingerprint": fingerprint(headline, body, image_url, f"{platform}:{ad_id}"),
447
+ "tier": "A",
448
+ }
449
+ row.update(extra or {})
450
+ return row
451
+
452
+
453
+ # ── collectors ──────────────────────────────────────────────────────────────
454
+
455
+ def _google_identity(rows: list[dict[str, Any]], domain: str) -> tuple[set[str], str]:
456
+ """Which advertisers of a domain's Transparency rows are the competitor.
457
+ Names that are (or contain) the brand win; when none does, the verified
458
+ advertiser behind at least three rows and three quarters of the domain's
459
+ rows is the competitor (its legal entity, e.g. Ryzac, Inc. for
460
+ codecademy.com); otherwise none is. The rest are third parties."""
461
+ brand = brand_token(domain)
462
+ by_advertiser: dict[str, list[str]] = {}
463
+ for ad in rows:
464
+ key = str(ad.get("advertiserId") or ad.get("advertiserName") or "").strip()
465
+ by_advertiser.setdefault(key, []).append(str(ad.get("advertiserName") or "").strip())
466
+ named = {key for key, names in by_advertiser.items() if any(CF._name_matches(n, brand) or CF._mentions(n, brand) for n in names if n)}
467
+ if named:
468
+ return named, "destination_and_advertiser"
469
+ if not by_advertiser:
470
+ return set(), "destination_and_advertiser"
471
+ dominant, ads = sorted(by_advertiser.items(), key=lambda kv: (-len(kv[1]), kv[0]))[0]
472
+ total = sum(len(names) for names in by_advertiser.values())
473
+ if len(ads) < GOOGLE_DOMINANT_MIN_ROWS or len(ads) < GOOGLE_DOMINANT_MIN_SHARE * total:
474
+ return set(), "destination_and_advertiser"
475
+ return {dominant}, "destination_dominant_advertiser"
476
+
477
+
478
+ def collect_google_transparency(client: OsClient, spec: dict[str, Any], domains: list[str], as_of: date, taxonomy: dict[str, Any], excluded: list[dict[str, Any]]) -> list[dict[str, Any]]:
479
+ """Ads Transparency Center by destination domain: format, first and last
480
+ shown, and the rendered image for text and image ads. Destination keying
481
+ does not prove the advertiser, so third-party advertisers are excluded."""
482
+ rows: list[dict[str, Any]] = []
483
+ batch = max(1, int(spec.get("batch_domains") or 5))
484
+ for start in range(0, len(domains), batch):
485
+ chunk = domains[start:start + batch]
486
+ result, retrieved_on = CF._call_with_context(client, ", ".join(chunk), "Google Ads Transparency", lambda chunk=chunk: client.run_actor(
487
+ spec["actor"], {"domains": chunk, "resultType": "ads", "region": "ANYWHERE", "maxAdsPerSearch": GOOGLE_ADS_PER_DOMAIN},
488
+ GOOGLE_ADS_PER_DOMAIN * len(chunk),
489
+ ))
490
+ by_domain: dict[str, list[dict[str, Any]]] = {}
491
+ seen: set[str] = set()
492
+ for ad in result:
493
+ domain = registrable_domain(str(ad.get("domain") or ""))
494
+ creative_id = str(ad.get("creativeId") or "")
495
+ if domain in chunk and creative_id and creative_id not in seen:
496
+ seen.add(creative_id)
497
+ by_domain.setdefault(domain, []).append(ad)
498
+ for domain in chunk:
499
+ ads = by_domain.get(domain) or []
500
+ own, verified_by = _google_identity(ads, domain)
501
+ third_party = sorted({str(ad.get("advertiserName") or "")[:80] for ad in ads
502
+ if str(ad.get("advertiserId") or ad.get("advertiserName") or "").strip() not in own})
503
+ for ad in ads:
504
+ if str(ad.get("advertiserId") or ad.get("advertiserName") or "").strip() not in own:
505
+ continue
506
+ creative_id = str(ad["creativeId"])
507
+ first = str(ad.get("firstShown") or "")[:10] or None
508
+ last = str(ad.get("lastShown") or "")[:10] or None
509
+ served = ad.get("shownForDays") if isinstance(ad.get("shownForDays"), int) else _days(first, last)
510
+ fmt = str(ad.get("format") or "").upper() or None
511
+ platform = "youtube" if fmt == "VIDEO" else "google"
512
+ image = ad.get("imageUrl") if fmt in ("TEXT", "IMAGE") and isinstance(ad.get("imageUrl"), str) else None
513
+ gap = _days(last, as_of.isoformat()) if last else None
514
+ rows.append(_creative(
515
+ domain=domain, platform=platform, source="google_transparency", ad_id=creative_id,
516
+ advertiser=str(ad.get("advertiserName") or "").strip() or None, headline=None, body=None, format_raw=fmt, image_url=image,
517
+ first_seen=first, last_seen=last, served_days=served, running=None,
518
+ age_basis="platform_dates", market_scope="global", verified_by=verified_by,
519
+ link=ad.get("adUrl") or None, taxonomy=_google_taxonomy(taxonomy, platform), extra={"retrieved_on": retrieved_on},
520
+ ))
521
+ if third_party:
522
+ excluded.append({"type": "coverage", "domain": domain, "source": "google_transparency", "status": "excluded",
523
+ "detail": f"{len(third_party)} third-party advertiser(s) pointing at the domain: {', '.join(third_party[:4])}", "tier": "C"})
524
+ return rows
525
+
526
+
527
+ def _google_taxonomy(taxonomy: dict[str, Any], platform: str) -> dict[str, Any]:
528
+ # YouTube creatives come from the Google library and use its format codes.
529
+ if platform != "youtube":
530
+ return taxonomy
531
+ aliases = dict(taxonomy.get("aliases") or {})
532
+ aliases["format_kind"] = {**(aliases.get("format_kind") or {}), "youtube:VIDEO": "video"}
533
+ return {**taxonomy, "aliases": aliases}
534
+
535
+
536
+ def collect_google_search(client: OsClient, domain: str, country: str, as_of: date, taxonomy: dict[str, Any]) -> list[dict[str, Any]]:
537
+ """Preserve native SpyFu ads; capture spans describe sightings, not delivery."""
538
+ recent_days = SPYFU_RECENT_WINDOW_DAYS
539
+ windows = (((as_of - timedelta(days=recent_days)).isoformat(), as_of.isoformat()),
540
+ ((as_of - timedelta(days=SPYFU_EARLIER_WINDOW[1])).isoformat(),
541
+ (as_of - timedelta(days=SPYFU_EARLIER_WINDOW[0])).isoformat()))
542
+ groups: dict[str, dict[str, Any]] = {}
543
+ for since, until in windows:
544
+ response, retrieved_on = client.spyfu_ad_history(domain, country, since, until, SPYFU_ROWS_PER_WINDOW)
545
+ for row in response.get("rows") or []:
546
+ if not isinstance(row, dict) or registrable_domain(str(row.get("display_domain") or row.get("url") or "")) != domain:
547
+ continue
548
+ headline, body = _clean_text(row.get("title"), None), _clean_text(row.get("body"), None)
549
+ seen_on = row.get("seen_on")
550
+ if not (headline or body) or not isinstance(seen_on, str) or not _DATE_RE.match(seen_on):
551
+ continue
552
+ native = str(row["ad_id"]) if row.get("ad_id") is not None else None
553
+ # Without a native ID, preserve each distinct source observation.
554
+ key = native or "observation-" + POLICY.canonical_hash(row)[:20]
555
+ group = groups.setdefault(key, {"headline": headline, "body": body, "dates": [],
556
+ "observations": [], "native": native, "retrieved_on": retrieved_on})
557
+ if row not in group["observations"]:
558
+ group["observations"].append(dict(row))
559
+ group["dates"].append(seen_on)
560
+ creatives = []
561
+ recent_start = (as_of - timedelta(days=recent_days)).isoformat()
562
+ for key, group in groups.items():
563
+ first, last = min(group["dates"]), max(group["dates"])
564
+ landing = destination_url([row.get("url") for row in group["observations"]])
565
+ creatives.append(_creative(
566
+ domain=domain, platform="google", source="spyfu_ad_history", ad_id=f"search-{domain}-{key}", advertiser=domain,
567
+ headline=group["headline"], body=group["body"], landing_url=landing, format_raw="TEXT",
568
+ first_seen=first, last_seen=last, served_days=None, running=None,
569
+ age_basis="capture_dates", market_scope="target", verified_by="display_domain", link=landing, taxonomy=taxonomy,
570
+ extra={"keywords": list(dict.fromkeys(row["keyword"] for row in group["observations"] if row.get("keyword"))),
571
+ "captures": len(group["dates"]), "spyfu_ad_ids": [group["native"]] if group["native"] else [],
572
+ "native_ad_id": group["native"] or key, "country": country, "countries": [country],
573
+ "duration_scope": "unknown", "source_observations": group["observations"], "retrieved_on": group["retrieved_on"]},
574
+ ))
575
+ return creatives
576
+
577
+
578
+ def _meta_identity(domain: str, brand: str, page_name: str, destinations: list[str]) -> str | None:
579
+ """Destination first: a usable address decides; the advertiser name
580
+ decides only when the ad carries no usable destination."""
581
+ hosts = {registrable_domain(d) for d in (usable_destination(x) for x in destinations) if d}
582
+ if hosts:
583
+ return "destination" if domain in hosts else None
584
+ return "exact_name" if CF._name_matches(page_name, brand) else None
585
+
586
+
587
+ def _meta_actor_creative(ad: dict[str, Any], domain: str, brand: str, as_of: date, retrieved_on: str, taxonomy: dict[str, Any]) -> dict[str, Any] | None:
588
+ snapshot = ad.get("snapshot") if isinstance(ad.get("snapshot"), dict) else {}
589
+ cards = [c for c in (snapshot.get("cards") or []) if isinstance(c, dict)]
590
+ page_name = str(ad.get("page_name") or snapshot.get("page_name") or "")
591
+ destinations = [snapshot.get("link_url"), snapshot.get("caption")] + [c.get("link_url") for c in cards]
592
+ verified = _meta_identity(domain, brand, page_name, [d for d in destinations if isinstance(d, str)])
593
+ ad_id = str(ad.get("ad_archive_id") or ad.get("adArchiveID") or "")
594
+ if not verified or not ad_id:
595
+ return None
596
+ headline = _clean_text(snapshot.get("title"), 300) or next((_clean_text(c.get("title"), 300) for c in cards if _clean_text(c.get("title"), 300)), None)
597
+ body = _clean_text((snapshot.get("body") or {}).get("text") if isinstance(snapshot.get("body"), dict) else snapshot.get("body"), None) \
598
+ or next((_clean_text(c.get("body"), 1200) for c in cards if _clean_text(c.get("body"), 1200)), None)
599
+ extra_texts: list[str] = []
600
+ for value in [snapshot.get("link_description")] + [x for c in cards for x in (c.get("title"), c.get("body"), c.get("link_description"))]:
601
+ text = _clean_text(value, None)
602
+ if text and text not in (headline, body) and text not in extra_texts:
603
+ extra_texts.append(text)
604
+ images = snapshot.get("images") if isinstance(snapshot.get("images"), list) else []
605
+ videos = snapshot.get("videos") if isinstance(snapshot.get("videos"), list) else []
606
+ image = next((i.get("original_image_url") or i.get("resized_image_url") for i in images if isinstance(i, dict) and (i.get("original_image_url") or i.get("resized_image_url"))), None) \
607
+ or next((v.get("video_preview_image_url") for v in videos if isinstance(v, dict) and v.get("video_preview_image_url")), None) \
608
+ or next((c.get("original_image_url") or c.get("resized_image_url") or c.get("video_preview_image_url") for c in cards if c.get("original_image_url") or c.get("resized_image_url") or c.get("video_preview_image_url")), None)
609
+ landing = destination_url(destinations)
610
+ first = CF._iso_from_epoch(ad.get("start_date"))
611
+ active = bool(ad.get("is_active"))
612
+ # A resumed attempt must not accrue evidence after its immutable cutoff.
613
+ last = min(retrieved_on, as_of.isoformat()) if active else CF._iso_from_epoch(ad.get("end_date"))
614
+ fmt = str(snapshot.get("display_format") or "").upper() or None
615
+ cta_code = snapshot.get("cta_type") or next((c.get("cta_type") for c in cards if c.get("cta_type")), None)
616
+ return _creative(
617
+ domain=domain, platform="meta", source="meta_library_actor", ad_id=ad_id, advertiser=page_name or None,
618
+ headline=headline, body=body, extra_texts=extra_texts, cta_label=_clean_text(snapshot.get("cta_text"), 60),
619
+ cta_code=cta_code if isinstance(cta_code, str) else None, landing_url=landing, format_raw=fmt, image_url=image,
620
+ first_seen=first, last_seen=last, served_days=_days(first, last), running=active, age_basis="platform_dates",
621
+ market_scope="target", verified_by=verified, link=f"https://www.facebook.com/ads/library/?id={ad_id}", taxonomy=taxonomy,
622
+ extra={"collation_count": ad.get("collation_count") if isinstance(ad.get("collation_count"), int) else None,
623
+ "publisher_platforms": [p for p in (ad.get("publisher_platform") or []) if isinstance(p, str)][:6],
624
+ "retrieved_on": retrieved_on,
625
+ "video_url": next((v.get("video_hd_url") or v.get("video_sd_url") or v.get("video_url") for v in videos + cards if isinstance(v, dict) and (v.get("video_hd_url") or v.get("video_sd_url") or v.get("video_url"))), None),
626
+ "media_urls": list(dict.fromkeys(u for v in images + videos + cards if isinstance(v, dict) for u in [v.get("original_image_url") or v.get("resized_image_url") or v.get("video_preview_image_url")] if u)),
627
+ "media_cards": [{k: c.get(k) for k in ("title","body","link_description","original_image_url","resized_image_url","video_preview_image_url","video_hd_url","video_sd_url","link_url")} for c in cards]},
628
+ )
629
+
630
+
631
+ def _eu_demographics(row: dict[str, Any]) -> dict[str, Any] | None:
632
+ breakdown = row.get("age_country_gender_reach_breakdown")
633
+ if not isinstance(breakdown, list):
634
+ return None
635
+ by_age: dict[str, int] = {}
636
+ by_gender = {"male": 0, "female": 0, "unknown": 0}
637
+ for country in breakdown:
638
+ for item in (country.get("age_gender_breakdowns") or []) if isinstance(country, dict) else []:
639
+ if not isinstance(item, dict):
640
+ continue
641
+ total = sum(int(item.get(g) or 0) for g in by_gender)
642
+ by_age[str(item.get("age_range"))] = by_age.get(str(item.get("age_range")), 0) + total
643
+ for g in by_gender:
644
+ by_gender[g] += int(item.get(g) or 0)
645
+ total = sum(by_gender.values())
646
+ if not total:
647
+ return None
648
+ top_ages = sorted(by_age.items(), key=lambda kv: -kv[1])[:2]
649
+ return {"top_age_ranges": [age for age, _ in top_ages], "female_share": round(by_gender["female"] / total, 2),
650
+ "reach_counted": total, "target_ages": row.get("target_ages") if isinstance(row.get("target_ages"), list) else None,
651
+ "target_gender": row.get("target_gender") if isinstance(row.get("target_gender"), str) else None}
652
+
653
+
654
+ def _meta_official_creative(row: dict[str, Any], domain: str, brand: str, as_of: date, retrieved_on: str, taxonomy: dict[str, Any]) -> dict[str, Any] | None:
655
+ def first_text(key: str, cap: int) -> str | None:
656
+ values = row.get(key)
657
+ values = [values] if isinstance(values, str) else (values if isinstance(values, list) else [])
658
+ return next((_clean_text(v, cap) for v in values if _clean_text(v, cap)), None)
659
+
660
+ captions = row.get("ad_creative_link_captions")
661
+ captions = [captions] if isinstance(captions, str) else [c for c in (captions or []) if isinstance(c, str)]
662
+ verified = _meta_identity(domain, brand, str(row.get("page_name") or ""), captions)
663
+ ad_id = str(row.get("id") or "")
664
+ if not verified or not ad_id.isdigit():
665
+ return None
666
+ first = str(row.get("ad_delivery_start_time") or "")[:10] or None
667
+ stop = str(row.get("ad_delivery_stop_time") or "")[:10] or None
668
+ last = stop or min(retrieved_on, as_of.isoformat())
669
+ return _creative(
670
+ domain=domain, platform="meta", source="meta_library_official_eu", ad_id=ad_id, advertiser=row.get("page_name"),
671
+ headline=first_text("ad_creative_link_titles", 300), body=first_text("ad_creative_bodies", None),
672
+ extra_texts=[t for key in ("ad_creative_bodies", "ad_creative_link_titles", "ad_creative_link_descriptions") for d in (row.get(key) if isinstance(row.get(key), list) else [row.get(key)]) if (t := _clean_text(d, None))], landing_url=None, format_raw=None,
673
+ first_seen=first, last_seen=last, served_days=_days(first, last), running=stop is None, age_basis="platform_dates",
674
+ market_scope="eu_proxy", verified_by=verified,
675
+ # The public permalink; the Graph render URL carries an access token.
676
+ link=f"https://www.facebook.com/ads/library/?id={ad_id}", taxonomy=taxonomy,
677
+ extra={"eu_total_reach": row.get("eu_total_reach") if isinstance(row.get("eu_total_reach"), int) else None,
678
+ "eu_demographics": _eu_demographics(row),
679
+ "publisher_platforms": [p for p in (row.get("publisher_platforms") or []) if isinstance(p, str)][:6],
680
+ "retrieved_on": retrieved_on},
681
+ )
682
+
683
+
684
+ META_EU_FIELDS = ",".join((
685
+ "id", "page_id", "page_name", "ad_delivery_start_time", "ad_delivery_stop_time", "ad_creative_bodies",
686
+ "ad_creative_link_titles", "ad_creative_link_descriptions", "ad_creative_link_captions", "publisher_platforms",
687
+ "languages", "eu_total_reach", "age_country_gender_reach_breakdown", "target_ages", "target_gender",
688
+ "target_locations", "beneficiary_payers",
689
+ ))
690
+
691
+
692
+ def collect_meta(client: OsClient, spec: dict[str, Any], domain: str, as_of: date, eu_country: str, taxonomy: dict[str, Any]) -> tuple[list[dict[str, Any]], int]:
693
+ """The creatives plus the number of US page-search rows: fewer rows than
694
+ requested means the listing was complete, the only case in which an ad
695
+ missing at a later cutoff can be read as stopped."""
696
+ brand = brand_token(domain)
697
+ # Running ads only: an all-status sample of 30 is mostly stopped ads of
698
+ # global brands; stopped ones come from the EU data and earlier snapshots.
699
+ search_url = CF.meta_search_url(brand, "US", active_status="active")
700
+ result, retrieved_on = CF._call_with_context(client, domain, "Meta Ad Library page search", lambda: client.run_actor(
701
+ spec["actor"], {"urls": [{"url": search_url}], "count": META_ADS_PER_BRAND}, META_ADS_PER_BRAND,
702
+ ))
703
+ rows = []
704
+ seen = set()
705
+ for ad in result:
706
+ creative = _meta_actor_creative(ad, domain, brand, as_of, retrieved_on, taxonomy)
707
+ if creative and creative["ad_id"] not in seen:
708
+ seen.add(creative["ad_id"])
709
+ creative.update(country="US", countries=["US"], duration_scope="global")
710
+ rows.append(creative)
711
+ try:
712
+ official, eu_retrieved = CF._call_with_context(client, domain, "Meta Ad Library EU", lambda: client.meta_library_search(
713
+ {"query": brand, "countries": [eu_country], "activeStatus": "ALL", "limit": META_EU_ADS_PER_BRAND, "fields": META_EU_FIELDS},
714
+ ))
715
+ except BudgetExhausted as exc:
716
+ if "cumulative" not in str(exc):
717
+ raise
718
+ client.budget_gaps.append({"type": "coverage", "domain": domain, "country": eu_country, "channel": "meta_ads", "source": "meta_library_official_eu", "status": "budget_deferred", "detail": str(exc), "tier": "C"})
719
+ return rows, len(result)
720
+ seen = set()
721
+ for row in official:
722
+ creative = _meta_official_creative(row, domain, brand, as_of, eu_retrieved, taxonomy)
723
+ if creative and creative["ad_id"] not in seen:
724
+ seen.add(creative["ad_id"])
725
+ creative.update(country=eu_country, countries=[eu_country], duration_scope="global")
726
+ rows.append(creative)
727
+ # Same ID in different markets stays separate. Global delivery dates and
728
+ # EU aggregate reach must not strengthen the US observation.
729
+ return rows, len(result)
730
+
731
+
732
+ def collect_linkedin(client: OsClient, spec: dict[str, Any], domain: str, slug: str | None, taxonomy: dict[str, Any]) -> list[dict[str, Any]]:
733
+ query = f"https://www.linkedin.com/company/{slug or brand_token(domain)}"
734
+ result, retrieved_on = CF._call_with_context(client, domain, "LinkedIn Ad Library", lambda: client.run_actor(
735
+ spec["actor"], {"queries": [query], "maxResults": LINKEDIN_ADS_PER_COMPANY, "enrich": True}, LINKEDIN_ADS_PER_COMPANY,
736
+ ))
737
+ rows: list[dict[str, Any]] = []
738
+ seen: set[str] = set()
739
+ for ad in result:
740
+ ad_id = str(ad.get("ad_id") or "")
741
+ if not ad_id or ad_id in seen or str(ad.get("query") or "").rstrip("/") != query \
742
+ or not CF._name_matches(str(ad.get("advertiser_name") or ""), brand_token(domain)):
743
+ continue
744
+ seen.add(ad_id)
745
+ images = [u for u in (ad.get("image_urls") or []) if isinstance(u, str)]
746
+ rows.append(_creative(
747
+ domain=domain, platform="linkedin", source="linkedin_library_actor", ad_id=ad_id, advertiser=ad.get("advertiser_name"),
748
+ headline=_clean_text(ad.get("headline"), 300), body=_clean_text(ad.get("ad_copy"), 1200), format_raw=str(ad.get("creative_format") or "") or None,
749
+ image_url=images[0] if images else None, first_seen=None, last_seen=None, served_days=None, running=None,
750
+ age_basis="none", market_scope="global", verified_by="company_page_query", link=ad.get("detail_url"), taxonomy=taxonomy,
751
+ extra={"retrieved_on": retrieved_on},
752
+ ))
753
+ return rows
754
+
755
+
756
+ def collect_reddit(client: OsClient, spec: dict[str, Any], domain: str, taxonomy: dict[str, Any]) -> list[dict[str, Any]]:
757
+ brand = brand_token(domain)
758
+ result, retrieved_on = CF._call_with_context(client, domain, "Reddit ad library", lambda: client.run_actor(
759
+ spec["actor"], {"query": brand, "maxItems": REDDIT_ADS_PER_BRAND}, REDDIT_ADS_PER_BRAND,
760
+ ))
761
+ rows: list[dict[str, Any]] = []
762
+ for ad in result:
763
+ ad_id = str(ad.get("id") or ad.get("creativeId") or "")
764
+ advertiser = str(ad.get("advertiserName") or "")
765
+ verified = _meta_identity(domain, brand, advertiser.removeprefix("u_"), [ad.get("destinationUrl") or ""])
766
+ if not verified or not ad_id:
767
+ continue
768
+ created = str(ad.get("createdAt") or "")[:10] or None
769
+ rows.append(_creative(
770
+ domain=domain, platform="reddit", source="reddit_library_actor", ad_id=ad_id, advertiser=advertiser or None,
771
+ headline=_clean_text(ad.get("headline"), 300), body=_clean_text(ad.get("body"), 1200),
772
+ cta_label=_clean_text(ad.get("callToAction"), 60), landing_url=destination_url([ad.get("destinationUrl")]),
773
+ format_raw=str(ad.get("creativeType") or "") or None, image_url=ad.get("mediaUrl") or ad.get("thumbnailUrl"),
774
+ first_seen=created, last_seen=created, served_days=None, running=None, age_basis="creation_only", market_scope="global",
775
+ verified_by=verified, link=ad.get("postUrl"), taxonomy=taxonomy,
776
+ extra={"objective": ad.get("objective"), "budget_category": ad.get("budgetCategory"),
777
+ "placements": [p for p in (ad.get("placements") or []) if isinstance(p, str)][:4], "retrieved_on": retrieved_on},
778
+ ))
779
+ return rows
780
+
781
+
782
+ # ── longevity and selection ─────────────────────────────────────────────────
783
+
784
+ def cap_at_cutoff(creatives: list[dict[str, Any]], as_of: date) -> list[dict[str, Any]]:
785
+ """No evidence after the cutoff, whatever the source: an ad first seen
786
+ after it did not exist yet, and one last seen after it was still running
787
+ at the cutoff, aged up to the cutoff only. Matters for an explicit
788
+ past --as-of and for calls made when a cutoff resumes on a later day."""
789
+ cutoff = as_of.isoformat()
790
+ kept: list[dict[str, Any]] = []
791
+ for creative in creatives:
792
+ first, last = creative.get("first_seen"), creative.get("last_seen")
793
+ if first and first > cutoff:
794
+ continue
795
+ if last and last > cutoff:
796
+ span = _days(first, cutoff) if first else None
797
+ served = creative.get("served_days")
798
+ if isinstance(served, int) and isinstance(span, int):
799
+ served = min(served, span)
800
+ elif isinstance(span, int):
801
+ served = span
802
+ creative = {**creative, "last_seen": cutoff, "served_days": served, "running_at_capture": True}
803
+ kept.append(creative)
804
+ return kept
805
+
806
+
807
+ def longevity_status(creative: dict[str, Any], long_running_days: int) -> str:
808
+ return MODEL.status(creative, long_running_days)
809
+
810
+
811
+ def deduplicate(creatives: list[dict[str, Any]], long_running_days: int) -> list[dict[str, Any]]:
812
+ return MODEL.individual_ads(creatives, long_running_days)
813
+
814
+
815
+ def load_snapshots(runtime: Path, as_of: date) -> list[tuple[date, dict[str, Any]]]:
816
+ """Snapshots stored at earlier cutoffs (`--refresh` runs), oldest first."""
817
+ history = []
818
+ for path in sorted((runtime / "snapshots").glob("*.json")):
819
+ try:
820
+ data = json.loads(path.read_text(encoding="utf-8"))
821
+ cutoff = date.fromisoformat(data["as_of"])
822
+ except (OSError, ValueError, KeyError, json.JSONDecodeError):
823
+ continue
824
+ if cutoff < as_of and isinstance(data.get("creatives"), list):
825
+ data["creatives"] = [c for c in data["creatives"] if isinstance(c, dict) and c.get("fingerprint") and c.get("creative_id")]
826
+ history.append((cutoff, data))
827
+ return history
828
+
829
+
830
+ def _complete_listing(listings: dict[str, Any], domain: str) -> bool:
831
+ rows = (listings.get(domain) or {}).get("meta_us_rows")
832
+ return isinstance(rows, int) and not isinstance(rows, bool) and 0 <= rows < META_ADS_PER_BRAND
833
+
834
+
835
+ def merge_snapshots(creatives: list[dict[str, Any]], history: list[tuple[date, dict[str, Any]]], as_of: date,
836
+ long_running_days: int, domains: list[str], listings: dict[str, Any]) -> list[dict[str, Any]]:
837
+ """Fold earlier cutoffs into this one, claiming only what the samples prove.
838
+
839
+ - Libraries that keep stopped ads listed (LinkedIn, Reddit) get a listing
840
+ history (`first_listed`, `snapshot_sightings`), never an age or a
841
+ running status.
842
+ - A US Meta ad that was running at an earlier cutoff and is missing now
843
+ has stopped (the US library lists active ads only), but only when this
844
+ cutoff queried that competitor's Meta listing and both listings were
845
+ complete (fewer rows than requested); a missing ad in a truncated sample
846
+ may simply be outside it. It comes back once, as `vanished_after` the
847
+ last cutoff that saw it.
848
+ """
849
+ if not history:
850
+ return creatives
851
+ seen_at: dict[tuple, list[date]] = {}
852
+ for cutoff, snapshot in history:
853
+ for record in snapshot["creatives"]:
854
+ key = (record["domain"], record["platform"], record.get("native_ad_id") or record["creative_id"], record.get("country"))
855
+ if cutoff not in seen_at.setdefault(key, []):
856
+ seen_at[key].append(cutoff)
857
+ current_ids = {c["creative_id"] for c in creatives}
858
+ current_keys = set()
859
+ for c in creatives:
860
+ key = (c["domain"], c["platform"], c.get("native_ad_id") or c["creative_id"], c.get("country"))
861
+ current_keys.add(key)
862
+ dates = seen_at.get(key)
863
+ if not dates:
864
+ continue
865
+ c["snapshot_sightings"] = len(dates) + 1
866
+ c["first_listed"] = min(dates).isoformat()
867
+ if c.get("source") in LISTS_STOPPED_ADS:
868
+ continue # listed is not delivered: no age, no running status
869
+ vanished: dict[str, dict[str, Any]] = {}
870
+ for cutoff, snapshot in history:
871
+ for record in snapshot["creatives"]:
872
+ domain = record["domain"]
873
+ key = (domain, record["platform"], record.get("native_ad_id") or record["creative_id"], record.get("country"))
874
+ if (record.get("source") != "meta_library_actor" or record.get("running_at_capture") is not True
875
+ or key in current_keys or record["creative_id"] in current_ids or domain not in domains
876
+ or not _complete_listing(listings, domain) or not _complete_listing(snapshot.get("listings") or {}, domain)):
877
+ continue
878
+ row = dict(record)
879
+ last = max(seen_at[key]).isoformat()
880
+ row.update(running_at_capture=False, vanished_after=last, snapshot_sightings=len(seen_at[key]),
881
+ last_observed=last)
882
+ # Listing sightings never extend native delivery dates or duration.
883
+ row["longevity_status"] = longevity_status(row, long_running_days)
884
+ vanished[record["creative_id"]] = row # the latest cutoff's record wins
885
+ return creatives + [vanished[k] for k in sorted(vanished)]
886
+
887
+
888
+ def write_snapshot(runtime: Path, as_of: date, creatives: list[dict[str, Any]], listings: dict[str, Any], domains: list[str],
889
+ sources: list[str]) -> None:
890
+ _write_atomic(runtime / "snapshots" / f"{as_of.isoformat()}.json",
891
+ json.dumps({"as_of": as_of.isoformat(), "domains": domains, "sources": sources, "listings": listings,
892
+ "creatives": creatives}, ensure_ascii=False))
893
+
894
+
895
+ def select_for_classification(creatives: list[dict[str, Any]]) -> list[dict[str, Any]]:
896
+ groups: dict[tuple, list] = {}
897
+ for c in creatives:
898
+ if c.get("relevance") == "irrelevant" or c.get("language_status") == "excluded":
899
+ continue
900
+ if c.get("headline") or c.get("body") or c.get("extra_texts") or c.get("image_url") or c.get("transcript") or c.get("media_analysis") or c.get("video_metadata"):
901
+ groups.setdefault((c.get("country") or "", c.get("channel") or c.get("platform") or "unknown", c.get("creative_type")), []).append(c)
902
+ for rows in groups.values():
903
+ rows.sort(key=MODEL.strength_key)
904
+ selected = []
905
+ while groups:
906
+ for key in sorted(list(groups), key=lambda k: (MODEL.strength_key(groups[k][0])[0], groups[k][0].get("channel_priority", 999), str(k))):
907
+ selected.append(groups[key].pop(0))
908
+ if not groups[key]:
909
+ del groups[key]
910
+ return selected
911
+
912
+
913
+ # ── classification ──────────────────────────────────────────────────────────
914
+
915
+ def classification_extra_texts(creative: dict) -> list[tuple[str, dict]]:
916
+ origins = creative.get("extra_text_sources") or []
917
+ extra = [(text, dict(origins[index]) if index < len(origins) and isinstance(origins[index], dict)
918
+ else {"kind": "ad_text", "ref": creative["creative_id"]})
919
+ for index, text in enumerate(creative.get("extra_texts") or []) if isinstance(text, str)]
920
+ metadata = creative.get("video_metadata") or {}
921
+ video_ref = f"youtube:{creative.get('youtube_video_id') or metadata.get('video_id') or creative['creative_id']}"
922
+ for field in ("title", "description"):
923
+ if isinstance(metadata.get(field), str) and metadata[field].strip():
924
+ extra.append((metadata[field], {"kind": f"video_metadata_{field}", "ref": video_ref}))
925
+ for frame in (creative.get("media_analysis") or {}).get("frames") or []:
926
+ extra.extend((text, {"kind": "video_frame_ocr", "ref": frame.get("ref") or creative["creative_id"]})
927
+ for text in frame.get("visible_text") or [] if isinstance(text, str))
928
+ return extra
929
+
930
+
931
+ def _classify_input(creative: dict[str, Any], frame_ref: str | None) -> dict[str, Any]:
932
+ item: dict[str, Any] = {"ad_ref": creative["creative_id"], "platform": "google" if creative["platform"] == "youtube" else creative["platform"]}
933
+ for key, cap in (("advertiser", 200), ("headline", 600), ("body", 4000), ("cta_label", 100), ("landing_url", 2000)):
934
+ value = creative.get(key)
935
+ if isinstance(value, str) and value.strip():
936
+ item[key] = value[:cap]
937
+ if isinstance(creative.get("transcript"), str) and creative["transcript"].strip():
938
+ item["transcript"] = creative["transcript"][:8000]
939
+ if isinstance(creative.get("video_sha256"), str) and re.fullmatch(r"[0-9a-f]{64}", creative["video_sha256"]):
940
+ item["video_sha256"] = creative["video_sha256"]
941
+ analysis = creative.get("media_analysis")
942
+ if isinstance(analysis, dict) and analysis:
943
+ derived = {
944
+ "schema_version": 1, "derived": True, "truncated": False,
945
+ "source": "media/analyze", "source_input_sha256": (creative.get("video_analysis_receipt") or {}).get("input_sha256"),
946
+ "video_analysis_status": creative.get("video_analysis_status"),
947
+ "frames": [{"ref": frame.get("ref"), "observations": frame.get("observations") or []}
948
+ for frame in analysis.get("frames") or [] if isinstance(frame, dict)],
949
+ "sequence_summary": analysis.get("sequence_summary"),
950
+ "visual_claims": [claim for claim in analysis.get("claims") or []
951
+ if isinstance(claim, dict) and claim.get("modality") == "visual"],
952
+ "gaps": analysis.get("gaps") or [],
953
+ }
954
+ encoded = json.dumps(derived, ensure_ascii=False, sort_keys=True)
955
+ if len(encoded) > 12000:
956
+ # Keep the wire payload valid JSON; never silently slice a JSON
957
+ # object or present derived descriptions as original ad words.
958
+ wrapped = {"schema_version": 1, "derived": True, "truncated": True,
959
+ "source": "media/analyze", "original_chars": len(encoded),
960
+ "original_sha256": hashlib.sha256(encoded.encode()).hexdigest(),
961
+ "observations_excerpt": encoded[:11000]}
962
+ shortened = json.dumps(wrapped, ensure_ascii=False, sort_keys=True)
963
+ while len(shortened) > 12000:
964
+ wrapped["observations_excerpt"] = wrapped["observations_excerpt"][:max(0, len(wrapped["observations_excerpt"]) - (len(shortened) - 12000) - 1)]
965
+ shortened = json.dumps(wrapped, ensure_ascii=False, sort_keys=True)
966
+ encoded = shortened
967
+ item["video_observations"] = encoded
968
+ extra = classification_extra_texts(creative)
969
+ if extra:
970
+ item["extra_texts"] = [text[:4000] for text, _ in extra[:16]]
971
+ item["extra_text_sources"] = [source for _, source in extra[:16]]
972
+ item["country"] = creative.get("country")
973
+ item["allowed_languages"] = creative.get("allowed_languages")
974
+ if creative.get("format_kind"):
975
+ item["format_hint"] = creative["format_kind"]
976
+ if creative.get("cta_intent"):
977
+ item["cta_hint"] = creative["cta_intent"]
978
+ if frame_ref:
979
+ item["frame_refs"] = [frame_ref]
980
+ return item
981
+
982
+
983
+ def _label_key(item: dict[str, Any], image: dict[str, Any] | None, taxonomy_sha256: str, product_context: str, semantic_context: dict | None = None, source_content_sha256: str | None = None) -> str:
984
+ """Per-creative cache key: the classification depends only on the ad's
985
+ content, its image, the taxonomy and the product context."""
986
+ content = {k: v for k, v in item.items() if k not in ("ad_ref", "frame_refs")}
987
+ value = {"kind": "creative_label", "classification_contract": 4, "taxonomy_sha256": taxonomy_sha256, "context_sha256": hashlib.sha256(product_context.encode("utf-8")).hexdigest(),
988
+ "semantic_context": semantic_context, "source_content_sha256": source_content_sha256,
989
+ "content": content, "image_sha256": image["sha256"] if image else None}
990
+ return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False).encode("utf-8")).hexdigest()
991
+
992
+
993
+ def classification_source_hash(creative: dict) -> str:
994
+ return POLICY.canonical_hash({name: creative.get(name) for name in (
995
+ "headline", "body", "extra_texts", "extra_text_sources", "transcript", "video_metadata", "media_analysis", "video_sha256")})
996
+
997
+
998
+ def classify_corpus(client: OsClient, creatives: list[dict[str, Any]], taxonomy: dict[str, Any], taxonomy_sha256: str,
999
+ product_context: str, coverage: list[dict[str, Any]], cached_only: list[dict[str, Any]] = ()) -> list[dict[str, Any]]:
1000
+ """Labels for `creatives` (cached per creative, new ones in batches) and,
1001
+ from the cache only, for `cached_only` (ads that stopped since an earlier
1002
+ cutoff were classified then)."""
1003
+ images: dict[str, dict[str, Any]] = {}
1004
+ deferred: dict[str, set[str]] = {}
1005
+ for creative in [*creatives, *cached_only]:
1006
+ url = creative.get("image_url")
1007
+ if isinstance(url, str) and url.startswith("https://"):
1008
+ image = client.fetch_image(url)
1009
+ if image and image.get("transient"):
1010
+ # Classifying it text-only now would cache a weaker label; it waits for the image.
1011
+ deferred.setdefault(creative["domain"], set()).add(creative["creative_id"])
1012
+ elif image:
1013
+ images[creative["creative_id"]] = image
1014
+ for domain, ids in sorted(deferred.items()):
1015
+ CF._coverage_gap(coverage, "classify", [domain], f"{len(ids)} image(s) could not be downloaded now; those creatives are classified on the next attempt")
1016
+
1017
+ def frame_ref(creative: dict[str, Any]) -> str:
1018
+ return f"frames/{creative['creative_id'].replace(':', '-')}.img"
1019
+
1020
+ labels: dict[str, tuple[dict[str, Any], dict[str, Any]]] = {}
1021
+ pending: list[tuple[dict[str, Any], str]] = []
1022
+ allowed_new = {c["creative_id"] for c in creatives}
1023
+ for creative in [*creatives, *cached_only]:
1024
+ cid = creative["creative_id"]
1025
+ if cid in labels or any(cid in ids for ids in deferred.values()):
1026
+ continue
1027
+ has_image = cid in images
1028
+ analyzed_frame_text = any(frame.get("visible_text") for frame in (creative.get("media_analysis") or {}).get("frames") or [])
1029
+ if not has_image and not (creative.get("headline") or creative.get("body") or creative.get("extra_texts")
1030
+ or creative.get("transcript") or analyzed_frame_text
1031
+ or (creative.get("video_metadata") or {}).get("title")
1032
+ or (creative.get("video_metadata") or {}).get("description")):
1033
+ continue
1034
+ item = _classify_input(creative, frame_ref(creative) if has_image else None)
1035
+ source_sha = classification_source_hash(creative)
1036
+ creative["classification_source_sha256"] = source_sha
1037
+ limits = []
1038
+ for name, maximum in (("headline", 600), ("body", 4000), ("transcript", 8000)):
1039
+ if len(creative.get(name) or "") > maximum:
1040
+ limits.append(f"{name}: first {maximum} characters supplied to classification")
1041
+ extra = classification_extra_texts(creative)
1042
+ if len(extra) > 16 or any(len(text) > 4000 for text, _ in extra):
1043
+ limits.append("extra texts (native copy, video metadata and OCR): at most 16 sources, 4000 characters each supplied to classification")
1044
+ if item.get("video_observations") and json.loads(item["video_observations"]).get("truncated"):
1045
+ limits.append("derived video observations: bounded JSON excerpt supplied to classification")
1046
+ if limits:
1047
+ creative["classification_input_limits"] = limits
1048
+ coverage.append({"type": "coverage", "creative_id": cid, "domain": creative["domain"], "country": creative.get("country"), "channel": creative["channel"], "source": "classify", "status": "truncated_input", "detail": "; ".join(limits) + "; full source content retained in evidence", "tier": "C"})
1049
+ key = _label_key(item, images.get(cid), taxonomy_sha256, product_context, client.semantic_context, source_sha)
1050
+ cached = client._cache_read(client._shared() / f"{key}.json")
1051
+ if cached is not None and isinstance(cached.get("response"), dict):
1052
+ client.cache_hits += 1
1053
+ # The answer came from one batch call: the ledger counts that call
1054
+ # (and its tokens) on every replay, so receipts stay byte-stable.
1055
+ if isinstance(cached.get("batch"), str):
1056
+ client.touched.add(cached["batch"])
1057
+ if cached["response"].get("unavailable"):
1058
+ coverage.append({"type": "coverage", "creative_id": cid, "domain": creative["domain"], "channel": creative["channel"], "country": creative.get("country"), "source": "classify", "status": "rejected", "detail": cached["response"]["unavailable"], "tier": "C"})
1059
+ continue
1060
+ labels[cid] = (cached["response"], cached.get("meta") or {})
1061
+ elif cid in allowed_new and not any(cid == c["creative_id"] for c, _ in pending):
1062
+ pending.append((creative, key))
1063
+
1064
+ text_only = [(c, k) for c, k in pending if c["creative_id"] not in images]
1065
+ with_frames = [(c, k) for c, k in pending if c["creative_id"] in images]
1066
+ def content_batches(entries, maximum):
1067
+ batch, text_chars = [], 0
1068
+ # Rich semantics and complete OCR require more response room than the
1069
+ # legacy labels. Long speech/text ads are classified individually.
1070
+ maximum = min(maximum, 4) if client.semantic_context is not None else maximum
1071
+ for creative, key in entries:
1072
+ item = _classify_input(creative, None)
1073
+ cost = sum(len(item.get(name) or "") for name in ("headline", "body", "transcript")) + sum(len(t) for t in item.get("extra_texts") or [])
1074
+ if batch and (len(batch) >= maximum or text_chars + cost > 12000):
1075
+ yield batch
1076
+ batch, text_chars = [], 0
1077
+ batch.append((creative, key))
1078
+ text_chars += cost
1079
+ if batch:
1080
+ yield batch
1081
+ batches = list(content_batches(text_only, TEXT_BATCH)) + list(content_batches(with_frames, IMAGE_BATCH))
1082
+ failures = 0
1083
+ for batch_position, batch in enumerate(batches):
1084
+ frames = {frame_ref(c): images[c["creative_id"]] for c, _ in batch if c["creative_id"] in images}
1085
+ request = {"schema_version": 1, "taxonomy_version": taxonomy["taxonomy_version"], "product_context": product_context or None,
1086
+ "creatives": [_classify_input(c, frame_ref(c) if c["creative_id"] in images else None) for c, _ in batch]}
1087
+ if client.semantic_context is not None:
1088
+ # Existing paid labels anchor canonical meanings for later batches.
1089
+ # The stable per-ad key binds the offer/language policy; each batch
1090
+ # receipt also binds the actual (possibly expanded) anchor context.
1091
+ prior_rows = [{**item, "ad_ref": cid} for cid, (item, _) in labels.items()]
1092
+ prior_by_id = {row["creative_id"]: row for row in [*creatives, *cached_only]}
1093
+ prior_labels = [copy.deepcopy({**row, "creative_id": row["ad_ref"], "with_image": row["ad_ref"] in images}) for row in prior_rows if row.get("ad_ref") in prior_by_id]
1094
+ prior_creatives = [copy.deepcopy(prior_by_id[row["creative_id"]]) for row in prior_labels]
1095
+ dictionary = SEMANTICS.apply_semantics(prior_creatives, prior_labels, client.semantic_context)
1096
+ original_counts = {field: len(dictionary[field]) for field in ("angles", "message_families")}
1097
+ for field in original_counts:
1098
+ dictionary[field] = sorted(dictionary[field], key=lambda row: (-len(row.get("members") or []), row["key"]))[:320]
1099
+ # Full observations remain in the final dictionary. Request anchors
1100
+ # obey the bounded public inference context, with visible coverage.
1101
+ while len(json.dumps(dictionary, ensure_ascii=False)) > 220000 and any(dictionary[field] for field in original_counts):
1102
+ field = max(original_counts, key=lambda name: len(json.dumps(dictionary[name], ensure_ascii=False)))
1103
+ dictionary[field].pop()
1104
+ if any(len(dictionary[field]) < original_counts[field] for field in original_counts):
1105
+ coverage.append({"type": "coverage", "source": "semantic_context", "status": "bounded_anchors", "detail": f"Request canonical anchors bounded from {original_counts}; full observations remain in output dictionary", "tier": "C"})
1106
+ request["semantic_context"] = SEMANTICS.build_semantic_context(product_context, client.semantic_context["allowed_languages"], dictionary)
1107
+ try:
1108
+ response, retrieved_on, batch_digest = client.classify(request, frames, taxonomy_sha256)
1109
+ except BudgetExhausted as exc:
1110
+ if "cumulative" not in str(exc):
1111
+ raise
1112
+ for creative, _ in [entry for remaining in batches[batch_position:] for entry in remaining]:
1113
+ coverage.append({"type": "coverage", "creative_id": creative["creative_id"], "domain": creative["domain"], "channel": creative["channel"],
1114
+ "country": creative.get("country"), "source": "classify", "status": "budget_deferred",
1115
+ "detail": str(exc), "tier": "C"})
1116
+ break
1117
+ except CreativePatternsError as exc:
1118
+ if not isinstance(exc, SourceUnavailable) and re.search(r"HTTP (?:404|405|409|503)\b", str(exc)):
1119
+ # The endpoint is not deployed, not configured or on another taxonomy: no batch can succeed.
1120
+ raise CreativePatternsError(f"creative classification is unavailable: {CF._provider_detail(exc)}") from exc
1121
+ if isinstance(exc, CF.OsUnreachable) or (not isinstance(exc, SourceUnavailable) and STOP_CLASSIFYING_RE.search(str(exc))):
1122
+ # Not about this batch: the OS is unreachable, the key is
1123
+ # refused, rate-limited or out of LLM budget, so later batches
1124
+ # would fail the same way. Recording gaps would complete the
1125
+ # section with creatives unlabelled; failing the attempt lets
1126
+ # a rerun continue from the cached answers.
1127
+ raise type(exc)(f"creative classification stopped: {CF._provider_detail(exc)}; rerun later to continue "
1128
+ f"({len(labels)} label(s) so far are cached)") from exc
1129
+ failures += 1
1130
+ if isinstance(exc, SourceUnavailable) and getattr(exc, "batch_digest", None):
1131
+ for creative, key in batch:
1132
+ client._cache_write(client._shared() / f"{key}.json", {"kind": "creative_label", "response": {"unavailable": str(exc)}, "batch": exc.batch_digest})
1133
+ CF._coverage_gap(coverage, "classify", sorted({c["domain"] for c, _ in batch}), f"a classification batch failed: {CF._provider_detail(exc)}")
1134
+ continue
1135
+ meta = {"taxonomy_version": response.get("taxonomy_version"), "taxonomy_sha256": response.get("taxonomy_sha256") or taxonomy_sha256,
1136
+ "model_role": response.get("model_role"), "effective_model": response.get("effective_model"), "classified_on": retrieved_on,
1137
+ "classification_request_context_sha256": (request.get("semantic_context") or {}).get("context_sha256")}
1138
+ by_ref = {c["creative_id"]: (c, k) for c, k in batch}
1139
+ for item in response.get("classifications") or []:
1140
+ if not isinstance(item, dict) or item.get("ad_ref") not in by_ref:
1141
+ continue
1142
+ creative, key = by_ref[item["ad_ref"]]
1143
+ labels[creative["creative_id"]] = (item, meta)
1144
+ client._cache_write(client._shared() / f"{key}.json", {"kind": "creative_label", "response": item, "meta": meta,
1145
+ "batch": batch_digest, "retrieved_on": retrieved_on})
1146
+ if batches and failures == len(batches) and not labels:
1147
+ raise CreativePatternsError("every creative classification batch failed; rerun later (no spend is lost: answers are cached)")
1148
+ if deferred and not labels:
1149
+ raise CreativePatternsError("transient image failures left no usable creative classifications; rerun later to continue from the cache")
1150
+
1151
+ rows: list[dict[str, Any]] = []
1152
+ for creative in [*creatives, *cached_only]:
1153
+ if creative["creative_id"] not in labels or any(r["creative_id"] == creative["creative_id"] for r in rows):
1154
+ continue
1155
+ item, meta = labels[creative["creative_id"]]
1156
+ rows.append({
1157
+ "type": "creative_classification",
1158
+ "creative_id": creative["creative_id"],
1159
+ "domain": creative["domain"],
1160
+ "platform": creative["platform"],
1161
+ "channel": creative["channel"],
1162
+ "fingerprint": creative["fingerprint"],
1163
+ "semantics": item.get("semantics") or {},
1164
+ "classification_request_context_sha256": meta.get("classification_request_context_sha256"),
1165
+ "classification_source_sha256": creative.get("classification_source_sha256"),
1166
+ "classification_input_limits": creative.get("classification_input_limits") or [],
1167
+ "observed_text": item.get("observed_text"),
1168
+ "observed_text_source": item.get("observed_text_source"),
1169
+ "observed_text_sources": item.get("observed_text_sources") or [],
1170
+ "texts_sources": item.get("texts_sources") or {},
1171
+ "observed_text_frame_refs": item.get("observed_text_frame_refs") or [],
1172
+ "country": creative.get("country"),
1173
+ "creative_type": creative.get("creative_type"),
1174
+ "format_kind": creative.get("format_kind"),
1175
+ "labels": item.get("labels") or {},
1176
+ "texts": item.get("texts") or {},
1177
+ "offer": item.get("offer") or {},
1178
+ "confidence": item.get("confidence"),
1179
+ "coerced": item.get("coerced") or [],
1180
+ "with_image": creative["creative_id"] in images,
1181
+ "with_video_frames": False,
1182
+ "with_analyzed_video_frames": bool(creative.get("video_frames")),
1183
+ "video_analysis_ref": (creative.get("video_analysis_receipt") or {}).get("input_sha256"),
1184
+ "input_text_sources": _classify_input(creative, None).get("extra_text_sources") or [],
1185
+ "taxonomy_version": meta.get("taxonomy_version"),
1186
+ "taxonomy_sha256": meta.get("taxonomy_sha256") or taxonomy_sha256,
1187
+ "model_role": meta.get("model_role"),
1188
+ "effective_model": meta.get("effective_model"),
1189
+ "classified_on": meta.get("classified_on"),
1190
+ "market_scope": creative["market_scope"],
1191
+ "tier": "C",
1192
+ })
1193
+ return rows
1194
+
1195
+
1196
+ # ── patterns ────────────────────────────────────────────────────────────────
1197
+
1198
+ def score_patterns(creatives: list[dict[str, Any]], classifications: list[dict[str, Any]], taxonomy: dict[str, Any], long_running_days: int, dictionary: dict | None = None) -> list[dict[str, Any]]:
1199
+ return MODEL.patterns(creatives, classifications, taxonomy, long_running_days, dictionary)
1200
+
1201
+
1202
+ def price_anchors(creatives: list[dict[str, Any]], classifications: list[dict] | None = None) -> list[dict[str, Any]]:
1203
+ return MODEL.promises(creatives, classifications)
1204
+
1205
+
1206
+ def _cell(value: Any, cap: int = 110) -> str:
1207
+ return CF._cell(value, cap)
1208
+
1209
+
1210
+ def _label(value: Any) -> str:
1211
+ return str(value).replace("_", " ") if value not in (None, "") else "—"
1212
+
1213
+
1214
+ def render_section(*, domains: list[str], creatives: list[dict[str, Any]], classifications: list[dict[str, Any]],
1215
+ patterns: list[dict[str, Any]], anchors: list[dict[str, Any]], coverage: list[dict[str, Any]],
1216
+ totals: dict[str, Any], as_of: str, retrieved_on: str, long_running_days: int, taxonomy_version: str, selection: dict | None = None) -> str:
1217
+ body = MODEL.section(creatives, patterns, anchors, coverage, long_running_days, selection)
1218
+ spend = (f"\nClassified {len(classifications)} of {len(creatives)} individual ads; "
1219
+ f"{totals['classify_calls']} classification calls and {totals['classify_prompt_tokens']} prompt tokens; "
1220
+ f"{totals.get('media_calls', 0)} media calls and {totals.get('media_prompt_tokens', 0)} media prompt tokens. "
1221
+ f"Cumulative paid starts: {totals.get('cumulative_paid_starts', 0)}/{totals.get('cumulative_paid_start_limit')}; "
1222
+ f"these are resource counts, not a currency estimate. "
1223
+ f"Cutoff {as_of}; sources retrieved {retrieved_on}; taxonomy {taxonomy_version}.\n")
1224
+ return body + spend
1225
+
1226
+
1227
+ # ── orchestration ───────────────────────────────────────────────────────────
1228
+
1229
+ def frozen_semantic_context(client: OsClient, inputs: RunInputs) -> dict:
1230
+ # Freeze anchors across compatible cutoffs. A dictionary learned during a
1231
+ # run must not change the key of already-paid labels on the next refresh.
1232
+ base = SEMANTICS.build_semantic_context(inputs.product_context, inputs.languages)
1233
+ base_sha = POLICY.canonical_hash({"context": base, "market_languages": inputs.market_languages})
1234
+ shared = client._shared() / "semantic-context.json"
1235
+ snapshot = client.cache_dir / "semantic-context.json"
1236
+ for path in (snapshot, shared):
1237
+ previous = client._cache_read(path)
1238
+ if previous and previous.get("base_sha256") == base_sha:
1239
+ _write_atomic(snapshot, json.dumps(previous, ensure_ascii=False))
1240
+ return previous["context"]
1241
+ record = {"base_sha256": base_sha, "context": base}
1242
+ _write_atomic(shared, json.dumps(record, ensure_ascii=False))
1243
+ _write_atomic(snapshot, json.dumps(record, ensure_ascii=False))
1244
+ return base
1245
+
1246
+
1247
+ def language_inputs(workspace: Path, args: argparse.Namespace, countries: list[str] | None = None) -> tuple[list[str], dict]:
1248
+ intake = CF.typed_intake(workspace)
1249
+ raw = str(_arg(args, "languages", "") or "")
1250
+ market = str(_arg(args, "market_languages", "") or "")
1251
+ languages = ([code.strip().lower() for code in raw.split(",") if code.strip()]
1252
+ if raw else list(intake.get("languages") or []))
1253
+ by_market = {} if raw or market else dict(intake.get("market_languages") or {})
1254
+ for entry in market.split(";"):
1255
+ if not entry.strip():
1256
+ continue
1257
+ if ":" not in entry:
1258
+ raise CreativePatternsError("market languages must use COUNTRY:language,language entries")
1259
+ country, codes = entry.split(":", 1)
1260
+ country = country.strip().upper().replace("UK", "GB")
1261
+ if not re.fullmatch(r"[A-Z]{2}", country):
1262
+ raise CreativePatternsError("market language country must be an ISO code")
1263
+ by_market[country] = [code.strip().lower() for code in codes.split(",") if code.strip()]
1264
+ if not by_market[country]:
1265
+ raise CreativePatternsError("market languages cannot be empty")
1266
+ if by_market:
1267
+ for country in countries or []:
1268
+ if country not in by_market:
1269
+ if not languages:
1270
+ raise CreativePatternsError(f"accepted languages are missing for market {country}; set a global list or that market explicitly")
1271
+ by_market[country] = list(languages)
1272
+ languages = sorted(set(languages + [c for values in by_market.values() for c in values]))
1273
+ SEMANTICS.build_semantic_context("", languages)
1274
+ return languages, by_market
1275
+
1276
+
1277
+ def full_product_context(workspace: Path) -> str:
1278
+ text = CF.IO.context_text(workspace, SECTION_ID)
1279
+ intake = CF.typed_intake(workspace)
1280
+ offer = {key: value for key, value in (intake.get("offer") or {}).items()
1281
+ if value not in (None, "", [])}
1282
+ if offer:
1283
+ text += "\n\nOffer card:\n" + json.dumps(offer, ensure_ascii=False)
1284
+ if len(text) > PRODUCT_CONTEXT_CHARS:
1285
+ raise CreativePatternsError("offer context exceeds 24000 characters; provide a bounded offer card rather than silently truncating it")
1286
+ return text
1287
+
1288
+
1289
+ def hydrate_footprint_media(creatives: list[dict], inputs: RunInputs, taxonomy: dict) -> None:
1290
+ if inputs.runtime is None:
1291
+ return
1292
+ by_native = {}
1293
+ for c in creatives:
1294
+ by_native.setdefault(c["native_ad_id"], []).append(c)
1295
+ for path in CF.IO.cache_entries(inputs.runtime.parent / "channel_footprint/cache"):
1296
+ try:
1297
+ entry = CF.IO.read_cache_entry(path)
1298
+ except (ValueError, OSError, EOFError):
1299
+ continue
1300
+ if not isinstance(entry, dict):
1301
+ continue
1302
+ captured = entry.get("cutoff") or entry.get("retrieved_on")
1303
+ if isinstance(captured, str) and captured[:10] > inputs.as_of.isoformat():
1304
+ continue # Future media must not enrich a historical cutoff.
1305
+ response = entry.get("response") or {}
1306
+ rows = response.get("rows") or response.get("data") or []
1307
+ if not isinstance(rows, list):
1308
+ continue
1309
+ for raw in rows:
1310
+ if not isinstance(raw, dict):
1311
+ continue
1312
+ native = str(raw.get("ad_archive_id") or raw.get("creativeId") or raw.get("adId") or raw.get("id") or "")
1313
+ for c in by_native.get(native, []):
1314
+ if c["platform"] == "meta" and raw.get("snapshot"):
1315
+ recovered = _meta_actor_creative(raw, c["domain"], CF.brand_token(c["domain"]), inputs.as_of,
1316
+ entry.get("retrieved_on") or inputs.as_of.isoformat(), taxonomy)
1317
+ if recovered:
1318
+ for field in ("headline", "body", "extra_texts", "image_url", "video_url", "media_urls", "media_cards", "format_raw", "format_kind"):
1319
+ if recovered.get(field):
1320
+ c[field] = recovered[field]
1321
+ c["creative_type"] = MODEL.creative_type(c)
1322
+ elif c["platform"] in ("google", "youtube"):
1323
+ for field, native_field in (("image_url", "imageUrl"), ("video_url", "videoUrl"), ("link", "adUrl")):
1324
+ if raw.get(native_field):
1325
+ c[field] = raw[native_field]
1326
+
1327
+
1328
+ def footprint_coverage(inputs: RunInputs, creatives: list[dict]) -> list[dict]:
1329
+ document = inputs.footprint or {}
1330
+ countries = document.get("inputs", {}).get("target_countries") or [None]
1331
+ rows = []
1332
+ for rank in document.get("rows", []):
1333
+ if rank.get("type") != "channel_observation":
1334
+ continue
1335
+ channel = rank.get("channel") or rank.get("platform")
1336
+ if not channel:
1337
+ continue
1338
+ for country in [rank["country"]] if rank.get("country") else countries:
1339
+ if inputs.deepened_cells is not None and (country, channel) not in inputs.deepened_cells:
1340
+ rows.extend({"type": "coverage", "domain": domain, "country": country, "channel": channel, "source": "channel_footprint",
1341
+ "status": "not_deepened", "detail": f"Bounded scope inspects the first {inputs.max_channels} observed channel(s) in stable name order "
1342
+ "per market; use --max-channels 0 for every cell.", "rank": rank.get("rank"), "tier": "C"}
1343
+ for domain in inputs.domains)
1344
+ continue
1345
+ for domain in inputs.domains:
1346
+ items = [c for c in creatives if c["channel"] == channel and c["domain"] == domain and c.get("country") == country]
1347
+ gaps = [g for g in document.get("rows", []) if g.get("type") == "coverage" and g.get("domain") == domain
1348
+ and (g.get("channel") or g.get("platform")) == channel and g.get("country") in (country, None)]
1349
+ rows.append({"type": "coverage", "domain": domain, "country": country, "channel": channel,
1350
+ "source": "channel_footprint", "status": "extracted" if items else "not_extracted",
1351
+ "detail": f"{len(items)} native ads in available sample; not a census" if items else
1352
+ "; ".join(str(g.get("detail") or g.get("reason") or g.get("status")) for g in gaps) or "No extractable ads in completed Footprint; channel presence retained",
1353
+ "rank": rank.get("rank"), "tier": "C"})
1354
+ return rows
1355
+
1356
+
1357
+ def prepare_media(client: OsClient, creatives: list[dict], inputs: RunInputs, config: dict, coverage: list[dict], *, analyze_videos: bool = True) -> None:
1358
+ selected_videos: dict[tuple, dict[str, str]] = {}
1359
+ for c in sorted(creatives, key=MODEL.strength_key):
1360
+ c["allowed_languages"] = inputs.market_languages.get(c.get("country"), inputs.languages)
1361
+ urls = [c.get("image_url")] + list(c.get("media_urls") or [])
1362
+ for url in dict.fromkeys(urls):
1363
+ if not isinstance(url, str) or not url.startswith("https://"):
1364
+ continue
1365
+ image = client.fetch_image(url)
1366
+ if image and image.get("file") and inputs.runtime:
1367
+ workspace = VIDEO._workspace(inputs.runtime)
1368
+ path = client._shared() / "images" / image["file"]
1369
+ if path.is_relative_to(workspace):
1370
+ asset = {"path": path.relative_to(workspace).as_posix(), "kind": "preview" if c["creative_type"] == "video" else "image", "sha256": image["sha256"]}
1371
+ if asset not in c.setdefault("media_assets", []):
1372
+ c["media_assets"].append(asset)
1373
+ if c["creative_type"] != "video" or inputs.runtime is None or not analyze_videos:
1374
+ continue
1375
+ key = (c["domain"], c["channel"], c.get("country"))
1376
+ # A market observation or second native ad is not a new video. Prefer
1377
+ # actual bytes, then exact provider video ID, before native-ad fallback.
1378
+ content_id = c.get("video_sha256") or c.get("youtube_video_id") or VIDEO.youtube_id(c.get("video_id")) \
1379
+ or VIDEO.youtube_id(c.get("video_url")) or VIDEO.youtube_id(c.get("link")) \
1380
+ or c.get("video_url") or c.get("native_ad_id") or c["creative_id"]
1381
+ content_id = str(content_id)
1382
+ selected = selected_videos.setdefault(key, {})
1383
+ excluded = c.get("relevance") == "irrelevant" or c.get("language_status") == "excluded"
1384
+ if excluded or (content_id not in selected and len(set(selected.values())) >= inputs.top_videos):
1385
+ c["video_analysis_status"] = "not_selected"
1386
+ c.setdefault("video_gaps", []).append("Irrelevant or excluded-language video" if excluded else "Beyond configured top-N unique videos")
1387
+ else:
1388
+ selected.setdefault(content_id, content_id)
1389
+ try:
1390
+ c.update(VIDEO.preserve_video_media(c, inputs.runtime, config, client=client))
1391
+ c.update(VIDEO.analyze_video(client, c, inputs.runtime, config))
1392
+ # Resolved IDs/hashes let subsequent ads reuse this video's
1393
+ # shared paid receipts, including after the cell reached N.
1394
+ if c.get("video_sha256") or c.get("youtube_video_id"):
1395
+ resolved_id = str(c.get("video_sha256") or c["youtube_video_id"])
1396
+ previous_id = selected[content_id]
1397
+ for alias in list(selected):
1398
+ if selected[alias] == previous_id:
1399
+ selected[alias] = resolved_id
1400
+ selected[resolved_id] = resolved_id
1401
+ except BudgetExhausted as exc:
1402
+ c["video_analysis_status"] = "budget_deferred"
1403
+ c.setdefault("video_gaps", []).append(str(exc))
1404
+ if c.get("video_gaps"):
1405
+ coverage.append({"type": "coverage", "domain": c["domain"], "channel": c["channel"], "country": c.get("country"),
1406
+ "source": "video_analysis", "status": c.get("video_analysis_status") or "unavailable", "detail": "; ".join(c["video_gaps"]), "tier": "C"})
1407
+
1408
+
1409
+ def footprint_input(workspace: Path, policy: dict[str, Any], policy_sha256: str) -> dict[str, Any] | None:
1410
+ """A v2 footprint is an owned input, never an unverified optional hint."""
1411
+ path = workspace / CHANNEL_FOOTPRINT_EVIDENCE_REL
1412
+ if not path.is_file():
1413
+ return None
1414
+ document = json.loads(path.read_text(encoding="utf-8"))
1415
+ if document.get("schema_version") != 2:
1416
+ raise CreativePatternsError("Channel Footprint must be refreshed to its scoped v2 contract before Creative Patterns")
1417
+ reader = _load_script("research-evidence.py", "pmr_cf_inputs_for_creative_patterns")
1418
+ result = reader.inspect_section(workspace, "channel_footprint")
1419
+ if result.get("status") != "completed":
1420
+ raise CreativePatternsError("Channel Footprint input is pending, stale or unverified: " + str(result.get("reason") or result.get("status")))
1421
+ inputs = document.get("inputs") or {}
1422
+ if inputs.get("effective_policy") != policy or inputs.get("policy_sha256") != policy_sha256:
1423
+ raise CreativePatternsError("Channel Footprint evidence policy changed; finalize it with the current shared taxonomy before Creative Patterns")
1424
+ return document
1425
+
1426
+
1427
+ def footprint_creatives(document: dict[str, Any], as_of: date, taxonomy: dict[str, Any]) -> list[dict[str, Any]]:
1428
+ """Reuse decided ads with per-market lineage, without re-admitting unrelated ads."""
1429
+ context = (document.get("inputs") or {}).get("competitor_context") or {}
1430
+ policy, _ = POLICY.policy_from_taxonomy(taxonomy)
1431
+ selected_countries = set((document.get("inputs") or {}).get("target_countries") or [])
1432
+ result = []
1433
+ platforms = {"google_ads": "google", "youtube_ads": "youtube", "meta_ads": "meta", "linkedin_ads": "linkedin", "reddit_ads": "reddit", "tiktok_ads": "tiktok", "display_native": "google"}
1434
+ for ad in document.get("rows") or []:
1435
+ if ad.get("type") != "channel_ad":
1436
+ continue
1437
+ base_channel = ad.get("channel") or ad.get("platform")
1438
+ if base_channel not in platforms:
1439
+ continue
1440
+ domain_context = context.get(ad.get("domain"), {}) if isinstance(context, dict) else {}
1441
+ role = ad.get("competitor_tier") or (domain_context.get("competitor_tier") if isinstance(domain_context, dict) else None) or "unknown"
1442
+ for observation in ad.get("observations") or []:
1443
+ channel = base_channel
1444
+ if channel == "google_ads":
1445
+ placement = str(observation.get("placement") or "").lower()
1446
+ if placement in {"youtube", "youtube_ads"}:
1447
+ channel = "youtube_ads"
1448
+ elif placement in {"display", "display_ads", "display_native"}:
1449
+ channel = "display_native"
1450
+ platform = platforms.get(channel, str(channel).removesuffix("_ads"))
1451
+ country = observation.get("country")
1452
+ if country and country not in selected_countries:
1453
+ continue
1454
+ native_date_scope = observation.get("date_scope") or "unknown"
1455
+ date_country = observation.get("date_country")
1456
+ date_scope = native_date_scope
1457
+ if date_scope == "country" and (not country or date_country != country):
1458
+ date_scope = "unknown"
1459
+ first = observation.get("first_shown")
1460
+ last = observation.get("last_shown")
1461
+ activity = observation.get("activity_status", observation.get("status"))
1462
+ activity_scope = observation.get("activity_scope") or "unknown"
1463
+ activity_country = observation.get("activity_country")
1464
+ # Activity observed in this country's query (global provider flag) or
1465
+ # native country activity both establish current use in the market.
1466
+ same_activity_country = bool(country and activity_scope in ("country", "global") and activity_country == country)
1467
+ capture = observation.get("observed_on") or observation.get("last_observed") or document.get("retrieved_on") or as_of.isoformat()
1468
+ capture = str(capture).split("..")[-1][:10]
1469
+ running = (True if activity == "active" else
1470
+ False if activity in ("stopped", "inactive", "ended", "archived") else None) if same_activity_country else None
1471
+ same_delivery_scope = ((date_scope == "global" and activity_scope == "global")
1472
+ or (date_scope == "country" and activity_scope == "country" and same_activity_country))
1473
+ counted_last = last
1474
+ if activity == "active" and same_activity_country and same_delivery_scope and first and (not last or str(last)[:10] > capture):
1475
+ # Counted through the footprint capture, as Channel Footprint
1476
+ # does: a later section cannot extend delivery because time
1477
+ # elapsed, and a scheduled future end adds no days.
1478
+ counted_last = capture
1479
+ last = last or capture
1480
+ # Listing observations deliberately have no delivery age or flag.
1481
+ served = _days(first, counted_last) if first and counted_last and date_scope in ("country", "global") else None
1482
+ if date_scope in ("country", "global") and counted_last and str(counted_last)[:10] > capture:
1483
+ served = None # Future delivery is not historical proof at source capture.
1484
+ native_id = str(ad.get("native_ad_id") or ad.get("ad_id"))
1485
+ scope_key = country or "global"
1486
+ identity = f"{native_id}:{channel}:{scope_key}:{date_scope}"
1487
+ creative = _creative(
1488
+ domain=ad["domain"], platform=platform, source=str(observation.get("source") or ad.get("source") or "channel_footprint"),
1489
+ ad_id=identity, advertiser=ad.get("advertiser"), headline=ad.get("headline"), body=ad.get("ad_text"),
1490
+ landing_url=ad.get("landing_url"), format_raw=str(ad.get("format") or "unknown").upper(), image_url=ad.get("image_url"),
1491
+ first_seen=first, last_seen=last, served_days=served, running=running,
1492
+ age_basis="platform_dates" if served is not None else "none", market_scope=observation.get("market_scope") or "global",
1493
+ verified_by=ad.get("verified_by") or "channel_footprint_identity", link=ad.get("source_url"), taxonomy=_google_taxonomy(taxonomy, platform),
1494
+ extra={"country": country, "countries": [country] if country else [], "competitor_tier": role,
1495
+ "relevance": ad.get("relevance") or "unknown", "relevance_reason": ad.get("relevance_reason"), "duration_scope": date_scope,
1496
+ "video_url": ad.get("video_url"), "media_urls": ad.get("media_urls") or [],
1497
+ "media_cards": ad.get("media_cards") or [], "format_kind": _alias(_google_taxonomy(taxonomy, platform), "format_kind", platform, str(ad.get("format") or "unknown").upper()),
1498
+ "footprint_ad_id": ad.get("ad_id"), "native_ad_id": native_id, "channel": channel,
1499
+ "first_observed": observation.get("first_observed"), "last_observed": observation.get("last_observed"),
1500
+ "retrieved_on": capture,
1501
+ "date_scope": native_date_scope, "date_country": date_country,
1502
+ "activity_scope": activity_scope, "activity_country": activity_country,
1503
+ "native_activity_status": activity, "placement": observation.get("placement")},
1504
+ )
1505
+ creative["source_members"] = [{k: creative.get(k) for k in (
1506
+ "creative_id", "domain", "country", "countries", "channel", "competitor_tier", "relevance", "duration_scope",
1507
+ "footprint_ad_id", "native_ad_id", "served_days", "running_at_capture", "first_seen", "last_seen",
1508
+ "date_scope", "date_country", "activity_scope", "activity_country", "native_activity_status", "placement") }]
1509
+ result.append(creative)
1510
+ return result
1511
+
1512
+
1513
+ @dataclass
1514
+ class RunInputs:
1515
+ domains: list[str]
1516
+ domain_sources: dict[str, str]
1517
+ product_domain: str
1518
+ sources: list[str]
1519
+ country: str
1520
+ eu_country: str
1521
+ as_of: date
1522
+ long_running_days: int
1523
+ product_context: str
1524
+ linkedin_slugs: dict[str, str] = field(default_factory=dict)
1525
+ runtime: Path | None = None
1526
+ footprint: dict[str, Any] | None = None
1527
+ effective_policy: dict[str, Any] = field(default_factory=dict)
1528
+ policy_sha256: str = ""
1529
+ restrict_sources: bool = False
1530
+ languages: list[str] = field(default_factory=list)
1531
+ market_languages: dict[str, list[str]] = field(default_factory=dict)
1532
+ top_videos: int = 5
1533
+ max_channels: int = 0 # express scope: deepen this many candidate channels per market; 0 keeps every cell
1534
+ deepened_cells: set[tuple[str | None, str]] | None = None
1535
+
1536
+
1537
+ def candidate_cells(footprint: dict[str, Any], max_channels: int) -> set[tuple[str | None, str]] | None:
1538
+ """Bound an inspection sample by stable channel name, not business merit.
1539
+ None means every observed comparable cell is deepened."""
1540
+ if not max_channels:
1541
+ return None
1542
+ rows = footprint.get("rows") or []
1543
+ channel = lambda row: row.get("channel") or row.get("platform") # Footprint presence rows name it `platform`
1544
+ current = {(r.get("country"), channel(r)) for r in rows if r.get("type") == "library_presence"
1545
+ and r.get("relevance") == "relevant" and channel(r)}
1546
+ ranks = {(r.get("country"), channel(r)): r.get("rank", 999) for r in rows if r.get("type") == "channel_observation"}
1547
+ cells: set[tuple[str | None, str]] = set()
1548
+ for country in {country for country, _ in current}:
1549
+ channels = sorted((name for market, name in current if market == country), key=str)
1550
+ cells.update((country, channel) for channel in channels[:max_channels])
1551
+ return cells
1552
+
1553
+
1554
+ def run_pipeline(client: OsClient, inputs: RunInputs, config: dict[str, Any], taxonomy: dict[str, Any], taxonomy_sha256: str) -> dict[str, Any]:
1555
+ actors = config["actors"]
1556
+ coverage: list[dict[str, Any]] = []
1557
+ raw: list[dict[str, Any]] = []
1558
+ listings: dict[str, Any] = {}
1559
+ def partial_source(source, domains, fn):
1560
+ try:
1561
+ return CF._attempt(client, coverage, source, domains, fn)
1562
+ except BudgetExhausted as exc:
1563
+ if "cumulative" not in str(exc):
1564
+ raise
1565
+ for domain in domains:
1566
+ coverage.append({"type": "coverage", "domain": domain, "source": source, "status": "budget_deferred", "detail": str(exc), "tier": "C"})
1567
+ return None
1568
+ if inputs.footprint is not None:
1569
+ raw = [c for c in footprint_creatives(inputs.footprint, inputs.as_of, taxonomy) if c["domain"] in inputs.domains]
1570
+ inputs.deepened_cells = candidate_cells(inputs.footprint, inputs.max_channels)
1571
+ if inputs.deepened_cells is not None:
1572
+ raw = [c for c in raw if (c.get("country"), c["channel"]) in inputs.deepened_cells]
1573
+ if inputs.restrict_sources:
1574
+ families = {"google_ads": "google_transparency", "youtube_ads": "google_transparency", "display_native": "google_transparency",
1575
+ "meta_ads": "meta", "linkedin_ads": "linkedin", "reddit_ads": "reddit"}
1576
+ raw = [c for c in raw if families.get(c["channel"]) in inputs.sources]
1577
+ hydrate_footprint_media(raw, inputs, taxonomy)
1578
+ else:
1579
+ if "google_transparency" in inputs.sources:
1580
+ for domain in inputs.domains:
1581
+ found = partial_source("google_transparency", [domain], lambda d=domain: collect_google_transparency(client, actors["google_ads"], [d], inputs.as_of, taxonomy, coverage))
1582
+ raw.extend(found or [])
1583
+ for domain in inputs.domains:
1584
+ if "meta" in inputs.sources:
1585
+ found = partial_source("meta", [domain], lambda d=domain: collect_meta(client, actors["meta_ads"], d, inputs.as_of, inputs.eu_country, taxonomy))
1586
+ if found:
1587
+ meta_rows, us_rows = found
1588
+ raw.extend(meta_rows)
1589
+ listings[domain] = {"meta_us_rows": us_rows}
1590
+ if "google_search" in inputs.sources:
1591
+ found = partial_source("google_search", [domain], lambda d=domain: collect_google_search(client, d, inputs.country, inputs.as_of, taxonomy))
1592
+ raw.extend(found or [])
1593
+ if "linkedin" in inputs.sources:
1594
+ found = partial_source("linkedin", [domain], lambda d=domain: collect_linkedin(client, actors["linkedin_ads"], d, inputs.linkedin_slugs.get(d), taxonomy))
1595
+ raw.extend(found or [])
1596
+ if "reddit" in inputs.sources:
1597
+ found = partial_source("reddit", [domain], lambda d=domain: collect_reddit(client, actors["reddit_ads"], d, taxonomy))
1598
+ raw.extend(found or [])
1599
+ # Meta (and Reddit) media URLs are signed and expire within days:
1600
+ # download this competitor's images now, not after every collector.
1601
+ for creative in raw:
1602
+ if creative["domain"] != domain:
1603
+ continue
1604
+ if isinstance(creative.get("image_url"), str) and creative["image_url"].startswith("https://"):
1605
+ client.fetch_image(creative["image_url"])
1606
+
1607
+ coverage.extend(client.budget_gaps)
1608
+
1609
+ raw = cap_at_cutoff(raw, inputs.as_of)
1610
+ creatives = deduplicate(raw, inputs.long_running_days)
1611
+ # Native identity and scope were reconciled above; copy never defines identity.
1612
+ by_id: dict[str, dict[str, Any]] = {}
1613
+ for c in sorted(creatives, key=MODEL.strength_key):
1614
+ by_id.setdefault(c["creative_id"], c)
1615
+ creatives = list(by_id.values())
1616
+ if inputs.runtime is not None and inputs.footprint is None:
1617
+ # Earlier cutoffs only; the corpus of this cutoff is written below.
1618
+ creatives = merge_snapshots(creatives, load_snapshots(inputs.runtime, inputs.as_of), inputs.as_of, inputs.long_running_days,
1619
+ inputs.domains, listings)
1620
+ creatives.sort(key=lambda c: (c["domain"], c["platform"], c["creative_id"]))
1621
+ stored = creatives
1622
+ if inputs.footprint is not None:
1623
+ coverage.extend(footprint_coverage(inputs, stored))
1624
+ else:
1625
+ for domain in inputs.domains:
1626
+ for source in inputs.sources:
1627
+ if not any(c["domain"] == domain and c["source"] in SOURCE_ROW_SOURCES[source] for c in raw):
1628
+ coverage.append({"type": "coverage", "domain": domain, "source": source, "status": "no_data", "detail": "no verified creative", "tier": "C"})
1629
+ # Copy/preview classification establishes exclusions before large downloads.
1630
+ prepare_media(client, stored, inputs, config, coverage, analyze_videos=False)
1631
+ client.semantic_context = frozen_semantic_context(client, inputs)
1632
+ selected = select_for_classification([c for c in stored if not c.get("vanished_after")])
1633
+ stopped_rows = [c for c in stored if c.get("vanished_after")]
1634
+ classifications = classify_corpus(client, selected, taxonomy, taxonomy_sha256, inputs.product_context, coverage,
1635
+ cached_only=stopped_rows) if (selected or stopped_rows) else []
1636
+ SEMANTICS.apply_semantics(stored, classifications, client.semantic_context)
1637
+ before_media = {c["creative_id"]: classification_source_hash(c) for c in stored}
1638
+ prepare_media(client, stored, inputs, config, coverage)
1639
+ enriched = [c for c in stored if not c.get("vanished_after")
1640
+ and classification_source_hash(c) != before_media[c["creative_id"]]]
1641
+ if enriched:
1642
+ changed_ids = {c["creative_id"] for c in enriched}
1643
+ # A preview classification cannot claim it observed subsequently added
1644
+ # speech/frames. Only a successful enriched classification replaces it.
1645
+ classifications = [cl for cl in classifications if cl["creative_id"] not in changed_ids]
1646
+ classifications += classify_corpus(client, select_for_classification(enriched), taxonomy, taxonomy_sha256,
1647
+ inputs.product_context, coverage)
1648
+ dictionary = SEMANTICS.apply_semantics(stored, classifications, client.semantic_context)
1649
+ if inputs.runtime is not None:
1650
+ write_snapshot(inputs.runtime, inputs.as_of, [c for c in stored if not c.get("vanished_after")], listings, inputs.domains, inputs.sources)
1651
+ for c in stored:
1652
+ for member in c.get("source_members") or []:
1653
+ member["language_allowed"] = c.get("language_allowed")
1654
+ member["language_status"] = c.get("language_status")
1655
+ by_creative = {c["creative_id"]: c for c in stored}
1656
+ for cl in classifications:
1657
+ c = by_creative[cl["creative_id"]]
1658
+ labels = cl.get("labels") or {}
1659
+ detail = labels.get("message_shape") if c.get("creative_type") == "text" else "+".join(sorted(labels.get("visual_treatment") or []))
1660
+ c["format_detail"] = detail or "unspecified"
1661
+ for name in ("language_allowed", "language_status", "country", "creative_type", "format_kind", "format_detail", "competitor_tier", "relevance", "duration_scope", "source_members"):
1662
+ cl[name] = by_creative[cl["creative_id"]].get(name)
1663
+ classified_ids = {c["creative_id"] for c in classifications}
1664
+ for c in stored:
1665
+ if c.get("relevance") != "irrelevant" and c.get("language_status") != "excluded" and c["creative_id"] not in classified_ids:
1666
+ coverage.append({"type": "coverage", "domain": c["domain"], "country": c.get("country"), "channel": c["channel"], "source": "classify", "status": "unclassified", "detail": "No usable classification; see media and budget gaps", "tier": "C"})
1667
+ patterns = score_patterns(stored, classifications, taxonomy, inputs.long_running_days, dictionary)
1668
+ anchors = price_anchors(stored, classifications)
1669
+ totals = client.ledger_totals()
1670
+ retrieved_on = totals["first_retrieved_on"] if totals["first_retrieved_on"] == totals["last_retrieved_on"] else f"{totals['first_retrieved_on']}..{totals['last_retrieved_on']}"
1671
+ if inputs.footprint is not None:
1672
+ retrieved_on = inputs.footprint.get("retrieved_on") or inputs.as_of.isoformat()
1673
+ evidence = {
1674
+ "schema_version": EVIDENCE_SCHEMA_VERSION,
1675
+ "section_id": SECTION_ID,
1676
+ "coverage_status": "partial" if any(r.get("status") in ("budget_deferred", "unclassified", "not_extracted", "unavailable", "media_gap", "truncated_input") for r in coverage) else "sample_completed",
1677
+ "angle_dictionary": dictionary,
1678
+ "generated_by": Path(__file__).name,
1679
+ "retrieved_on": retrieved_on,
1680
+ "inputs": {"domains": inputs.domains, "domain_sources": inputs.domain_sources, "product_domain": inputs.product_domain,
1681
+ "sources": inputs.sources, "country": inputs.country, "eu_country": inputs.eu_country, "as_of": inputs.as_of.isoformat(),
1682
+ "effective_policy": inputs.effective_policy,
1683
+ "policy_sha256": inputs.policy_sha256, "target_countries": (inputs.footprint or {}).get("inputs", {}).get("target_countries", sorted({inputs.country, inputs.eu_country})),
1684
+ "collection_mode": "channel_footprint_reuse" if inputs.footprint is not None else "standalone",
1685
+ "accepted_languages": inputs.languages, "market_languages": inputs.market_languages,
1686
+ "top_videos_per_competitor_channel_market": inputs.top_videos,
1687
+ "deepened_cells": None if inputs.deepened_cells is None else sorted([country, channel] for country, channel in inputs.deepened_cells),
1688
+ "semantic_context_sha256": client.semantic_context["context_sha256"],
1689
+ "footprint_document_sha256": POLICY.canonical_hash(inputs.footprint) if inputs.footprint is not None else None,
1690
+ "taxonomy_version": taxonomy["taxonomy_version"],
1691
+ "taxonomy_sha256": taxonomy_sha256},
1692
+ # Patterns first by score, then every other type interleaved by
1693
+ # competitor: any sample of the file (the AI projection keeps the
1694
+ # first rows of each type) covers the competitors, not the first one.
1695
+ "rows": patterns + _interleave_by_domain(stored) + _interleave_by_domain(classifications)
1696
+ + _interleave_by_domain(anchors) + _interleave_by_domain(coverage),
1697
+ "totals": {k: v for k, v in totals.items() if k not in ("first_retrieved_on", "last_retrieved_on")},
1698
+ }
1699
+ section = render_section(domains=inputs.domains, creatives=stored, classifications=classifications, patterns=patterns, anchors=anchors,
1700
+ coverage=coverage, totals=totals, as_of=inputs.as_of.isoformat(), retrieved_on=retrieved_on,
1701
+ long_running_days=inputs.long_running_days, taxonomy_version=taxonomy["taxonomy_version"], selection=MODEL.SELECTION.build_selection(evidence))
1702
+ return {"evidence": evidence, "section_markdown": section, "totals": totals,
1703
+ "attempt": {"actor_runs": client.actor_runs, "meta_searches": client.meta_searches, "spyfu_calls": client.spyfu_calls,
1704
+ "classify_calls": client.classify_calls, "media_calls": client.media_calls, "image_fetches": client.image_fetches, "cache_hits": client.cache_hits}}
1705
+
1706
+
1707
+ def publish_previews(rows: list[dict], workspace: Path, selection: dict | None = None, countries: list[str] | None = None) -> tuple[Path, ...]:
1708
+ """Make previews portable in the research artifact folder, hash-verified: the
1709
+ top selection plus every row a campaign pack may cite as its original."""
1710
+ selection = selection or MODEL.SELECTION.build_selection({"rows": rows})
1711
+ selected = {(ref["country"], ref["channel"], ref["creative_type"], ref["domain"], ref["creative_id"]) for ref in selection["selected_creatives"]}
1712
+ publishable = [row for row in rows if row.get("type") == "creative"
1713
+ and (not countries or row.get("country") in countries) and (
1714
+ (row.get("country"), row.get("channel"), row.get("creative_type", "unknown"), row.get("domain"), row.get("creative_id")) in selected
1715
+ or MODEL.SELECTION.citable(row, countries))]
1716
+ published = set()
1717
+ for row in publishable:
1718
+ assets = row.get("media_assets") or []
1719
+ previews = [a for a in assets if a.get("kind") in ("image", "preview")]
1720
+ if not previews:
1721
+ previews = [a for a in assets if a.get("kind") == "frame"][:1]
1722
+ for asset in previews:
1723
+ try:
1724
+ ref = Path(str(asset["path"]))
1725
+ source = (workspace / ref).resolve()
1726
+ if ref.is_absolute() or ".." in ref.parts or not source.is_relative_to(workspace):
1727
+ raise ValueError("preview path escapes workspace")
1728
+ if source.stat().st_size > VIDEO.MAX_FRAME_BYTES:
1729
+ raise ValueError("preview exceeds the bounded image size")
1730
+ content = source.read_bytes()
1731
+ digest = hashlib.sha256(content).hexdigest()
1732
+ if asset.get("sha256") and asset["sha256"] != digest:
1733
+ raise ValueError("preview hash mismatch")
1734
+ mime = CF.sniff_image(content)
1735
+ if mime is None:
1736
+ raise ValueError("preview is not a supported image")
1737
+ extension = {"image/jpeg": "jpg", "image/png": "png", "image/webp": "webp"}[mime]
1738
+ dest = workspace / _artifact_path("egbe-docs/research/market/assets/creative-patterns") / f"{digest}.{extension}"
1739
+ if not dest.resolve().is_relative_to(workspace):
1740
+ raise ValueError("published preview path escapes workspace")
1741
+ # The cached bytes themselves, hard-linked when on one filesystem.
1742
+ CF.IO.link_or_copy(source, dest)
1743
+ if hashlib.sha256(dest.read_bytes()).hexdigest() != digest:
1744
+ raise ValueError("published preview changed while it was linked")
1745
+ asset.setdefault("source_path", asset["path"])
1746
+ asset.update(path=dest.relative_to(workspace).as_posix(), sha256=digest)
1747
+ published.add(dest.relative_to(workspace))
1748
+ except (OSError, ValueError, KeyError) as exc:
1749
+ asset["publication_status"] = "unavailable"
1750
+ rows.append({"type": "coverage", "source": "preview_publication", "domain": row["domain"], "country": row.get("country"), "channel": row["channel"], "creative_id": row.get("creative_id"), "status": "media_gap", "detail": str(exc), "tier": "C"})
1751
+ return tuple(sorted(published))
1752
+
1753
+
1754
+ def _interleave_by_domain(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
1755
+ groups: dict[str, list[dict[str, Any]]] = {}
1756
+ for row in rows:
1757
+ groups.setdefault(str(row.get("domain") or ""), []).append(row)
1758
+ out: list[dict[str, Any]] = []
1759
+ while groups:
1760
+ for domain in sorted(list(groups)):
1761
+ out.append(groups[domain].pop(0))
1762
+ if not groups[domain]:
1763
+ del groups[domain]
1764
+ return out
1765
+
1766
+
1767
+ def _arg(args: argparse.Namespace, name: str, default: Any) -> Any:
1768
+ value = getattr(args, name, None)
1769
+ return default if value is None else value
1770
+
1771
+
1772
+ def _write_atomic(path: Path, content: str) -> None:
1773
+ path.parent.mkdir(parents=True, exist_ok=True)
1774
+ tmp = path.with_suffix(path.suffix + ".tmp")
1775
+ tmp.write_text(content, encoding="utf-8")
1776
+ tmp.replace(path)
1777
+
1778
+
1779
+ def resolve_as_of(runtime: Path, override: str, refresh: bool) -> date:
1780
+ """The cutoff is fixed at the first attempt so that every rerun asks the
1781
+ providers the same questions and replays from the cache. --as-of picks a
1782
+ cutoff; --refresh starts today's cutoff, but only once the current one has
1783
+ completed: a refresh that stopped on a budget resumes, it never buys the
1784
+ same calls again under a new date."""
1785
+ marker = runtime / "as-of.json"
1786
+ try:
1787
+ state = json.loads(marker.read_text(encoding="utf-8"))
1788
+ current = date.fromisoformat(state["as_of"])
1789
+ completed = bool(state.get("completed"))
1790
+ except (OSError, ValueError, KeyError, json.JSONDecodeError):
1791
+ current, completed = None, False
1792
+ if override:
1793
+ if not _DATE_RE.match(override):
1794
+ raise CreativePatternsError("--as-of must be YYYY-MM-DD")
1795
+ chosen = date.fromisoformat(override)
1796
+ elif current is None:
1797
+ chosen = date.today()
1798
+ elif refresh and completed:
1799
+ chosen = date.today()
1800
+ else:
1801
+ if refresh and current != date.today():
1802
+ print(f"Creative Patterns: cutoff {current.isoformat()} has not completed yet; resuming it instead of starting a new one.", file=sys.stderr)
1803
+ chosen = current
1804
+ if chosen != current:
1805
+ completed = False
1806
+ _write_atomic(marker, json.dumps({"as_of": chosen.isoformat(), "completed": completed}))
1807
+ return chosen
1808
+
1809
+
1810
+ def mark_cutoff_completed(runtime: Path, as_of: date) -> None:
1811
+ _write_atomic(runtime / "as-of.json", json.dumps({"as_of": as_of.isoformat(), "completed": True}))
1812
+
1813
+
1814
+ def linkedin_slugs(workspace: Path) -> dict[str, str]:
1815
+ """Company page slugs that Channel Footprint read from competitor sites."""
1816
+ path = workspace / CHANNEL_FOOTPRINT_EVIDENCE_REL
1817
+ try:
1818
+ rows = json.loads(path.read_text(encoding="utf-8")).get("rows") or []
1819
+ except (OSError, json.JSONDecodeError, AttributeError):
1820
+ return {}
1821
+ return {r["domain"]: r["linkedin_company"] for r in rows if isinstance(r, dict) and r.get("type") == "pixel_scan" and r.get("linkedin_company")}
1822
+
1823
+
1824
+ def run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
1825
+ workspace = workspace.resolve()
1826
+ io = _load_script("research-artifact-io.py", "pmr_artifact_io")
1827
+ with io.section_attempt(workspace, SECTION_ID, context_file=getattr(args, "context_file", None)):
1828
+ return _run(workspace, args)
1829
+
1830
+
1831
+ def _run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
1832
+ runtime = workspace / RUNTIME_REL
1833
+ runtime.mkdir(parents=True, exist_ok=True)
1834
+ api_key = (os.environ.get("OS_API_KEY") or "").strip()
1835
+ if not api_key:
1836
+ raise CreativePatternsError("OS_API_KEY is required")
1837
+ os_url = (os.environ.get("OS_URL") or DEFAULT_OS_URL).strip() or DEFAULT_OS_URL
1838
+ config = CF.load_sources_config(Path(_arg(args, "sources_config", "")) if _arg(args, "sources_config", "") else None)
1839
+ taxonomy, taxonomy_sha256 = load_taxonomy()
1840
+ effective_policy, policy_sha256 = POLICY.policy_from_taxonomy(taxonomy)
1841
+ footprint = footprint_input(workspace, effective_policy, policy_sha256)
1842
+ raw_sources = str(_arg(args, "sources", "") or "")
1843
+ chosen = [s.strip().lower() for s in raw_sources.split(",") if s.strip()] if raw_sources.strip() else list(ALL_SOURCES)
1844
+ unknown = [s for s in chosen if s not in ALL_SOURCES]
1845
+ if unknown:
1846
+ raise CreativePatternsError(f"unknown sources: {', '.join(unknown)} (known: {', '.join(ALL_SOURCES)})")
1847
+ sources = [s for s in ALL_SOURCES if s in chosen]
1848
+
1849
+ max_competitors = max(1, int(_arg(args, "max_competitors", MAX_COMPETITORS)))
1850
+ product_domain = registrable_domain(str(_arg(args, "product_domain", "") or ""))
1851
+ explicit = [d for d in CF.parse_domains(_arg(args, "domains", ""), MAX_EXPLICIT_DOMAINS) if d != product_domain]
1852
+ evidence_mod = _load_script("research-evidence.py", "pmr_evidence_for_creative_patterns")
1853
+ tiered = evidence_mod.competitor_set_inputs(workspace, consumer=SECTION_ID, cap=MAX_COMPETITORS)
1854
+ rejected = evidence_mod.competitor_set_excluded_domains(workspace) - set(explicit)
1855
+ discovered = evidence_mod.keyword_inputs(workspace, consumer=SECTION_ID, row_type="competitor_domain", field="domain", cap=MAX_COMPETITORS, exclude=rejected)
1856
+ domains: list[str] = []
1857
+ domain_sources: dict[str, str] = {}
1858
+ for source, candidates in (("explicit", explicit), ("competitor_set", tiered), ("keyword_economics", discovered)):
1859
+ for domain in candidates:
1860
+ domain = registrable_domain(domain)
1861
+ if len(domains) < max_competitors and domain and domain != product_domain and domain not in domains:
1862
+ domains.append(domain)
1863
+ domain_sources[domain] = source
1864
+ if footprint is not None:
1865
+ footprint_domains = footprint["inputs"]["domains"]
1866
+ if any(d not in footprint_domains for d in explicit):
1867
+ raise CreativePatternsError("explicit competitor is outside the completed Channel Footprint; collect and finalize it there first")
1868
+ domains = list(dict.fromkeys(explicit + footprint_domains))[:max_competitors]
1869
+ domain_sources = {d: "channel_footprint" for d in domains}
1870
+ if not domains:
1871
+ raise CreativePatternsError("no competitor domains available: pass --domains, or complete the Competitor Set (or Keyword Economics) section first")
1872
+ target_markets = (footprint or {}).get("inputs", {}).get("target_countries") or [str(_arg(args, "country", "US") or "US").upper(), str(_arg(args, "eu_country", "DE") or "DE").upper()]
1873
+ languages, market_languages = language_inputs(workspace, args, target_markets)
1874
+ if not languages:
1875
+ raise CreativePatternsError("accepted languages must be explicitly supplied in intake or --languages before paid calls")
1876
+ for name in ("top_videos", "max_channels", "max_media_calls", "max_total_paid_starts", "max_actor_runs", "max_meta_searches", "max_spyfu_calls", "max_classify_calls", "max_image_fetches"):
1877
+ value = _arg(args, name, None)
1878
+ if value is not None and int(value) < 0:
1879
+ raise CreativePatternsError(f"--{name.replace('_', '-')} cannot be negative")
1880
+ inputs = RunInputs(
1881
+ domains=domains, domain_sources=domain_sources, product_domain=product_domain, sources=sources,
1882
+ country=str(_arg(args, "country", "US") or "US").upper(), eu_country=str(_arg(args, "eu_country", "DE") or "DE").upper(),
1883
+ as_of=resolve_as_of(runtime, str(_arg(args, "as_of", "") or ""), bool(_arg(args, "refresh", False))),
1884
+ long_running_days=None,
1885
+ footprint=footprint, effective_policy=effective_policy, policy_sha256=policy_sha256,
1886
+ restrict_sources=bool(str(_arg(args, "sources", "") or "").strip()),
1887
+ product_context=full_product_context(workspace),
1888
+ languages=languages, market_languages=market_languages, top_videos=int(_arg(args, "top_videos", 5)),
1889
+ max_channels=int(_arg(args, "max_channels", 0) or 0),
1890
+ linkedin_slugs=linkedin_slugs(workspace),
1891
+ runtime=runtime,
1892
+ )
1893
+ client = OsClient(
1894
+ # One cache namespace per cutoff: a new cutoff asks every provider again.
1895
+ os_url=os_url, api_key=api_key, cache_dir=runtime / "cache" / inputs.as_of.isoformat(), shared_dir=runtime / "cache" / "shared",
1896
+ cutoff=inputs.as_of.isoformat(), key_scope=f"{SECTION_ID}:{inputs.as_of.isoformat()}:",
1897
+ max_actor_runs=int(_arg(args, "max_actor_runs", MAX_ACTOR_RUNS_PER_ATTEMPT)),
1898
+ max_meta_searches=int(_arg(args, "max_meta_searches", MAX_META_SEARCHES_PER_ATTEMPT)),
1899
+ max_spyfu_calls=int(_arg(args, "max_spyfu_calls", MAX_SPYFU_CALLS_PER_ATTEMPT)),
1900
+ max_total_paid_starts=int(_arg(args, "max_total_paid_starts", 240)),
1901
+ cumulative_budget_path=runtime / "budget.json",
1902
+ max_media_calls=int(_arg(args, "max_media_calls", 100)),
1903
+ max_classify_calls=int(_arg(args, "max_classify_calls", MAX_CLASSIFY_CALLS_PER_ATTEMPT)),
1904
+ max_image_fetches=int(_arg(args, "max_image_fetches", CF.MAX_IMAGE_FETCHES_PER_ATTEMPT)),
1905
+ )
1906
+ existing_budget = client._cache_read(client.cumulative_budget_path)
1907
+ if existing_budget is None and client.cumulative_budget_path.is_file():
1908
+ raise CreativePatternsError("invalid cumulative budget ledger; do not reset its accepted limit")
1909
+ budget = existing_budget or {"schema_version": 1, "limit": client.max_total_paid_starts, "paid_starts": []}
1910
+ budget["limit"] = min(int(budget["limit"]), client.max_total_paid_starts)
1911
+ budget["spent_starts"] = len(budget["paid_starts"])
1912
+ _write_atomic(client.cumulative_budget_path, json.dumps(budget, indent=1))
1913
+ outcome = run_pipeline(client, inputs, config, taxonomy, taxonomy_sha256)
1914
+ selection = MODEL.SELECTION.build_selection(outcome["evidence"], workspace)
1915
+ published_previews = publish_previews(outcome["evidence"]["rows"], workspace, selection, outcome["evidence"]["inputs"]["target_countries"])
1916
+ if any(r.get("status") == "media_gap" for r in outcome["evidence"]["rows"]):
1917
+ outcome["evidence"]["coverage_status"] = "partial"
1918
+ # Regenerate the markdown with portable preview refs after materialization.
1919
+ rows = outcome["evidence"]["rows"]
1920
+ outcome["section_markdown"] = render_section(domains=inputs.domains, creatives=[r for r in rows if r["type"] == "creative"],
1921
+ classifications=[r for r in rows if r["type"] == "creative_classification"], patterns=[r for r in rows if r["type"] == "pattern_score"],
1922
+ anchors=[r for r in rows if r["type"] == "price_anchor"], coverage=[r for r in rows if r["type"] == "coverage"], totals=outcome["totals"],
1923
+ as_of=inputs.as_of.isoformat(), retrieved_on=outcome["evidence"]["retrieved_on"], long_running_days=inputs.long_running_days, taxonomy_version=taxonomy["taxonomy_version"], selection=selection)
1924
+ if outcome["totals"]["provider_calls"] == 0 and inputs.footprint is None and not any(r.get("status") == "budget_deferred" for r in outcome["evidence"]["rows"]):
1925
+ raise CreativePatternsError("no provider answered for any competitor; the section cannot complete without one successful call")
1926
+
1927
+ selection = MODEL.SELECTION.build_selection(outcome["evidence"], workspace)
1928
+ selection_json = json.dumps(selection, indent=1, ensure_ascii=False)
1929
+ selection_sha256 = hashlib.sha256(selection_json.encode("utf-8")).hexdigest()
1930
+ _write_atomic(workspace / MODEL.SELECTION.SELECTION_REL, selection_json)
1931
+ outcome["evidence"]["selection"] = selection
1932
+ outcome["evidence"]["inputs"].update(creative_selection_path=MODEL.SELECTION.SELECTION_REL.as_posix(), creative_selection_sha256=selection_sha256)
1933
+ evidence_json = json.dumps(outcome["evidence"], indent=1, ensure_ascii=False)
1934
+ _write_atomic(workspace / EVIDENCE_REL, evidence_json)
1935
+ _write_atomic(runtime / "result.md", outcome["section_markdown"])
1936
+ upsert = _load_script("upsert-markdown-section.py", "pmr_upsert_markdown_section")
1937
+ report_path = workspace / REPORT_REL
1938
+ report_path.parent.mkdir(parents=True, exist_ok=True)
1939
+ upsert.upsert_markdown_section(report_path, SECTION_HEADING, outcome["section_markdown"])
1940
+ persist = _load_script("persist-research-report.py", "pmr_persist_research_report")
1941
+ persistence = persist.persist_research_report(
1942
+ workspace, commit_message=f"research: update {SECTION_HEADING.lower()} section",
1943
+ extra_paths=(Path(_artifact_path("egbe-docs/research/market/evidence/creative-patterns.json")), Path(_artifact_path("egbe-docs/research/market/evidence/creative-selection.v1.json")), *published_previews),
1944
+ )
1945
+ totals, attempt = outcome["totals"], outcome["attempt"]
1946
+ state = {
1947
+ "schema_version": SCHEMA_VERSION,
1948
+ "section_id": SECTION_ID,
1949
+ "status": "completed",
1950
+ "coverage_status": outcome["evidence"]["coverage_status"],
1951
+ "creative_selection_path": MODEL.SELECTION.SELECTION_REL.as_posix(),
1952
+ "creative_selection_sha256": selection_sha256,
1953
+ "intake_sha256": hashlib.sha256((workspace / _artifact_path("egbe-docs/research/market/intake.json")).read_bytes()).hexdigest() if (workspace / _artifact_path("egbe-docs/research/market/intake.json")).is_file() else None,
1954
+ "media_calls": totals["media_calls"], "reused_media_calls": totals["reused_media_calls"],
1955
+ "media_prompt_tokens": totals["media_prompt_tokens"], "media_completion_tokens": totals["media_completion_tokens"],
1956
+ "cumulative_paid_starts": totals["cumulative_paid_starts"], "cumulative_paid_start_limit": totals["cumulative_paid_start_limit"],
1957
+ "domains": inputs.domains,
1958
+ "domain_sources": inputs.domain_sources,
1959
+ "sources": inputs.sources,
1960
+ "country": inputs.country,
1961
+ "eu_country": inputs.eu_country,
1962
+ "as_of": inputs.as_of.isoformat(),
1963
+ "taxonomy_version": taxonomy["taxonomy_version"],
1964
+ "effective_policy": effective_policy, "policy_sha256": policy_sha256,
1965
+ "channel_footprint_sha256": hashlib.sha256((workspace / CHANNEL_FOOTPRINT_EVIDENCE_REL).read_bytes()).hexdigest() if footprint is not None else None,
1966
+ "taxonomy_sha256": taxonomy_sha256,
1967
+ "evidence_path": EVIDENCE_REL.as_posix(),
1968
+ "evidence_sha256": hashlib.sha256(evidence_json.encode("utf-8")).hexdigest(),
1969
+ "result_path": (RUNTIME_REL / "result.md").as_posix(),
1970
+ "result_sha256": hashlib.sha256(outcome["section_markdown"].encode("utf-8")).hexdigest(),
1971
+ "provider_calls": totals["provider_calls"],
1972
+ "actor_runs": totals["actor_runs"],
1973
+ "meta_searches": totals["meta_searches"],
1974
+ "spyfu_calls": totals["spyfu_calls"],
1975
+ "classify_calls": totals["classify_calls"],
1976
+ "reused_classify_calls": totals["reused_classify_calls"],
1977
+ "image_fetches": totals["image_fetches"],
1978
+ "rejected_calls": totals["rejected_calls"],
1979
+ "observations": totals["observations"],
1980
+ "classify_prompt_tokens": totals["classify_prompt_tokens"],
1981
+ "classify_completion_tokens": totals["classify_completion_tokens"],
1982
+ "creatives": sum(1 for r in outcome["evidence"]["rows"] if r["type"] == "creative"),
1983
+ "classifications": sum(1 for r in outcome["evidence"]["rows"] if r["type"] == "creative_classification"),
1984
+ "patterns": sum(1 for r in outcome["evidence"]["rows"] if r["type"] == "pattern_score"),
1985
+ "attempt_actor_runs": attempt["actor_runs"],
1986
+ "attempt_meta_searches": attempt["meta_searches"],
1987
+ "attempt_spyfu_calls": attempt["spyfu_calls"],
1988
+ "attempt_classify_calls": attempt["classify_calls"],
1989
+ "attempt_image_fetches": attempt["image_fetches"],
1990
+ "attempt_cache_hits": attempt["cache_hits"],
1991
+ "report_commit_sha": persistence.commit_sha,
1992
+ "publication_mode": persistence.publication_mode,
1993
+ "research_context": _load_script("research-artifact-io.py", "pmr_input_io").context_binding(workspace, SECTION_ID),
1994
+ "error": None,
1995
+ }
1996
+ _write_atomic(runtime / "request.json", json.dumps(state, indent=1))
1997
+ _write_atomic(runtime / "semantic-dictionary.json", json.dumps({**outcome["evidence"]["angle_dictionary"],
1998
+ "as_of": inputs.as_of.isoformat(), "product_sha256": hashlib.sha256(inputs.product_context.encode()).hexdigest(),
1999
+ "languages": inputs.languages}, ensure_ascii=False))
2000
+ mark_cutoff_completed(runtime, inputs.as_of)
2001
+ return state
2002
+
2003
+
2004
+ def main() -> int:
2005
+ parser = argparse.ArgumentParser(description=__doc__)
2006
+ parser.add_argument("--context-file", type=Path, help="Explicit context for this study; no project files are discovered implicitly.")
2007
+ parser.add_argument("--workspace", type=Path, required=True)
2008
+ parser.add_argument("--domains", default="", help="Comma-separated competitor domains; merged ahead of the Competitor Set and Keyword Economics domains.")
2009
+ parser.add_argument("--product-domain", default="", help="The product's own domain; never collected as a competitor.")
2010
+ parser.add_argument("--sources", default="", help=f"Comma-separated subset of {', '.join(ALL_SOURCES)} (default: all).")
2011
+ parser.add_argument("--sources-config", default="", help="Path to an alternative actor config (default: the Channel Footprint sources config).")
2012
+ parser.add_argument("--max-competitors", type=int, default=MAX_COMPETITORS)
2013
+ parser.add_argument("--country", default="US", help="Market for SpyFu Google search ads (default US).")
2014
+ parser.add_argument("--eu-country", default="DE", help="EU country for official Meta delivery data (default DE).")
2015
+ parser.add_argument("--as-of", default="", help="Cutoff date YYYY-MM-DD (default: the first attempt's date, kept across reruns).")
2016
+ parser.add_argument("--refresh", action="store_true", help="Start a new cutoff at today's date (new provider calls).")
2017
+ parser.add_argument("--max-actor-runs", type=int, default=MAX_ACTOR_RUNS_PER_ATTEMPT)
2018
+ parser.add_argument("--max-meta-searches", type=int, default=MAX_META_SEARCHES_PER_ATTEMPT)
2019
+ parser.add_argument("--max-spyfu-calls", type=int, default=MAX_SPYFU_CALLS_PER_ATTEMPT, help="SpyFu ad-history lookups per attempt (two per competitor).")
2020
+ parser.add_argument("--max-classify-calls", type=int, default=MAX_CLASSIFY_CALLS_PER_ATTEMPT, help="Classification calls per attempt (up to ten creatives each).")
2021
+ parser.add_argument("--max-image-fetches", type=int, default=CF.MAX_IMAGE_FETCHES_PER_ATTEMPT)
2022
+ parser.add_argument("--languages", default="", help="Explicit accepted language codes, e.g. en,es")
2023
+ parser.add_argument("--market-languages", default="", help="Per-market language codes: US:en,es;GB:en")
2024
+ parser.add_argument("--top-videos", type=int, default=5)
2025
+ parser.add_argument("--max-channels", type=int, default=0, help="Express scope: deepen only this many candidate channels per market (0: every cell).")
2026
+ parser.add_argument("--max-media-calls", type=int, default=100)
2027
+ parser.add_argument("--max-total-paid-starts", type=int, default=240, help="Durable section-wide cap; reruns do not reset it")
2028
+ args = parser.parse_args()
2029
+ try:
2030
+ state = run(args.workspace, args)
2031
+ except CreativePatternsError as exc:
2032
+ print(f"Creative Patterns could not complete: {exc}", file=sys.stderr)
2033
+ return 1
2034
+ except ValueError as exc:
2035
+ print(f"Creative Patterns could not complete: {exc}", file=sys.stderr)
2036
+ return 1
2037
+ print(json.dumps({k: state[k] for k in ("status", "domains", "creatives", "classifications", "patterns", "provider_calls")}, indent=1))
2038
+ return 0
2039
+
2040
+
2041
+ if __name__ == "__main__":
2042
+ raise SystemExit(main())