@egbe-ai/opencode 0.0.0-stage → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +124 -2
- package/index.js +5 -0
- package/package.json +55 -4
- package/plugin.js +230 -0
- package/session-log.js +315 -0
- package/skills/.egbe-catalog/manifest.json +1 -0
- package/skills/egbe-os-analytics/SKILL.md +499 -0
- package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
- package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
- package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
- package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
- package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
- package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
- package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
- package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
- package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
- package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
- package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
- package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
- package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
- package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
- package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
- package/skills/egbe-os-db/SKILL.md +174 -0
- package/skills/egbe-os-deploy/SKILL.md +412 -0
- package/skills/egbe-os-design/SKILL.md +182 -0
- package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
- package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
- package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
- package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
- package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
- package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
- package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
- package/skills/egbe-os-destination-readiness/references/report.md +116 -0
- package/skills/egbe-os-domains/SKILL.md +189 -0
- package/skills/egbe-os-errors/SKILL.md +259 -0
- package/skills/egbe-os-git/SKILL.md +275 -0
- package/skills/egbe-os-git/references/issues.md +33 -0
- package/skills/egbe-os-google-ads/SKILL.md +163 -0
- package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
- package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
- package/skills/egbe-os-google-ads/references/conversions.md +153 -0
- package/skills/egbe-os-google-ads/references/delivery.md +128 -0
- package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
- package/skills/egbe-os-google-ads/references/reporting.md +105 -0
- package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
- package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
- package/skills/egbe-os-google-auth/SKILL.md +285 -0
- package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
- package/skills/egbe-os-images/SKILL.md +163 -0
- package/skills/egbe-os-images/references/stock-images.md +21 -0
- package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
- package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
- package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
- package/skills/egbe-os-llm/SKILL.md +160 -0
- package/skills/egbe-os-mail/SKILL.md +180 -0
- package/skills/egbe-os-me/SKILL.md +119 -0
- package/skills/egbe-os-meta-ads/SKILL.md +114 -0
- package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
- package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
- package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
- package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
- package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
- package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
- package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
- package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
- package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
- package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
- package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
- package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
- package/skills/egbe-os-notifications/SKILL.md +78 -0
- package/skills/egbe-os-openai-ads/SKILL.md +149 -0
- package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
- package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
- package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
- package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
- package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
- package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
- package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
- package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
- package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
- package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
- package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
- package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
- package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
- package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
- package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
- package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
- package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
- package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
- package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
- package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
- package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
- package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
- package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
- package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
- package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
- package/skills/egbe-os-payments/SKILL.md +215 -0
- package/skills/egbe-os-payments/agents/openai.yaml +4 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
- package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
- package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
- package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
- package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
- package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
- package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
- package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
- package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
- package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
- package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
- package/skills/egbe-os-playbook/SKILL.md +79 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
- package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
- package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
- package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
- package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
- package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
- package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
- package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
- package/skills/egbe-os-playbook/playbooks/product.md +67 -0
- package/skills/egbe-os-playbook/references/catalog.md +127 -0
- package/skills/egbe-os-playbook/references/changes.md +167 -0
- package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
- package/skills/egbe-os-product-auth/SKILL.md +151 -0
- package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
- package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
- package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
- package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
- package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
- package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
- package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
- package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
- package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
- package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
- package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
- package/skills/egbe-os-product-build/SKILL.md +111 -0
- package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-build/references/capabilities.md +25 -0
- package/skills/egbe-os-product-build/references/engineering.md +72 -0
- package/skills/egbe-os-product-build/references/execution.md +55 -0
- package/skills/egbe-os-product-build/references/foundations.md +93 -0
- package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
- package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
- package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
- package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
- package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
- package/skills/egbe-os-product-market-research/SKILL.md +159 -0
- package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
- package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
- package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
- package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
- package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
- package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
- package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
- package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
- package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
- package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
- package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
- package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
- package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
- package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
- package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
- package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
- package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
- package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
- package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
- package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
- package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
- package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
- package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
- package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
- package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
- package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
- package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
- package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
- package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
- package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
- package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
- package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
- package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
- package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
- package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
- package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
- package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
- package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
- package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
- package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
- package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
- package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
- package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
- package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
- package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
- package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
- package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
- package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
- package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
- package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
- package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
- package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
- package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
- package/skills/egbe-os-product-report/SKILL.md +150 -0
- package/skills/egbe-os-product-report/references/artifacts.md +84 -0
- package/skills/egbe-os-product-report/references/brief.md +34 -0
- package/skills/egbe-os-product-report/references/daily.md +76 -0
- package/skills/egbe-os-product-report/references/sources.md +68 -0
- package/skills/egbe-os-product-routines/SKILL.md +166 -0
- package/skills/egbe-os-product-routines/references/catalog.md +22 -0
- package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
- package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
- package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
- package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
- package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
- package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
- package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
- package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
- package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
- package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
- package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
- package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
- package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
- package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
- package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
- package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
- package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
- package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
- package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
- package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
- package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
- package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
- package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
- package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
- package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
- package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
- package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
- package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
- package/skills/egbe-os-report-ui/SKILL.md +133 -0
- package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
- package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
- package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
- package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
- package/skills/egbe-os-report-ui/references/components.md +209 -0
- package/skills/egbe-os-research/SKILL.md +125 -0
- package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
- package/skills/egbe-os-scrape/SKILL.md +160 -0
- package/skills/egbe-os-search/SKILL.md +182 -0
- package/skills/egbe-os-search/references/google-places.md +81 -0
- package/skills/egbe-os-support/SKILL.md +149 -0
- package/skills/egbe-os-telemetry/SKILL.md +344 -0
- package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
- package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
- package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
- package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
- package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
- package/skills/egbe-os-tools/SKILL.md +180 -0
- package/skills/egbe-os-tools/references/theona.md +26 -0
- package/skills/egbe-os-wallet/SKILL.md +181 -0
- package/skills/egbe-setup/EGBE.md +484 -0
- package/skills/egbe-setup/SKILL.md +116 -0
- package/skills/egbe-setup/references/project.md +80 -0
- package/state.js +37 -0
- package/update.js +361 -0
|
@@ -0,0 +1,2042 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Collect individual competitor ads and derive scoped creative patterns.
|
|
3
|
+
|
|
4
|
+
Reuse completed Channel Footprint inputs, restore native text and available
|
|
5
|
+
media, and classify comparable offers through OS. Generic taxonomy dimensions
|
|
6
|
+
and category-specific message meanings remain distinct. Confirmed serving
|
|
7
|
+
history and current examples support patterns within market/channel/type/format;
|
|
8
|
+
listing spans never establish delivery duration or conversion performance.
|
|
9
|
+
Cached provider responses, labels and video analysis survive compatible retries,
|
|
10
|
+
with a durable cumulative resource ceiling and explicit partial coverage.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import sys as _artifact_sys
|
|
15
|
+
from pathlib import Path as _ArtifactPath
|
|
16
|
+
_artifact_sys.path.insert(0, str(_ArtifactPath(__file__).resolve().parent))
|
|
17
|
+
from egbe_artifacts import artifact_path as _artifact_path
|
|
18
|
+
|
|
19
|
+
import argparse
|
|
20
|
+
import base64
|
|
21
|
+
import copy
|
|
22
|
+
import hashlib
|
|
23
|
+
import importlib.util
|
|
24
|
+
import json
|
|
25
|
+
import os
|
|
26
|
+
import re
|
|
27
|
+
import sys
|
|
28
|
+
import time
|
|
29
|
+
import urllib.error
|
|
30
|
+
import urllib.parse
|
|
31
|
+
import urllib.request
|
|
32
|
+
from dataclasses import dataclass, field
|
|
33
|
+
from datetime import date, timedelta
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
from types import ModuleType
|
|
36
|
+
from typing import Any, Callable
|
|
37
|
+
|
|
38
|
+
SCHEMA_VERSION = 1
|
|
39
|
+
EVIDENCE_SCHEMA_VERSION = 2
|
|
40
|
+
SECTION_ID = "creative_patterns"
|
|
41
|
+
SECTION_HEADING = "Creative patterns"
|
|
42
|
+
DEFAULT_OS_URL = "https://os.hz.egbe.dev"
|
|
43
|
+
RUNTIME_REL = Path(_artifact_path("egbe-docs/research/market/work")) / SECTION_ID
|
|
44
|
+
EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/creative-patterns.json"))
|
|
45
|
+
REPORT_REL = Path(_artifact_path("egbe-docs/research/market/full-report.md"))
|
|
46
|
+
TAXONOMY_REL = Path("references/config/creative-taxonomy.v1.json")
|
|
47
|
+
CHANNEL_FOOTPRINT_EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/channel-footprint.json"))
|
|
48
|
+
|
|
49
|
+
MAX_COMPETITORS = 10
|
|
50
|
+
MAX_EXPLICIT_DOMAINS = 10
|
|
51
|
+
# Ten competitors with every source need 32 runs (two Google batches, then
|
|
52
|
+
# Meta, LinkedIn and Reddit per competitor): one attempt covers a default run.
|
|
53
|
+
MAX_ACTOR_RUNS_PER_ATTEMPT = 36
|
|
54
|
+
MAX_META_SEARCHES_PER_ATTEMPT = 12
|
|
55
|
+
MAX_SPYFU_CALLS_PER_ATTEMPT = 24
|
|
56
|
+
MAX_CLASSIFY_CALLS_PER_ATTEMPT = 40
|
|
57
|
+
GOOGLE_ADS_PER_DOMAIN = 40
|
|
58
|
+
# A non-brand advertiser is the competitor's legal entity only when it clearly
|
|
59
|
+
# dominates the domain's rows. Live 2026-10-02: freecodecamp.org runs no
|
|
60
|
+
# Google ads; four unrelated advertisers pointed at it and the largest, with
|
|
61
|
+
# 2 of the 4 rows, was taken for the competitor.
|
|
62
|
+
GOOGLE_DOMINANT_MIN_ROWS = 3
|
|
63
|
+
GOOGLE_DOMINANT_MIN_SHARE = 0.75
|
|
64
|
+
META_ADS_PER_BRAND = 30
|
|
65
|
+
META_EU_ADS_PER_BRAND = 30
|
|
66
|
+
# The LinkedIn library actor works through only the first company page of a
|
|
67
|
+
# multi-page input (observed live), so each competitor gets its own run.
|
|
68
|
+
LINKEDIN_ADS_PER_COMPANY = 30
|
|
69
|
+
REDDIT_ADS_PER_BRAND = 30
|
|
70
|
+
SPYFU_ROWS_PER_WINDOW = 100
|
|
71
|
+
SPYFU_EARLIER_WINDOW = (105, 365)
|
|
72
|
+
TEXT_BATCH = 10
|
|
73
|
+
IMAGE_BATCH = 8
|
|
74
|
+
CLASSIFY_TIMEOUT_SECONDS = 330.0
|
|
75
|
+
# Classification errors that are about the caller, not the batch.
|
|
76
|
+
STOP_CLASSIFYING_RE = re.compile(r"HTTP (?:401|402|403|429)\b|budget_exceeded|Budget has been exceeded|ExceededBudget")
|
|
77
|
+
PRODUCT_CONTEXT_CHARS = 24000
|
|
78
|
+
ALL_SOURCES = ("google_transparency", "meta", "google_search", "linkedin", "reddit")
|
|
79
|
+
# Libraries that keep stopped ads listed (LinkedIn keeps them a year after the
|
|
80
|
+
# last impression): being listed at a cutoff proves neither age nor delivery.
|
|
81
|
+
LISTS_STOPPED_ADS = {"linkedin_library_actor", "reddit_library_actor"}
|
|
82
|
+
# Rows kept per competitor and platform for ads that stopped since an earlier
|
|
83
|
+
# cutoff; they never take the places of ads seen at this cutoff.
|
|
84
|
+
# Which creative `source` values each selectable source produces.
|
|
85
|
+
SOURCE_ROW_SOURCES = {
|
|
86
|
+
"google_transparency": {"google_transparency"},
|
|
87
|
+
"meta": {"meta_library_actor", "meta_library_official_eu"},
|
|
88
|
+
"google_search": {"spyfu_ad_history"},
|
|
89
|
+
"linkedin": {"linkedin_library_actor"},
|
|
90
|
+
"reddit": {"reddit_library_actor"},
|
|
91
|
+
}
|
|
92
|
+
PLATFORM_CHANNELS = {"google": "google_ads", "youtube": "youtube_ads", "meta": "meta_ads", "linkedin": "linkedin_ads", "reddit": "reddit_ads"}
|
|
93
|
+
# A price needs an amount: a currency sign or code next to a number, or a
|
|
94
|
+
# percent discount. Period words alone ("in a month") are not prices.
|
|
95
|
+
_WS_RE = re.compile(r"\s+")
|
|
96
|
+
_TEMPLATE_RE = re.compile(r"\{\{[^}]+\}\}")
|
|
97
|
+
# SpyFu renders sitelinks and callouts as Markdown links to "#".
|
|
98
|
+
_MD_LINK_RE = re.compile(r"\[([^\]]*)\]\(#?\)")
|
|
99
|
+
_SITELINK_SEP_RE = re.compile(r"\s*(?:·|\\n|\n)\s*")
|
|
100
|
+
_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _load_script(filename: str, module_name: str) -> ModuleType:
|
|
104
|
+
path = Path(__file__).resolve().parent / filename
|
|
105
|
+
spec = importlib.util.spec_from_file_location(module_name, path)
|
|
106
|
+
if spec is None or spec.loader is None:
|
|
107
|
+
raise RuntimeError(f"cannot load bundled script {filename}")
|
|
108
|
+
module = importlib.util.module_from_spec(spec)
|
|
109
|
+
sys.modules[spec.name] = module
|
|
110
|
+
previous = sys.dont_write_bytecode
|
|
111
|
+
sys.dont_write_bytecode = True
|
|
112
|
+
try:
|
|
113
|
+
spec.loader.exec_module(module)
|
|
114
|
+
finally:
|
|
115
|
+
sys.dont_write_bytecode = previous
|
|
116
|
+
return module
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
# The Channel Footprint runner owns the OS client (cache, journaled starts,
|
|
120
|
+
# per-attempt budgets, the per-source breaker) and the identity rules
|
|
121
|
+
# (destination first, strict advertiser names); this runner reuses them.
|
|
122
|
+
CF = _load_script("run-channel-footprint.py", "pmr_channel_footprint_for_creative_patterns")
|
|
123
|
+
POLICY = _load_script("research-policy.py", "pmr_policy_for_creative_patterns")
|
|
124
|
+
MODEL = _load_script("creative-model.py", "pmr_creative_model")
|
|
125
|
+
SEMANTICS = _load_script("creative-semantics.py", "pmr_creative_semantics")
|
|
126
|
+
VIDEO = _load_script("creative-video.py", "pmr_creative_video")
|
|
127
|
+
SPYFU_RECENT_WINDOW_DAYS = 30 # Bounded provider query window; never an activity or success threshold.
|
|
128
|
+
CreativePatternsError = CF.ChannelFootprintError
|
|
129
|
+
BudgetExhausted = CF.BudgetExhausted
|
|
130
|
+
SourceUnavailable = CF.SourceUnavailable
|
|
131
|
+
registrable_domain = CF.registrable_domain
|
|
132
|
+
brand_token = CF.brand_token
|
|
133
|
+
usable_destination = CF.usable_destination
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def load_taxonomy(path: Path | None = None) -> tuple[dict[str, Any], str]:
|
|
137
|
+
taxonomy_path = path or (Path(__file__).resolve().parent.parent / TAXONOMY_REL)
|
|
138
|
+
raw = taxonomy_path.read_bytes()
|
|
139
|
+
taxonomy = json.loads(raw)
|
|
140
|
+
if taxonomy.get("schema_version") != 1 or not isinstance(taxonomy.get("dimensions"), dict):
|
|
141
|
+
raise CreativePatternsError("the creative taxonomy file has an unexpected shape")
|
|
142
|
+
return taxonomy, hashlib.sha256(raw).hexdigest()
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# ── OS client: the Channel Footprint client plus SpyFu ad history and
|
|
146
|
+
# creative classification
|
|
147
|
+
|
|
148
|
+
@dataclass
|
|
149
|
+
class OsClient(CF.OsClient):
|
|
150
|
+
max_classify_calls: int = MAX_CLASSIFY_CALLS_PER_ATTEMPT
|
|
151
|
+
classify_calls: int = 0
|
|
152
|
+
# Classifications and images depend only on the creative, so they live in
|
|
153
|
+
# one cache shared by every cutoff (a refresh never pays for them twice).
|
|
154
|
+
shared_dir: Path | None = None
|
|
155
|
+
semantic_context: dict[str, Any] | None = None
|
|
156
|
+
media_calls: int = 0
|
|
157
|
+
max_media_calls: int = 100
|
|
158
|
+
budget_gaps: list[dict] = field(default_factory=list)
|
|
159
|
+
# Deployment and shared AI-account failures are scoped to this attempt.
|
|
160
|
+
# Successful content caches stay available; a later client can recover.
|
|
161
|
+
media_endpoint_blocked: dict[str, str] = field(default_factory=dict)
|
|
162
|
+
paid_ai_blocked_reason: str = ""
|
|
163
|
+
# GET /v1/media capability flags, read once per attempt on the first uncached transcribe.
|
|
164
|
+
get: Callable[[str, float], dict[str, Any]] | None = None
|
|
165
|
+
media_capabilities: dict[str, Any] | None = None
|
|
166
|
+
|
|
167
|
+
def _transcription_configured(self) -> bool:
|
|
168
|
+
"""False only when GET /v1/media says so; an unreadable answer keeps transcribe."""
|
|
169
|
+
if self.media_capabilities is None:
|
|
170
|
+
try:
|
|
171
|
+
self.media_capabilities = (self.get or self._http_get)("/v1/media", 30.0)
|
|
172
|
+
except Exception:
|
|
173
|
+
self.media_capabilities = {}
|
|
174
|
+
flags = self.media_capabilities.get("transcription")
|
|
175
|
+
return not (isinstance(flags, dict) and flags.get("configured") is False)
|
|
176
|
+
|
|
177
|
+
def _http_get(self, path: str, timeout: float) -> dict[str, Any]:
|
|
178
|
+
req = urllib.request.Request(f"{self.os_url.rstrip('/')}{path}", method="GET",
|
|
179
|
+
headers={"Authorization": f"Bearer {self.api_key}", "Accept": "application/json"})
|
|
180
|
+
for name, value in CF._connection_headers(req.full_url).items():
|
|
181
|
+
req.add_unredirected_header(name, value)
|
|
182
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
183
|
+
envelope = json.loads(resp.read().decode("utf-8"))
|
|
184
|
+
data = envelope.get("data") if isinstance(envelope, dict) else None
|
|
185
|
+
return data if isinstance(data, dict) else {}
|
|
186
|
+
|
|
187
|
+
def media_call(self, path: str, payload: dict[str, Any], *, kind: str, timeout: float = 330.0) -> dict[str, Any]:
|
|
188
|
+
key_payload = {"kind": kind, "path": path, "payload": payload}
|
|
189
|
+
digest = hashlib.sha256(json.dumps(key_payload, sort_keys=True).encode()).hexdigest()
|
|
190
|
+
cache = self._shared() / f"{digest}.json"
|
|
191
|
+
entry = self._cache_read(cache)
|
|
192
|
+
self.touched.add(digest)
|
|
193
|
+
if entry is not None:
|
|
194
|
+
self.cache_hits += 1
|
|
195
|
+
return entry.get("response") or {}
|
|
196
|
+
if path.startswith("/v1/media/") and self.paid_ai_blocked_reason:
|
|
197
|
+
raise CreativePatternsError(self.paid_ai_blocked_reason)
|
|
198
|
+
if path in self.media_endpoint_blocked:
|
|
199
|
+
raise SourceUnavailable(self.media_endpoint_blocked[path])
|
|
200
|
+
if path == "/v1/media/transcribe" and not self._transcription_configured():
|
|
201
|
+
self.media_endpoint_blocked[path] = "GET /v1/media reports transcription.configured is false; no audio was sent"
|
|
202
|
+
raise SourceUnavailable(self.media_endpoint_blocked[path])
|
|
203
|
+
if self.media_calls >= self.max_media_calls:
|
|
204
|
+
raise BudgetExhausted("media call budget exhausted for this attempt; remaining videos are budget_deferred")
|
|
205
|
+
self._reserve_paid_start(kind, digest)
|
|
206
|
+
self.media_calls += 1
|
|
207
|
+
try:
|
|
208
|
+
response = (self.request or self._http)(path, payload, timeout, idempotency_key=None)
|
|
209
|
+
except Exception as exc:
|
|
210
|
+
reason = str(exc)[:500]
|
|
211
|
+
if path.startswith("/v1/media/") and STOP_CLASSIFYING_RE.search(reason):
|
|
212
|
+
self.paid_ai_blocked_reason = reason
|
|
213
|
+
raise CreativePatternsError(reason) from exc
|
|
214
|
+
if re.search(r"HTTP (?:404|405|503)\b", reason):
|
|
215
|
+
self.media_endpoint_blocked[path] = reason
|
|
216
|
+
raise SourceUnavailable(reason) from exc
|
|
217
|
+
raise
|
|
218
|
+
self._cache_write(cache, {"kind": kind, "response": response, "cutoff": self.cutoff, "retrieved_on": date.today().isoformat()})
|
|
219
|
+
return response
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _shared(self) -> Path:
|
|
223
|
+
base = self.shared_dir or self.cache_dir
|
|
224
|
+
base.mkdir(parents=True, exist_ok=True)
|
|
225
|
+
return base
|
|
226
|
+
|
|
227
|
+
def spyfu_ad_history(self, domain: str, country: str, since: str, until: str, max_rows: int) -> tuple[dict[str, Any], str]:
|
|
228
|
+
payload = {"domain": domain, "country_code": country, "max_rows": max_rows, "since": since, "until": until}
|
|
229
|
+
response, retrieved_on = self._metered(
|
|
230
|
+
"spyfu", "/tools/spyfu/ad-history", payload, 90.0,
|
|
231
|
+
budget_attr="spyfu_calls", budget_max=self.max_spyfu_calls, label="SpyFu lookup", paid=False,
|
|
232
|
+
retry_delay=CF.FREE_RETRY_DELAY_SECONDS,
|
|
233
|
+
)
|
|
234
|
+
if response.get("unavailable"):
|
|
235
|
+
raise SourceUnavailable(f"SpyFu rejected the request: {response['unavailable']}")
|
|
236
|
+
if response.get("limited_by_quota"):
|
|
237
|
+
# The gateway cut the page to the claw's remaining daily rows: not
|
|
238
|
+
# the window's answer. Drop it from the cache and retry later.
|
|
239
|
+
digest = hashlib.sha256(json.dumps({"kind": "spyfu", "path": "/tools/spyfu/ad-history", "payload": payload}, sort_keys=True).encode("utf-8")).hexdigest()
|
|
240
|
+
CF.IO.drop_cache_entry(self.cache_dir / f"{digest}.json")
|
|
241
|
+
raise CreativePatternsError("SpyFu answer was cut by the daily spyfu:rows limit; rerun after the limit resets")
|
|
242
|
+
return response, retrieved_on
|
|
243
|
+
|
|
244
|
+
def classify(self, request: dict[str, Any], images: dict[str, dict[str, Any]], taxonomy_sha256: str) -> tuple[dict[str, Any], str, str]:
|
|
245
|
+
"""One POST /v1/media/classify-creative call. The cache key covers the
|
|
246
|
+
request with frame hashes (never the image bytes) and the local
|
|
247
|
+
taxonomy file's hash, so an edited taxonomy is never served old answers."""
|
|
248
|
+
key_payload = {**request, "taxonomy_sha256": taxonomy_sha256, "classification_contract": 4,
|
|
249
|
+
"frames": [{"ref": ref, "sha256": images[ref]["sha256"], "media_type": images[ref]["media_type"]} for ref in sorted(images)]}
|
|
250
|
+
digest = hashlib.sha256(json.dumps({"kind": "classify", "path": "/v1/media/classify-creative", "payload": key_payload}, sort_keys=True).encode("utf-8")).hexdigest()
|
|
251
|
+
cache_file = self._shared() / f"{digest}.json"
|
|
252
|
+
self.touched.add(digest)
|
|
253
|
+
entry = self._cache_read(cache_file)
|
|
254
|
+
if entry is not None:
|
|
255
|
+
self.cache_hits += 1
|
|
256
|
+
response = entry.get("response")
|
|
257
|
+
if isinstance(response, dict) and response.get("unavailable"):
|
|
258
|
+
error = SourceUnavailable(f"classification batch was rejected: {response['unavailable']}")
|
|
259
|
+
error.batch_digest = digest
|
|
260
|
+
raise error
|
|
261
|
+
return (response if isinstance(response, dict) else {}), entry.get("retrieved_on") or date.today().isoformat(), digest
|
|
262
|
+
wire = {**{k: v for k, v in request.items()}, "frames": [
|
|
263
|
+
{"ref": ref, "timestamp_seconds": 0, "media_type": images[ref]["media_type"], "sha256": images[ref]["sha256"],
|
|
264
|
+
"content_base64": base64.b64encode(self.image_bytes(images[ref])).decode("ascii")}
|
|
265
|
+
for ref in sorted(images)
|
|
266
|
+
]}
|
|
267
|
+
if self.paid_ai_blocked_reason:
|
|
268
|
+
raise CreativePatternsError(self.paid_ai_blocked_reason)
|
|
269
|
+
transport = self.request or self._http
|
|
270
|
+
attempts = 0
|
|
271
|
+
while True:
|
|
272
|
+
if self.classify_calls >= self.max_classify_calls:
|
|
273
|
+
raise BudgetExhausted(f"classification budget exhausted ({self.max_classify_calls} calls per attempt); {self.cache_writes} new answer(s) cached in this attempt; rerun to continue from the cache")
|
|
274
|
+
self._reserve_paid_start("classify", digest)
|
|
275
|
+
self.classify_calls += 1
|
|
276
|
+
try:
|
|
277
|
+
response = transport("/v1/media/classify-creative", wire, CLASSIFY_TIMEOUT_SECONDS, idempotency_key=None)
|
|
278
|
+
break
|
|
279
|
+
except CreativePatternsError as exc:
|
|
280
|
+
if re.search(r"HTTP 422\b", str(exc)):
|
|
281
|
+
# A rejected request is a fact about this batch: cache it
|
|
282
|
+
# so reruns never send (or pay for) it again.
|
|
283
|
+
self._cache_write(cache_file, {"kind": "classify", "path": "/v1/media/classify-creative", "payload": key_payload,
|
|
284
|
+
"response": {"unavailable": str(exc)[:300]}, "cutoff": self.cutoff,
|
|
285
|
+
"retrieved_on": date.today().isoformat()})
|
|
286
|
+
error = SourceUnavailable(f"classification batch was rejected: {str(exc)[:300]}")
|
|
287
|
+
error.batch_digest = digest
|
|
288
|
+
raise error from exc
|
|
289
|
+
# A server error (502 included: the model's answer may fail
|
|
290
|
+
# validation once) or an unreachable OS gets one retry.
|
|
291
|
+
unreachable = isinstance(exc, CF.OsUnreachable)
|
|
292
|
+
if attempts >= 1 or not (unreachable or re.search(r"HTTP 5(?:00|02|03|04)\b", str(exc))):
|
|
293
|
+
raise
|
|
294
|
+
attempts += 1
|
|
295
|
+
(self.sleep or time.sleep)(CF.OS_UNREACHABLE_RETRY_DELAY_SECONDS if unreachable else CF.FREE_RETRY_DELAY_SECONDS)
|
|
296
|
+
retrieved_on = date.today().isoformat()
|
|
297
|
+
self._cache_write(cache_file, {"kind": "classify", "path": "/v1/media/classify-creative", "payload": key_payload, "response": response,
|
|
298
|
+
"cutoff": self.cutoff, "retrieved_on": retrieved_on})
|
|
299
|
+
return response, retrieved_on, digest
|
|
300
|
+
|
|
301
|
+
def ledger_totals(self) -> dict[str, Any]:
|
|
302
|
+
kinds = ("actor", "meta-ad-library", "spyfu", "classify", "image", "video_metadata", "video_transcription", "video_analysis")
|
|
303
|
+
counts = dict.fromkeys(kinds, 0)
|
|
304
|
+
reused = dict.fromkeys(kinds, 0)
|
|
305
|
+
rejected = 0
|
|
306
|
+
usage = {"classify_prompt_tokens": 0, "classify_completion_tokens": 0,
|
|
307
|
+
"media_prompt_tokens": 0, "media_completion_tokens": 0}
|
|
308
|
+
dates = []
|
|
309
|
+
for digest in sorted(self.touched):
|
|
310
|
+
path = self.cache_dir / f"{digest}.json"
|
|
311
|
+
if CF.IO.cache_entry_path(path) is None and self.shared_dir is not None:
|
|
312
|
+
path = self.shared_dir / f"{digest}.json"
|
|
313
|
+
entry = self._cache_read(path)
|
|
314
|
+
if not entry:
|
|
315
|
+
continue
|
|
316
|
+
kind, response = entry.get("kind"), entry.get("response") or {}
|
|
317
|
+
if response.get("unavailable"):
|
|
318
|
+
rejected += 1
|
|
319
|
+
elif kind in kinds:
|
|
320
|
+
if kind in ("classify", "image", "video_metadata", "video_transcription", "video_analysis") and self.cutoff and entry.get("cutoff") not in (None, self.cutoff):
|
|
321
|
+
reused[kind] += 1
|
|
322
|
+
else:
|
|
323
|
+
counts[kind] += 1
|
|
324
|
+
token_usage = response.get("usage") or {}
|
|
325
|
+
prefix = "classify" if kind == "classify" else "media"
|
|
326
|
+
if kind != "image":
|
|
327
|
+
for field in ("prompt_tokens", "completion_tokens"):
|
|
328
|
+
key = prefix + "_" + field
|
|
329
|
+
value = token_usage.get(field)
|
|
330
|
+
if key in usage and type(value) is int:
|
|
331
|
+
usage[key] += value
|
|
332
|
+
if isinstance(entry.get("retrieved_on"), str):
|
|
333
|
+
dates.append(entry["retrieved_on"])
|
|
334
|
+
media_calls = sum(counts[k] for k in ("video_metadata", "video_transcription", "video_analysis"))
|
|
335
|
+
provider = sum(counts[k] for k in kinds if k != "image")
|
|
336
|
+
budget = self._cache_read(self.cumulative_budget_path) if self.cumulative_budget_path else None
|
|
337
|
+
return {"actor_runs": counts["actor"], "meta_searches": counts["meta-ad-library"], "spyfu_calls": counts["spyfu"],
|
|
338
|
+
"classify_calls": counts["classify"], "reused_classify_calls": reused["classify"], "image_fetches": counts["image"],
|
|
339
|
+
"media_calls": media_calls, "video_metadata_calls": counts["video_metadata"],
|
|
340
|
+
"video_transcription_calls": counts["video_transcription"], "video_analysis_calls": counts["video_analysis"],
|
|
341
|
+
"reused_media_calls": sum(reused[k] for k in ("video_metadata", "video_transcription", "video_analysis")),
|
|
342
|
+
"provider_calls": provider, "rejected_calls": rejected, "observations": provider + counts["image"], **usage,
|
|
343
|
+
"cumulative_paid_starts": len((budget or {}).get("paid_starts", [])),
|
|
344
|
+
"cumulative_paid_start_limit": (budget or {}).get("limit", self.max_total_paid_starts),
|
|
345
|
+
"first_retrieved_on": min(dates) if dates else date.today().isoformat(),
|
|
346
|
+
"last_retrieved_on": max(dates) if dates else date.today().isoformat()}
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
# ── normalization helpers ───────────────────────────────────────────────────
|
|
350
|
+
|
|
351
|
+
def _clean_text(value: Any, cap: int | None = 600) -> str | None:
|
|
352
|
+
if not isinstance(value, str):
|
|
353
|
+
return None
|
|
354
|
+
if "](#" in value or "]()" in value:
|
|
355
|
+
value = _MD_LINK_RE.sub(lambda m: "; " + m.group(1).strip() if m.group(1).strip() else "", value)
|
|
356
|
+
value = _SITELINK_SEP_RE.sub(" ", value).replace(" ;", ";").strip(" ;")
|
|
357
|
+
text = _WS_RE.sub(" ", value).strip()
|
|
358
|
+
if not text or _TEMPLATE_RE.fullmatch(text) or text.startswith("{{"):
|
|
359
|
+
return None
|
|
360
|
+
return text[:cap]
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _norm(value: str | None) -> str:
|
|
364
|
+
return _WS_RE.sub(" ", (value or "").casefold()).strip()
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def fingerprint(headline: str | None, body: str | None, media: str | None = None, ad_key: str | None = None) -> str:
|
|
368
|
+
"""Similarity key for copy/visual variants; never an ad identity or age proof."""
|
|
369
|
+
if _norm(headline) or _norm(body):
|
|
370
|
+
text = "copy:" + _norm(headline) + " | " + _norm(body)
|
|
371
|
+
elif media:
|
|
372
|
+
text = "media:" + media
|
|
373
|
+
else:
|
|
374
|
+
text = "ad:" + (ad_key or "")
|
|
375
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()[:20]
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def destination_url(values: list[Any]) -> str | None:
|
|
379
|
+
"""The first full landing URL whose host is usable (redirect wrappers
|
|
380
|
+
unwrapped, platform and link-hub hosts skipped); the host alone is what
|
|
381
|
+
identity checks use, the URL is what the classifier and funnels need."""
|
|
382
|
+
for value in values:
|
|
383
|
+
if not isinstance(value, str) or not value.strip():
|
|
384
|
+
continue
|
|
385
|
+
url = value.strip()
|
|
386
|
+
parsed = urllib.parse.urlsplit(url if "//" in url else "https://" + url)
|
|
387
|
+
host = (parsed.hostname or "").lower()
|
|
388
|
+
if host in CF.REDIRECT_WRAPPER_HOSTS:
|
|
389
|
+
wrapped = urllib.parse.parse_qs(parsed.query).get("u") or urllib.parse.parse_qs(parsed.query).get("q")
|
|
390
|
+
if not wrapped:
|
|
391
|
+
continue
|
|
392
|
+
parsed = urllib.parse.urlsplit(urllib.parse.unquote(wrapped[0]))
|
|
393
|
+
if parsed.scheme not in ("http", "https") or not usable_destination(urllib.parse.urlunsplit(parsed)):
|
|
394
|
+
continue
|
|
395
|
+
return urllib.parse.urlunsplit(parsed)[:2000]
|
|
396
|
+
return None
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def _days(first: str | None, last: str | None) -> int | None:
|
|
400
|
+
return CF._days_between(first, last)
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def _alias(taxonomy: dict[str, Any], dimension: str, platform: str, raw: Any) -> str | None:
|
|
404
|
+
if not isinstance(raw, str) or not raw:
|
|
405
|
+
return None
|
|
406
|
+
return (taxonomy.get("aliases", {}).get(dimension) or {}).get(f"{platform}:{raw}")
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _creative(*, domain: str, platform: str, source: str, ad_id: str, advertiser: str | None, headline: str | None,
|
|
410
|
+
body: str | None, extra_texts: list[str] | None = None, cta_label: str | None = None, cta_code: str | None = None,
|
|
411
|
+
landing_url: str | None = None, format_raw: str | None = None, image_url: str | None = None,
|
|
412
|
+
first_seen: str | None, last_seen: str | None, served_days: int | None, running: bool | None,
|
|
413
|
+
age_basis: str, market_scope: str, verified_by: str, link: str | None, extra: dict[str, Any] | None = None,
|
|
414
|
+
taxonomy: dict[str, Any]) -> dict[str, Any]:
|
|
415
|
+
extra_texts = [t for t in (extra_texts or []) if t]
|
|
416
|
+
row = {
|
|
417
|
+
"type": "creative",
|
|
418
|
+
"creative_id": f"{platform}:{ad_id}",
|
|
419
|
+
"domain": domain,
|
|
420
|
+
"platform": platform,
|
|
421
|
+
"channel": PLATFORM_CHANNELS.get(platform, platform),
|
|
422
|
+
"source": source,
|
|
423
|
+
"ad_id": ad_id,
|
|
424
|
+
"competitor_tier": "unknown", "relevance": "unknown", "country": None, "countries": [],
|
|
425
|
+
"duration_scope": "global" if age_basis == "platform_dates" else "unknown",
|
|
426
|
+
"advertiser": advertiser,
|
|
427
|
+
"headline": headline,
|
|
428
|
+
"body": body,
|
|
429
|
+
"extra_texts": extra_texts,
|
|
430
|
+
"cta_label": cta_label,
|
|
431
|
+
"cta_code": cta_code,
|
|
432
|
+
"cta_intent": _alias(taxonomy, "cta_intent", platform, cta_code),
|
|
433
|
+
"landing_url": landing_url,
|
|
434
|
+
"landing_host": registrable_domain(landing_url) if landing_url else None,
|
|
435
|
+
"format_raw": format_raw,
|
|
436
|
+
"format_kind": _alias(taxonomy, "format_kind", platform, format_raw),
|
|
437
|
+
"image_url": image_url,
|
|
438
|
+
"first_seen": first_seen,
|
|
439
|
+
"last_seen": last_seen,
|
|
440
|
+
"served_days": served_days,
|
|
441
|
+
"running_at_capture": running,
|
|
442
|
+
"age_basis": age_basis,
|
|
443
|
+
"market_scope": market_scope,
|
|
444
|
+
"verified_by": verified_by,
|
|
445
|
+
"link": link,
|
|
446
|
+
"fingerprint": fingerprint(headline, body, image_url, f"{platform}:{ad_id}"),
|
|
447
|
+
"tier": "A",
|
|
448
|
+
}
|
|
449
|
+
row.update(extra or {})
|
|
450
|
+
return row
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
# ── collectors ──────────────────────────────────────────────────────────────
|
|
454
|
+
|
|
455
|
+
def _google_identity(rows: list[dict[str, Any]], domain: str) -> tuple[set[str], str]:
|
|
456
|
+
"""Which advertisers of a domain's Transparency rows are the competitor.
|
|
457
|
+
Names that are (or contain) the brand win; when none does, the verified
|
|
458
|
+
advertiser behind at least three rows and three quarters of the domain's
|
|
459
|
+
rows is the competitor (its legal entity, e.g. Ryzac, Inc. for
|
|
460
|
+
codecademy.com); otherwise none is. The rest are third parties."""
|
|
461
|
+
brand = brand_token(domain)
|
|
462
|
+
by_advertiser: dict[str, list[str]] = {}
|
|
463
|
+
for ad in rows:
|
|
464
|
+
key = str(ad.get("advertiserId") or ad.get("advertiserName") or "").strip()
|
|
465
|
+
by_advertiser.setdefault(key, []).append(str(ad.get("advertiserName") or "").strip())
|
|
466
|
+
named = {key for key, names in by_advertiser.items() if any(CF._name_matches(n, brand) or CF._mentions(n, brand) for n in names if n)}
|
|
467
|
+
if named:
|
|
468
|
+
return named, "destination_and_advertiser"
|
|
469
|
+
if not by_advertiser:
|
|
470
|
+
return set(), "destination_and_advertiser"
|
|
471
|
+
dominant, ads = sorted(by_advertiser.items(), key=lambda kv: (-len(kv[1]), kv[0]))[0]
|
|
472
|
+
total = sum(len(names) for names in by_advertiser.values())
|
|
473
|
+
if len(ads) < GOOGLE_DOMINANT_MIN_ROWS or len(ads) < GOOGLE_DOMINANT_MIN_SHARE * total:
|
|
474
|
+
return set(), "destination_and_advertiser"
|
|
475
|
+
return {dominant}, "destination_dominant_advertiser"
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def collect_google_transparency(client: OsClient, spec: dict[str, Any], domains: list[str], as_of: date, taxonomy: dict[str, Any], excluded: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
479
|
+
"""Ads Transparency Center by destination domain: format, first and last
|
|
480
|
+
shown, and the rendered image for text and image ads. Destination keying
|
|
481
|
+
does not prove the advertiser, so third-party advertisers are excluded."""
|
|
482
|
+
rows: list[dict[str, Any]] = []
|
|
483
|
+
batch = max(1, int(spec.get("batch_domains") or 5))
|
|
484
|
+
for start in range(0, len(domains), batch):
|
|
485
|
+
chunk = domains[start:start + batch]
|
|
486
|
+
result, retrieved_on = CF._call_with_context(client, ", ".join(chunk), "Google Ads Transparency", lambda chunk=chunk: client.run_actor(
|
|
487
|
+
spec["actor"], {"domains": chunk, "resultType": "ads", "region": "ANYWHERE", "maxAdsPerSearch": GOOGLE_ADS_PER_DOMAIN},
|
|
488
|
+
GOOGLE_ADS_PER_DOMAIN * len(chunk),
|
|
489
|
+
))
|
|
490
|
+
by_domain: dict[str, list[dict[str, Any]]] = {}
|
|
491
|
+
seen: set[str] = set()
|
|
492
|
+
for ad in result:
|
|
493
|
+
domain = registrable_domain(str(ad.get("domain") or ""))
|
|
494
|
+
creative_id = str(ad.get("creativeId") or "")
|
|
495
|
+
if domain in chunk and creative_id and creative_id not in seen:
|
|
496
|
+
seen.add(creative_id)
|
|
497
|
+
by_domain.setdefault(domain, []).append(ad)
|
|
498
|
+
for domain in chunk:
|
|
499
|
+
ads = by_domain.get(domain) or []
|
|
500
|
+
own, verified_by = _google_identity(ads, domain)
|
|
501
|
+
third_party = sorted({str(ad.get("advertiserName") or "")[:80] for ad in ads
|
|
502
|
+
if str(ad.get("advertiserId") or ad.get("advertiserName") or "").strip() not in own})
|
|
503
|
+
for ad in ads:
|
|
504
|
+
if str(ad.get("advertiserId") or ad.get("advertiserName") or "").strip() not in own:
|
|
505
|
+
continue
|
|
506
|
+
creative_id = str(ad["creativeId"])
|
|
507
|
+
first = str(ad.get("firstShown") or "")[:10] or None
|
|
508
|
+
last = str(ad.get("lastShown") or "")[:10] or None
|
|
509
|
+
served = ad.get("shownForDays") if isinstance(ad.get("shownForDays"), int) else _days(first, last)
|
|
510
|
+
fmt = str(ad.get("format") or "").upper() or None
|
|
511
|
+
platform = "youtube" if fmt == "VIDEO" else "google"
|
|
512
|
+
image = ad.get("imageUrl") if fmt in ("TEXT", "IMAGE") and isinstance(ad.get("imageUrl"), str) else None
|
|
513
|
+
gap = _days(last, as_of.isoformat()) if last else None
|
|
514
|
+
rows.append(_creative(
|
|
515
|
+
domain=domain, platform=platform, source="google_transparency", ad_id=creative_id,
|
|
516
|
+
advertiser=str(ad.get("advertiserName") or "").strip() or None, headline=None, body=None, format_raw=fmt, image_url=image,
|
|
517
|
+
first_seen=first, last_seen=last, served_days=served, running=None,
|
|
518
|
+
age_basis="platform_dates", market_scope="global", verified_by=verified_by,
|
|
519
|
+
link=ad.get("adUrl") or None, taxonomy=_google_taxonomy(taxonomy, platform), extra={"retrieved_on": retrieved_on},
|
|
520
|
+
))
|
|
521
|
+
if third_party:
|
|
522
|
+
excluded.append({"type": "coverage", "domain": domain, "source": "google_transparency", "status": "excluded",
|
|
523
|
+
"detail": f"{len(third_party)} third-party advertiser(s) pointing at the domain: {', '.join(third_party[:4])}", "tier": "C"})
|
|
524
|
+
return rows
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def _google_taxonomy(taxonomy: dict[str, Any], platform: str) -> dict[str, Any]:
|
|
528
|
+
# YouTube creatives come from the Google library and use its format codes.
|
|
529
|
+
if platform != "youtube":
|
|
530
|
+
return taxonomy
|
|
531
|
+
aliases = dict(taxonomy.get("aliases") or {})
|
|
532
|
+
aliases["format_kind"] = {**(aliases.get("format_kind") or {}), "youtube:VIDEO": "video"}
|
|
533
|
+
return {**taxonomy, "aliases": aliases}
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def collect_google_search(client: OsClient, domain: str, country: str, as_of: date, taxonomy: dict[str, Any]) -> list[dict[str, Any]]:
|
|
537
|
+
"""Preserve native SpyFu ads; capture spans describe sightings, not delivery."""
|
|
538
|
+
recent_days = SPYFU_RECENT_WINDOW_DAYS
|
|
539
|
+
windows = (((as_of - timedelta(days=recent_days)).isoformat(), as_of.isoformat()),
|
|
540
|
+
((as_of - timedelta(days=SPYFU_EARLIER_WINDOW[1])).isoformat(),
|
|
541
|
+
(as_of - timedelta(days=SPYFU_EARLIER_WINDOW[0])).isoformat()))
|
|
542
|
+
groups: dict[str, dict[str, Any]] = {}
|
|
543
|
+
for since, until in windows:
|
|
544
|
+
response, retrieved_on = client.spyfu_ad_history(domain, country, since, until, SPYFU_ROWS_PER_WINDOW)
|
|
545
|
+
for row in response.get("rows") or []:
|
|
546
|
+
if not isinstance(row, dict) or registrable_domain(str(row.get("display_domain") or row.get("url") or "")) != domain:
|
|
547
|
+
continue
|
|
548
|
+
headline, body = _clean_text(row.get("title"), None), _clean_text(row.get("body"), None)
|
|
549
|
+
seen_on = row.get("seen_on")
|
|
550
|
+
if not (headline or body) or not isinstance(seen_on, str) or not _DATE_RE.match(seen_on):
|
|
551
|
+
continue
|
|
552
|
+
native = str(row["ad_id"]) if row.get("ad_id") is not None else None
|
|
553
|
+
# Without a native ID, preserve each distinct source observation.
|
|
554
|
+
key = native or "observation-" + POLICY.canonical_hash(row)[:20]
|
|
555
|
+
group = groups.setdefault(key, {"headline": headline, "body": body, "dates": [],
|
|
556
|
+
"observations": [], "native": native, "retrieved_on": retrieved_on})
|
|
557
|
+
if row not in group["observations"]:
|
|
558
|
+
group["observations"].append(dict(row))
|
|
559
|
+
group["dates"].append(seen_on)
|
|
560
|
+
creatives = []
|
|
561
|
+
recent_start = (as_of - timedelta(days=recent_days)).isoformat()
|
|
562
|
+
for key, group in groups.items():
|
|
563
|
+
first, last = min(group["dates"]), max(group["dates"])
|
|
564
|
+
landing = destination_url([row.get("url") for row in group["observations"]])
|
|
565
|
+
creatives.append(_creative(
|
|
566
|
+
domain=domain, platform="google", source="spyfu_ad_history", ad_id=f"search-{domain}-{key}", advertiser=domain,
|
|
567
|
+
headline=group["headline"], body=group["body"], landing_url=landing, format_raw="TEXT",
|
|
568
|
+
first_seen=first, last_seen=last, served_days=None, running=None,
|
|
569
|
+
age_basis="capture_dates", market_scope="target", verified_by="display_domain", link=landing, taxonomy=taxonomy,
|
|
570
|
+
extra={"keywords": list(dict.fromkeys(row["keyword"] for row in group["observations"] if row.get("keyword"))),
|
|
571
|
+
"captures": len(group["dates"]), "spyfu_ad_ids": [group["native"]] if group["native"] else [],
|
|
572
|
+
"native_ad_id": group["native"] or key, "country": country, "countries": [country],
|
|
573
|
+
"duration_scope": "unknown", "source_observations": group["observations"], "retrieved_on": group["retrieved_on"]},
|
|
574
|
+
))
|
|
575
|
+
return creatives
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def _meta_identity(domain: str, brand: str, page_name: str, destinations: list[str]) -> str | None:
|
|
579
|
+
"""Destination first: a usable address decides; the advertiser name
|
|
580
|
+
decides only when the ad carries no usable destination."""
|
|
581
|
+
hosts = {registrable_domain(d) for d in (usable_destination(x) for x in destinations) if d}
|
|
582
|
+
if hosts:
|
|
583
|
+
return "destination" if domain in hosts else None
|
|
584
|
+
return "exact_name" if CF._name_matches(page_name, brand) else None
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
def _meta_actor_creative(ad: dict[str, Any], domain: str, brand: str, as_of: date, retrieved_on: str, taxonomy: dict[str, Any]) -> dict[str, Any] | None:
|
|
588
|
+
snapshot = ad.get("snapshot") if isinstance(ad.get("snapshot"), dict) else {}
|
|
589
|
+
cards = [c for c in (snapshot.get("cards") or []) if isinstance(c, dict)]
|
|
590
|
+
page_name = str(ad.get("page_name") or snapshot.get("page_name") or "")
|
|
591
|
+
destinations = [snapshot.get("link_url"), snapshot.get("caption")] + [c.get("link_url") for c in cards]
|
|
592
|
+
verified = _meta_identity(domain, brand, page_name, [d for d in destinations if isinstance(d, str)])
|
|
593
|
+
ad_id = str(ad.get("ad_archive_id") or ad.get("adArchiveID") or "")
|
|
594
|
+
if not verified or not ad_id:
|
|
595
|
+
return None
|
|
596
|
+
headline = _clean_text(snapshot.get("title"), 300) or next((_clean_text(c.get("title"), 300) for c in cards if _clean_text(c.get("title"), 300)), None)
|
|
597
|
+
body = _clean_text((snapshot.get("body") or {}).get("text") if isinstance(snapshot.get("body"), dict) else snapshot.get("body"), None) \
|
|
598
|
+
or next((_clean_text(c.get("body"), 1200) for c in cards if _clean_text(c.get("body"), 1200)), None)
|
|
599
|
+
extra_texts: list[str] = []
|
|
600
|
+
for value in [snapshot.get("link_description")] + [x for c in cards for x in (c.get("title"), c.get("body"), c.get("link_description"))]:
|
|
601
|
+
text = _clean_text(value, None)
|
|
602
|
+
if text and text not in (headline, body) and text not in extra_texts:
|
|
603
|
+
extra_texts.append(text)
|
|
604
|
+
images = snapshot.get("images") if isinstance(snapshot.get("images"), list) else []
|
|
605
|
+
videos = snapshot.get("videos") if isinstance(snapshot.get("videos"), list) else []
|
|
606
|
+
image = next((i.get("original_image_url") or i.get("resized_image_url") for i in images if isinstance(i, dict) and (i.get("original_image_url") or i.get("resized_image_url"))), None) \
|
|
607
|
+
or next((v.get("video_preview_image_url") for v in videos if isinstance(v, dict) and v.get("video_preview_image_url")), None) \
|
|
608
|
+
or next((c.get("original_image_url") or c.get("resized_image_url") or c.get("video_preview_image_url") for c in cards if c.get("original_image_url") or c.get("resized_image_url") or c.get("video_preview_image_url")), None)
|
|
609
|
+
landing = destination_url(destinations)
|
|
610
|
+
first = CF._iso_from_epoch(ad.get("start_date"))
|
|
611
|
+
active = bool(ad.get("is_active"))
|
|
612
|
+
# A resumed attempt must not accrue evidence after its immutable cutoff.
|
|
613
|
+
last = min(retrieved_on, as_of.isoformat()) if active else CF._iso_from_epoch(ad.get("end_date"))
|
|
614
|
+
fmt = str(snapshot.get("display_format") or "").upper() or None
|
|
615
|
+
cta_code = snapshot.get("cta_type") or next((c.get("cta_type") for c in cards if c.get("cta_type")), None)
|
|
616
|
+
return _creative(
|
|
617
|
+
domain=domain, platform="meta", source="meta_library_actor", ad_id=ad_id, advertiser=page_name or None,
|
|
618
|
+
headline=headline, body=body, extra_texts=extra_texts, cta_label=_clean_text(snapshot.get("cta_text"), 60),
|
|
619
|
+
cta_code=cta_code if isinstance(cta_code, str) else None, landing_url=landing, format_raw=fmt, image_url=image,
|
|
620
|
+
first_seen=first, last_seen=last, served_days=_days(first, last), running=active, age_basis="platform_dates",
|
|
621
|
+
market_scope="target", verified_by=verified, link=f"https://www.facebook.com/ads/library/?id={ad_id}", taxonomy=taxonomy,
|
|
622
|
+
extra={"collation_count": ad.get("collation_count") if isinstance(ad.get("collation_count"), int) else None,
|
|
623
|
+
"publisher_platforms": [p for p in (ad.get("publisher_platform") or []) if isinstance(p, str)][:6],
|
|
624
|
+
"retrieved_on": retrieved_on,
|
|
625
|
+
"video_url": next((v.get("video_hd_url") or v.get("video_sd_url") or v.get("video_url") for v in videos + cards if isinstance(v, dict) and (v.get("video_hd_url") or v.get("video_sd_url") or v.get("video_url"))), None),
|
|
626
|
+
"media_urls": list(dict.fromkeys(u for v in images + videos + cards if isinstance(v, dict) for u in [v.get("original_image_url") or v.get("resized_image_url") or v.get("video_preview_image_url")] if u)),
|
|
627
|
+
"media_cards": [{k: c.get(k) for k in ("title","body","link_description","original_image_url","resized_image_url","video_preview_image_url","video_hd_url","video_sd_url","link_url")} for c in cards]},
|
|
628
|
+
)
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
def _eu_demographics(row: dict[str, Any]) -> dict[str, Any] | None:
|
|
632
|
+
breakdown = row.get("age_country_gender_reach_breakdown")
|
|
633
|
+
if not isinstance(breakdown, list):
|
|
634
|
+
return None
|
|
635
|
+
by_age: dict[str, int] = {}
|
|
636
|
+
by_gender = {"male": 0, "female": 0, "unknown": 0}
|
|
637
|
+
for country in breakdown:
|
|
638
|
+
for item in (country.get("age_gender_breakdowns") or []) if isinstance(country, dict) else []:
|
|
639
|
+
if not isinstance(item, dict):
|
|
640
|
+
continue
|
|
641
|
+
total = sum(int(item.get(g) or 0) for g in by_gender)
|
|
642
|
+
by_age[str(item.get("age_range"))] = by_age.get(str(item.get("age_range")), 0) + total
|
|
643
|
+
for g in by_gender:
|
|
644
|
+
by_gender[g] += int(item.get(g) or 0)
|
|
645
|
+
total = sum(by_gender.values())
|
|
646
|
+
if not total:
|
|
647
|
+
return None
|
|
648
|
+
top_ages = sorted(by_age.items(), key=lambda kv: -kv[1])[:2]
|
|
649
|
+
return {"top_age_ranges": [age for age, _ in top_ages], "female_share": round(by_gender["female"] / total, 2),
|
|
650
|
+
"reach_counted": total, "target_ages": row.get("target_ages") if isinstance(row.get("target_ages"), list) else None,
|
|
651
|
+
"target_gender": row.get("target_gender") if isinstance(row.get("target_gender"), str) else None}
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
def _meta_official_creative(row: dict[str, Any], domain: str, brand: str, as_of: date, retrieved_on: str, taxonomy: dict[str, Any]) -> dict[str, Any] | None:
|
|
655
|
+
def first_text(key: str, cap: int) -> str | None:
|
|
656
|
+
values = row.get(key)
|
|
657
|
+
values = [values] if isinstance(values, str) else (values if isinstance(values, list) else [])
|
|
658
|
+
return next((_clean_text(v, cap) for v in values if _clean_text(v, cap)), None)
|
|
659
|
+
|
|
660
|
+
captions = row.get("ad_creative_link_captions")
|
|
661
|
+
captions = [captions] if isinstance(captions, str) else [c for c in (captions or []) if isinstance(c, str)]
|
|
662
|
+
verified = _meta_identity(domain, brand, str(row.get("page_name") or ""), captions)
|
|
663
|
+
ad_id = str(row.get("id") or "")
|
|
664
|
+
if not verified or not ad_id.isdigit():
|
|
665
|
+
return None
|
|
666
|
+
first = str(row.get("ad_delivery_start_time") or "")[:10] or None
|
|
667
|
+
stop = str(row.get("ad_delivery_stop_time") or "")[:10] or None
|
|
668
|
+
last = stop or min(retrieved_on, as_of.isoformat())
|
|
669
|
+
return _creative(
|
|
670
|
+
domain=domain, platform="meta", source="meta_library_official_eu", ad_id=ad_id, advertiser=row.get("page_name"),
|
|
671
|
+
headline=first_text("ad_creative_link_titles", 300), body=first_text("ad_creative_bodies", None),
|
|
672
|
+
extra_texts=[t for key in ("ad_creative_bodies", "ad_creative_link_titles", "ad_creative_link_descriptions") for d in (row.get(key) if isinstance(row.get(key), list) else [row.get(key)]) if (t := _clean_text(d, None))], landing_url=None, format_raw=None,
|
|
673
|
+
first_seen=first, last_seen=last, served_days=_days(first, last), running=stop is None, age_basis="platform_dates",
|
|
674
|
+
market_scope="eu_proxy", verified_by=verified,
|
|
675
|
+
# The public permalink; the Graph render URL carries an access token.
|
|
676
|
+
link=f"https://www.facebook.com/ads/library/?id={ad_id}", taxonomy=taxonomy,
|
|
677
|
+
extra={"eu_total_reach": row.get("eu_total_reach") if isinstance(row.get("eu_total_reach"), int) else None,
|
|
678
|
+
"eu_demographics": _eu_demographics(row),
|
|
679
|
+
"publisher_platforms": [p for p in (row.get("publisher_platforms") or []) if isinstance(p, str)][:6],
|
|
680
|
+
"retrieved_on": retrieved_on},
|
|
681
|
+
)
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
META_EU_FIELDS = ",".join((
|
|
685
|
+
"id", "page_id", "page_name", "ad_delivery_start_time", "ad_delivery_stop_time", "ad_creative_bodies",
|
|
686
|
+
"ad_creative_link_titles", "ad_creative_link_descriptions", "ad_creative_link_captions", "publisher_platforms",
|
|
687
|
+
"languages", "eu_total_reach", "age_country_gender_reach_breakdown", "target_ages", "target_gender",
|
|
688
|
+
"target_locations", "beneficiary_payers",
|
|
689
|
+
))
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
def collect_meta(client: OsClient, spec: dict[str, Any], domain: str, as_of: date, eu_country: str, taxonomy: dict[str, Any]) -> tuple[list[dict[str, Any]], int]:
|
|
693
|
+
"""The creatives plus the number of US page-search rows: fewer rows than
|
|
694
|
+
requested means the listing was complete, the only case in which an ad
|
|
695
|
+
missing at a later cutoff can be read as stopped."""
|
|
696
|
+
brand = brand_token(domain)
|
|
697
|
+
# Running ads only: an all-status sample of 30 is mostly stopped ads of
|
|
698
|
+
# global brands; stopped ones come from the EU data and earlier snapshots.
|
|
699
|
+
search_url = CF.meta_search_url(brand, "US", active_status="active")
|
|
700
|
+
result, retrieved_on = CF._call_with_context(client, domain, "Meta Ad Library page search", lambda: client.run_actor(
|
|
701
|
+
spec["actor"], {"urls": [{"url": search_url}], "count": META_ADS_PER_BRAND}, META_ADS_PER_BRAND,
|
|
702
|
+
))
|
|
703
|
+
rows = []
|
|
704
|
+
seen = set()
|
|
705
|
+
for ad in result:
|
|
706
|
+
creative = _meta_actor_creative(ad, domain, brand, as_of, retrieved_on, taxonomy)
|
|
707
|
+
if creative and creative["ad_id"] not in seen:
|
|
708
|
+
seen.add(creative["ad_id"])
|
|
709
|
+
creative.update(country="US", countries=["US"], duration_scope="global")
|
|
710
|
+
rows.append(creative)
|
|
711
|
+
try:
|
|
712
|
+
official, eu_retrieved = CF._call_with_context(client, domain, "Meta Ad Library EU", lambda: client.meta_library_search(
|
|
713
|
+
{"query": brand, "countries": [eu_country], "activeStatus": "ALL", "limit": META_EU_ADS_PER_BRAND, "fields": META_EU_FIELDS},
|
|
714
|
+
))
|
|
715
|
+
except BudgetExhausted as exc:
|
|
716
|
+
if "cumulative" not in str(exc):
|
|
717
|
+
raise
|
|
718
|
+
client.budget_gaps.append({"type": "coverage", "domain": domain, "country": eu_country, "channel": "meta_ads", "source": "meta_library_official_eu", "status": "budget_deferred", "detail": str(exc), "tier": "C"})
|
|
719
|
+
return rows, len(result)
|
|
720
|
+
seen = set()
|
|
721
|
+
for row in official:
|
|
722
|
+
creative = _meta_official_creative(row, domain, brand, as_of, eu_retrieved, taxonomy)
|
|
723
|
+
if creative and creative["ad_id"] not in seen:
|
|
724
|
+
seen.add(creative["ad_id"])
|
|
725
|
+
creative.update(country=eu_country, countries=[eu_country], duration_scope="global")
|
|
726
|
+
rows.append(creative)
|
|
727
|
+
# Same ID in different markets stays separate. Global delivery dates and
|
|
728
|
+
# EU aggregate reach must not strengthen the US observation.
|
|
729
|
+
return rows, len(result)
|
|
730
|
+
|
|
731
|
+
|
|
732
|
+
def collect_linkedin(client: OsClient, spec: dict[str, Any], domain: str, slug: str | None, taxonomy: dict[str, Any]) -> list[dict[str, Any]]:
|
|
733
|
+
query = f"https://www.linkedin.com/company/{slug or brand_token(domain)}"
|
|
734
|
+
result, retrieved_on = CF._call_with_context(client, domain, "LinkedIn Ad Library", lambda: client.run_actor(
|
|
735
|
+
spec["actor"], {"queries": [query], "maxResults": LINKEDIN_ADS_PER_COMPANY, "enrich": True}, LINKEDIN_ADS_PER_COMPANY,
|
|
736
|
+
))
|
|
737
|
+
rows: list[dict[str, Any]] = []
|
|
738
|
+
seen: set[str] = set()
|
|
739
|
+
for ad in result:
|
|
740
|
+
ad_id = str(ad.get("ad_id") or "")
|
|
741
|
+
if not ad_id or ad_id in seen or str(ad.get("query") or "").rstrip("/") != query \
|
|
742
|
+
or not CF._name_matches(str(ad.get("advertiser_name") or ""), brand_token(domain)):
|
|
743
|
+
continue
|
|
744
|
+
seen.add(ad_id)
|
|
745
|
+
images = [u for u in (ad.get("image_urls") or []) if isinstance(u, str)]
|
|
746
|
+
rows.append(_creative(
|
|
747
|
+
domain=domain, platform="linkedin", source="linkedin_library_actor", ad_id=ad_id, advertiser=ad.get("advertiser_name"),
|
|
748
|
+
headline=_clean_text(ad.get("headline"), 300), body=_clean_text(ad.get("ad_copy"), 1200), format_raw=str(ad.get("creative_format") or "") or None,
|
|
749
|
+
image_url=images[0] if images else None, first_seen=None, last_seen=None, served_days=None, running=None,
|
|
750
|
+
age_basis="none", market_scope="global", verified_by="company_page_query", link=ad.get("detail_url"), taxonomy=taxonomy,
|
|
751
|
+
extra={"retrieved_on": retrieved_on},
|
|
752
|
+
))
|
|
753
|
+
return rows
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
def collect_reddit(client: OsClient, spec: dict[str, Any], domain: str, taxonomy: dict[str, Any]) -> list[dict[str, Any]]:
|
|
757
|
+
brand = brand_token(domain)
|
|
758
|
+
result, retrieved_on = CF._call_with_context(client, domain, "Reddit ad library", lambda: client.run_actor(
|
|
759
|
+
spec["actor"], {"query": brand, "maxItems": REDDIT_ADS_PER_BRAND}, REDDIT_ADS_PER_BRAND,
|
|
760
|
+
))
|
|
761
|
+
rows: list[dict[str, Any]] = []
|
|
762
|
+
for ad in result:
|
|
763
|
+
ad_id = str(ad.get("id") or ad.get("creativeId") or "")
|
|
764
|
+
advertiser = str(ad.get("advertiserName") or "")
|
|
765
|
+
verified = _meta_identity(domain, brand, advertiser.removeprefix("u_"), [ad.get("destinationUrl") or ""])
|
|
766
|
+
if not verified or not ad_id:
|
|
767
|
+
continue
|
|
768
|
+
created = str(ad.get("createdAt") or "")[:10] or None
|
|
769
|
+
rows.append(_creative(
|
|
770
|
+
domain=domain, platform="reddit", source="reddit_library_actor", ad_id=ad_id, advertiser=advertiser or None,
|
|
771
|
+
headline=_clean_text(ad.get("headline"), 300), body=_clean_text(ad.get("body"), 1200),
|
|
772
|
+
cta_label=_clean_text(ad.get("callToAction"), 60), landing_url=destination_url([ad.get("destinationUrl")]),
|
|
773
|
+
format_raw=str(ad.get("creativeType") or "") or None, image_url=ad.get("mediaUrl") or ad.get("thumbnailUrl"),
|
|
774
|
+
first_seen=created, last_seen=created, served_days=None, running=None, age_basis="creation_only", market_scope="global",
|
|
775
|
+
verified_by=verified, link=ad.get("postUrl"), taxonomy=taxonomy,
|
|
776
|
+
extra={"objective": ad.get("objective"), "budget_category": ad.get("budgetCategory"),
|
|
777
|
+
"placements": [p for p in (ad.get("placements") or []) if isinstance(p, str)][:4], "retrieved_on": retrieved_on},
|
|
778
|
+
))
|
|
779
|
+
return rows
|
|
780
|
+
|
|
781
|
+
|
|
782
|
+
# ── longevity and selection ─────────────────────────────────────────────────
|
|
783
|
+
|
|
784
|
+
def cap_at_cutoff(creatives: list[dict[str, Any]], as_of: date) -> list[dict[str, Any]]:
|
|
785
|
+
"""No evidence after the cutoff, whatever the source: an ad first seen
|
|
786
|
+
after it did not exist yet, and one last seen after it was still running
|
|
787
|
+
at the cutoff, aged up to the cutoff only. Matters for an explicit
|
|
788
|
+
past --as-of and for calls made when a cutoff resumes on a later day."""
|
|
789
|
+
cutoff = as_of.isoformat()
|
|
790
|
+
kept: list[dict[str, Any]] = []
|
|
791
|
+
for creative in creatives:
|
|
792
|
+
first, last = creative.get("first_seen"), creative.get("last_seen")
|
|
793
|
+
if first and first > cutoff:
|
|
794
|
+
continue
|
|
795
|
+
if last and last > cutoff:
|
|
796
|
+
span = _days(first, cutoff) if first else None
|
|
797
|
+
served = creative.get("served_days")
|
|
798
|
+
if isinstance(served, int) and isinstance(span, int):
|
|
799
|
+
served = min(served, span)
|
|
800
|
+
elif isinstance(span, int):
|
|
801
|
+
served = span
|
|
802
|
+
creative = {**creative, "last_seen": cutoff, "served_days": served, "running_at_capture": True}
|
|
803
|
+
kept.append(creative)
|
|
804
|
+
return kept
|
|
805
|
+
|
|
806
|
+
|
|
807
|
+
def longevity_status(creative: dict[str, Any], long_running_days: int) -> str:
|
|
808
|
+
return MODEL.status(creative, long_running_days)
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
def deduplicate(creatives: list[dict[str, Any]], long_running_days: int) -> list[dict[str, Any]]:
|
|
812
|
+
return MODEL.individual_ads(creatives, long_running_days)
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
def load_snapshots(runtime: Path, as_of: date) -> list[tuple[date, dict[str, Any]]]:
|
|
816
|
+
"""Snapshots stored at earlier cutoffs (`--refresh` runs), oldest first."""
|
|
817
|
+
history = []
|
|
818
|
+
for path in sorted((runtime / "snapshots").glob("*.json")):
|
|
819
|
+
try:
|
|
820
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
821
|
+
cutoff = date.fromisoformat(data["as_of"])
|
|
822
|
+
except (OSError, ValueError, KeyError, json.JSONDecodeError):
|
|
823
|
+
continue
|
|
824
|
+
if cutoff < as_of and isinstance(data.get("creatives"), list):
|
|
825
|
+
data["creatives"] = [c for c in data["creatives"] if isinstance(c, dict) and c.get("fingerprint") and c.get("creative_id")]
|
|
826
|
+
history.append((cutoff, data))
|
|
827
|
+
return history
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
def _complete_listing(listings: dict[str, Any], domain: str) -> bool:
|
|
831
|
+
rows = (listings.get(domain) or {}).get("meta_us_rows")
|
|
832
|
+
return isinstance(rows, int) and not isinstance(rows, bool) and 0 <= rows < META_ADS_PER_BRAND
|
|
833
|
+
|
|
834
|
+
|
|
835
|
+
def merge_snapshots(creatives: list[dict[str, Any]], history: list[tuple[date, dict[str, Any]]], as_of: date,
|
|
836
|
+
long_running_days: int, domains: list[str], listings: dict[str, Any]) -> list[dict[str, Any]]:
|
|
837
|
+
"""Fold earlier cutoffs into this one, claiming only what the samples prove.
|
|
838
|
+
|
|
839
|
+
- Libraries that keep stopped ads listed (LinkedIn, Reddit) get a listing
|
|
840
|
+
history (`first_listed`, `snapshot_sightings`), never an age or a
|
|
841
|
+
running status.
|
|
842
|
+
- A US Meta ad that was running at an earlier cutoff and is missing now
|
|
843
|
+
has stopped (the US library lists active ads only), but only when this
|
|
844
|
+
cutoff queried that competitor's Meta listing and both listings were
|
|
845
|
+
complete (fewer rows than requested); a missing ad in a truncated sample
|
|
846
|
+
may simply be outside it. It comes back once, as `vanished_after` the
|
|
847
|
+
last cutoff that saw it.
|
|
848
|
+
"""
|
|
849
|
+
if not history:
|
|
850
|
+
return creatives
|
|
851
|
+
seen_at: dict[tuple, list[date]] = {}
|
|
852
|
+
for cutoff, snapshot in history:
|
|
853
|
+
for record in snapshot["creatives"]:
|
|
854
|
+
key = (record["domain"], record["platform"], record.get("native_ad_id") or record["creative_id"], record.get("country"))
|
|
855
|
+
if cutoff not in seen_at.setdefault(key, []):
|
|
856
|
+
seen_at[key].append(cutoff)
|
|
857
|
+
current_ids = {c["creative_id"] for c in creatives}
|
|
858
|
+
current_keys = set()
|
|
859
|
+
for c in creatives:
|
|
860
|
+
key = (c["domain"], c["platform"], c.get("native_ad_id") or c["creative_id"], c.get("country"))
|
|
861
|
+
current_keys.add(key)
|
|
862
|
+
dates = seen_at.get(key)
|
|
863
|
+
if not dates:
|
|
864
|
+
continue
|
|
865
|
+
c["snapshot_sightings"] = len(dates) + 1
|
|
866
|
+
c["first_listed"] = min(dates).isoformat()
|
|
867
|
+
if c.get("source") in LISTS_STOPPED_ADS:
|
|
868
|
+
continue # listed is not delivered: no age, no running status
|
|
869
|
+
vanished: dict[str, dict[str, Any]] = {}
|
|
870
|
+
for cutoff, snapshot in history:
|
|
871
|
+
for record in snapshot["creatives"]:
|
|
872
|
+
domain = record["domain"]
|
|
873
|
+
key = (domain, record["platform"], record.get("native_ad_id") or record["creative_id"], record.get("country"))
|
|
874
|
+
if (record.get("source") != "meta_library_actor" or record.get("running_at_capture") is not True
|
|
875
|
+
or key in current_keys or record["creative_id"] in current_ids or domain not in domains
|
|
876
|
+
or not _complete_listing(listings, domain) or not _complete_listing(snapshot.get("listings") or {}, domain)):
|
|
877
|
+
continue
|
|
878
|
+
row = dict(record)
|
|
879
|
+
last = max(seen_at[key]).isoformat()
|
|
880
|
+
row.update(running_at_capture=False, vanished_after=last, snapshot_sightings=len(seen_at[key]),
|
|
881
|
+
last_observed=last)
|
|
882
|
+
# Listing sightings never extend native delivery dates or duration.
|
|
883
|
+
row["longevity_status"] = longevity_status(row, long_running_days)
|
|
884
|
+
vanished[record["creative_id"]] = row # the latest cutoff's record wins
|
|
885
|
+
return creatives + [vanished[k] for k in sorted(vanished)]
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
def write_snapshot(runtime: Path, as_of: date, creatives: list[dict[str, Any]], listings: dict[str, Any], domains: list[str],
|
|
889
|
+
sources: list[str]) -> None:
|
|
890
|
+
_write_atomic(runtime / "snapshots" / f"{as_of.isoformat()}.json",
|
|
891
|
+
json.dumps({"as_of": as_of.isoformat(), "domains": domains, "sources": sources, "listings": listings,
|
|
892
|
+
"creatives": creatives}, ensure_ascii=False))
|
|
893
|
+
|
|
894
|
+
|
|
895
|
+
def select_for_classification(creatives: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
896
|
+
groups: dict[tuple, list] = {}
|
|
897
|
+
for c in creatives:
|
|
898
|
+
if c.get("relevance") == "irrelevant" or c.get("language_status") == "excluded":
|
|
899
|
+
continue
|
|
900
|
+
if c.get("headline") or c.get("body") or c.get("extra_texts") or c.get("image_url") or c.get("transcript") or c.get("media_analysis") or c.get("video_metadata"):
|
|
901
|
+
groups.setdefault((c.get("country") or "", c.get("channel") or c.get("platform") or "unknown", c.get("creative_type")), []).append(c)
|
|
902
|
+
for rows in groups.values():
|
|
903
|
+
rows.sort(key=MODEL.strength_key)
|
|
904
|
+
selected = []
|
|
905
|
+
while groups:
|
|
906
|
+
for key in sorted(list(groups), key=lambda k: (MODEL.strength_key(groups[k][0])[0], groups[k][0].get("channel_priority", 999), str(k))):
|
|
907
|
+
selected.append(groups[key].pop(0))
|
|
908
|
+
if not groups[key]:
|
|
909
|
+
del groups[key]
|
|
910
|
+
return selected
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
# ── classification ──────────────────────────────────────────────────────────
|
|
914
|
+
|
|
915
|
+
def classification_extra_texts(creative: dict) -> list[tuple[str, dict]]:
|
|
916
|
+
origins = creative.get("extra_text_sources") or []
|
|
917
|
+
extra = [(text, dict(origins[index]) if index < len(origins) and isinstance(origins[index], dict)
|
|
918
|
+
else {"kind": "ad_text", "ref": creative["creative_id"]})
|
|
919
|
+
for index, text in enumerate(creative.get("extra_texts") or []) if isinstance(text, str)]
|
|
920
|
+
metadata = creative.get("video_metadata") or {}
|
|
921
|
+
video_ref = f"youtube:{creative.get('youtube_video_id') or metadata.get('video_id') or creative['creative_id']}"
|
|
922
|
+
for field in ("title", "description"):
|
|
923
|
+
if isinstance(metadata.get(field), str) and metadata[field].strip():
|
|
924
|
+
extra.append((metadata[field], {"kind": f"video_metadata_{field}", "ref": video_ref}))
|
|
925
|
+
for frame in (creative.get("media_analysis") or {}).get("frames") or []:
|
|
926
|
+
extra.extend((text, {"kind": "video_frame_ocr", "ref": frame.get("ref") or creative["creative_id"]})
|
|
927
|
+
for text in frame.get("visible_text") or [] if isinstance(text, str))
|
|
928
|
+
return extra
|
|
929
|
+
|
|
930
|
+
|
|
931
|
+
def _classify_input(creative: dict[str, Any], frame_ref: str | None) -> dict[str, Any]:
|
|
932
|
+
item: dict[str, Any] = {"ad_ref": creative["creative_id"], "platform": "google" if creative["platform"] == "youtube" else creative["platform"]}
|
|
933
|
+
for key, cap in (("advertiser", 200), ("headline", 600), ("body", 4000), ("cta_label", 100), ("landing_url", 2000)):
|
|
934
|
+
value = creative.get(key)
|
|
935
|
+
if isinstance(value, str) and value.strip():
|
|
936
|
+
item[key] = value[:cap]
|
|
937
|
+
if isinstance(creative.get("transcript"), str) and creative["transcript"].strip():
|
|
938
|
+
item["transcript"] = creative["transcript"][:8000]
|
|
939
|
+
if isinstance(creative.get("video_sha256"), str) and re.fullmatch(r"[0-9a-f]{64}", creative["video_sha256"]):
|
|
940
|
+
item["video_sha256"] = creative["video_sha256"]
|
|
941
|
+
analysis = creative.get("media_analysis")
|
|
942
|
+
if isinstance(analysis, dict) and analysis:
|
|
943
|
+
derived = {
|
|
944
|
+
"schema_version": 1, "derived": True, "truncated": False,
|
|
945
|
+
"source": "media/analyze", "source_input_sha256": (creative.get("video_analysis_receipt") or {}).get("input_sha256"),
|
|
946
|
+
"video_analysis_status": creative.get("video_analysis_status"),
|
|
947
|
+
"frames": [{"ref": frame.get("ref"), "observations": frame.get("observations") or []}
|
|
948
|
+
for frame in analysis.get("frames") or [] if isinstance(frame, dict)],
|
|
949
|
+
"sequence_summary": analysis.get("sequence_summary"),
|
|
950
|
+
"visual_claims": [claim for claim in analysis.get("claims") or []
|
|
951
|
+
if isinstance(claim, dict) and claim.get("modality") == "visual"],
|
|
952
|
+
"gaps": analysis.get("gaps") or [],
|
|
953
|
+
}
|
|
954
|
+
encoded = json.dumps(derived, ensure_ascii=False, sort_keys=True)
|
|
955
|
+
if len(encoded) > 12000:
|
|
956
|
+
# Keep the wire payload valid JSON; never silently slice a JSON
|
|
957
|
+
# object or present derived descriptions as original ad words.
|
|
958
|
+
wrapped = {"schema_version": 1, "derived": True, "truncated": True,
|
|
959
|
+
"source": "media/analyze", "original_chars": len(encoded),
|
|
960
|
+
"original_sha256": hashlib.sha256(encoded.encode()).hexdigest(),
|
|
961
|
+
"observations_excerpt": encoded[:11000]}
|
|
962
|
+
shortened = json.dumps(wrapped, ensure_ascii=False, sort_keys=True)
|
|
963
|
+
while len(shortened) > 12000:
|
|
964
|
+
wrapped["observations_excerpt"] = wrapped["observations_excerpt"][:max(0, len(wrapped["observations_excerpt"]) - (len(shortened) - 12000) - 1)]
|
|
965
|
+
shortened = json.dumps(wrapped, ensure_ascii=False, sort_keys=True)
|
|
966
|
+
encoded = shortened
|
|
967
|
+
item["video_observations"] = encoded
|
|
968
|
+
extra = classification_extra_texts(creative)
|
|
969
|
+
if extra:
|
|
970
|
+
item["extra_texts"] = [text[:4000] for text, _ in extra[:16]]
|
|
971
|
+
item["extra_text_sources"] = [source for _, source in extra[:16]]
|
|
972
|
+
item["country"] = creative.get("country")
|
|
973
|
+
item["allowed_languages"] = creative.get("allowed_languages")
|
|
974
|
+
if creative.get("format_kind"):
|
|
975
|
+
item["format_hint"] = creative["format_kind"]
|
|
976
|
+
if creative.get("cta_intent"):
|
|
977
|
+
item["cta_hint"] = creative["cta_intent"]
|
|
978
|
+
if frame_ref:
|
|
979
|
+
item["frame_refs"] = [frame_ref]
|
|
980
|
+
return item
|
|
981
|
+
|
|
982
|
+
|
|
983
|
+
def _label_key(item: dict[str, Any], image: dict[str, Any] | None, taxonomy_sha256: str, product_context: str, semantic_context: dict | None = None, source_content_sha256: str | None = None) -> str:
|
|
984
|
+
"""Per-creative cache key: the classification depends only on the ad's
|
|
985
|
+
content, its image, the taxonomy and the product context."""
|
|
986
|
+
content = {k: v for k, v in item.items() if k not in ("ad_ref", "frame_refs")}
|
|
987
|
+
value = {"kind": "creative_label", "classification_contract": 4, "taxonomy_sha256": taxonomy_sha256, "context_sha256": hashlib.sha256(product_context.encode("utf-8")).hexdigest(),
|
|
988
|
+
"semantic_context": semantic_context, "source_content_sha256": source_content_sha256,
|
|
989
|
+
"content": content, "image_sha256": image["sha256"] if image else None}
|
|
990
|
+
return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False).encode("utf-8")).hexdigest()
|
|
991
|
+
|
|
992
|
+
|
|
993
|
+
def classification_source_hash(creative: dict) -> str:
|
|
994
|
+
return POLICY.canonical_hash({name: creative.get(name) for name in (
|
|
995
|
+
"headline", "body", "extra_texts", "extra_text_sources", "transcript", "video_metadata", "media_analysis", "video_sha256")})
|
|
996
|
+
|
|
997
|
+
|
|
998
|
+
def classify_corpus(client: OsClient, creatives: list[dict[str, Any]], taxonomy: dict[str, Any], taxonomy_sha256: str,
|
|
999
|
+
product_context: str, coverage: list[dict[str, Any]], cached_only: list[dict[str, Any]] = ()) -> list[dict[str, Any]]:
|
|
1000
|
+
"""Labels for `creatives` (cached per creative, new ones in batches) and,
|
|
1001
|
+
from the cache only, for `cached_only` (ads that stopped since an earlier
|
|
1002
|
+
cutoff were classified then)."""
|
|
1003
|
+
images: dict[str, dict[str, Any]] = {}
|
|
1004
|
+
deferred: dict[str, set[str]] = {}
|
|
1005
|
+
for creative in [*creatives, *cached_only]:
|
|
1006
|
+
url = creative.get("image_url")
|
|
1007
|
+
if isinstance(url, str) and url.startswith("https://"):
|
|
1008
|
+
image = client.fetch_image(url)
|
|
1009
|
+
if image and image.get("transient"):
|
|
1010
|
+
# Classifying it text-only now would cache a weaker label; it waits for the image.
|
|
1011
|
+
deferred.setdefault(creative["domain"], set()).add(creative["creative_id"])
|
|
1012
|
+
elif image:
|
|
1013
|
+
images[creative["creative_id"]] = image
|
|
1014
|
+
for domain, ids in sorted(deferred.items()):
|
|
1015
|
+
CF._coverage_gap(coverage, "classify", [domain], f"{len(ids)} image(s) could not be downloaded now; those creatives are classified on the next attempt")
|
|
1016
|
+
|
|
1017
|
+
def frame_ref(creative: dict[str, Any]) -> str:
|
|
1018
|
+
return f"frames/{creative['creative_id'].replace(':', '-')}.img"
|
|
1019
|
+
|
|
1020
|
+
labels: dict[str, tuple[dict[str, Any], dict[str, Any]]] = {}
|
|
1021
|
+
pending: list[tuple[dict[str, Any], str]] = []
|
|
1022
|
+
allowed_new = {c["creative_id"] for c in creatives}
|
|
1023
|
+
for creative in [*creatives, *cached_only]:
|
|
1024
|
+
cid = creative["creative_id"]
|
|
1025
|
+
if cid in labels or any(cid in ids for ids in deferred.values()):
|
|
1026
|
+
continue
|
|
1027
|
+
has_image = cid in images
|
|
1028
|
+
analyzed_frame_text = any(frame.get("visible_text") for frame in (creative.get("media_analysis") or {}).get("frames") or [])
|
|
1029
|
+
if not has_image and not (creative.get("headline") or creative.get("body") or creative.get("extra_texts")
|
|
1030
|
+
or creative.get("transcript") or analyzed_frame_text
|
|
1031
|
+
or (creative.get("video_metadata") or {}).get("title")
|
|
1032
|
+
or (creative.get("video_metadata") or {}).get("description")):
|
|
1033
|
+
continue
|
|
1034
|
+
item = _classify_input(creative, frame_ref(creative) if has_image else None)
|
|
1035
|
+
source_sha = classification_source_hash(creative)
|
|
1036
|
+
creative["classification_source_sha256"] = source_sha
|
|
1037
|
+
limits = []
|
|
1038
|
+
for name, maximum in (("headline", 600), ("body", 4000), ("transcript", 8000)):
|
|
1039
|
+
if len(creative.get(name) or "") > maximum:
|
|
1040
|
+
limits.append(f"{name}: first {maximum} characters supplied to classification")
|
|
1041
|
+
extra = classification_extra_texts(creative)
|
|
1042
|
+
if len(extra) > 16 or any(len(text) > 4000 for text, _ in extra):
|
|
1043
|
+
limits.append("extra texts (native copy, video metadata and OCR): at most 16 sources, 4000 characters each supplied to classification")
|
|
1044
|
+
if item.get("video_observations") and json.loads(item["video_observations"]).get("truncated"):
|
|
1045
|
+
limits.append("derived video observations: bounded JSON excerpt supplied to classification")
|
|
1046
|
+
if limits:
|
|
1047
|
+
creative["classification_input_limits"] = limits
|
|
1048
|
+
coverage.append({"type": "coverage", "creative_id": cid, "domain": creative["domain"], "country": creative.get("country"), "channel": creative["channel"], "source": "classify", "status": "truncated_input", "detail": "; ".join(limits) + "; full source content retained in evidence", "tier": "C"})
|
|
1049
|
+
key = _label_key(item, images.get(cid), taxonomy_sha256, product_context, client.semantic_context, source_sha)
|
|
1050
|
+
cached = client._cache_read(client._shared() / f"{key}.json")
|
|
1051
|
+
if cached is not None and isinstance(cached.get("response"), dict):
|
|
1052
|
+
client.cache_hits += 1
|
|
1053
|
+
# The answer came from one batch call: the ledger counts that call
|
|
1054
|
+
# (and its tokens) on every replay, so receipts stay byte-stable.
|
|
1055
|
+
if isinstance(cached.get("batch"), str):
|
|
1056
|
+
client.touched.add(cached["batch"])
|
|
1057
|
+
if cached["response"].get("unavailable"):
|
|
1058
|
+
coverage.append({"type": "coverage", "creative_id": cid, "domain": creative["domain"], "channel": creative["channel"], "country": creative.get("country"), "source": "classify", "status": "rejected", "detail": cached["response"]["unavailable"], "tier": "C"})
|
|
1059
|
+
continue
|
|
1060
|
+
labels[cid] = (cached["response"], cached.get("meta") or {})
|
|
1061
|
+
elif cid in allowed_new and not any(cid == c["creative_id"] for c, _ in pending):
|
|
1062
|
+
pending.append((creative, key))
|
|
1063
|
+
|
|
1064
|
+
text_only = [(c, k) for c, k in pending if c["creative_id"] not in images]
|
|
1065
|
+
with_frames = [(c, k) for c, k in pending if c["creative_id"] in images]
|
|
1066
|
+
def content_batches(entries, maximum):
|
|
1067
|
+
batch, text_chars = [], 0
|
|
1068
|
+
# Rich semantics and complete OCR require more response room than the
|
|
1069
|
+
# legacy labels. Long speech/text ads are classified individually.
|
|
1070
|
+
maximum = min(maximum, 4) if client.semantic_context is not None else maximum
|
|
1071
|
+
for creative, key in entries:
|
|
1072
|
+
item = _classify_input(creative, None)
|
|
1073
|
+
cost = sum(len(item.get(name) or "") for name in ("headline", "body", "transcript")) + sum(len(t) for t in item.get("extra_texts") or [])
|
|
1074
|
+
if batch and (len(batch) >= maximum or text_chars + cost > 12000):
|
|
1075
|
+
yield batch
|
|
1076
|
+
batch, text_chars = [], 0
|
|
1077
|
+
batch.append((creative, key))
|
|
1078
|
+
text_chars += cost
|
|
1079
|
+
if batch:
|
|
1080
|
+
yield batch
|
|
1081
|
+
batches = list(content_batches(text_only, TEXT_BATCH)) + list(content_batches(with_frames, IMAGE_BATCH))
|
|
1082
|
+
failures = 0
|
|
1083
|
+
for batch_position, batch in enumerate(batches):
|
|
1084
|
+
frames = {frame_ref(c): images[c["creative_id"]] for c, _ in batch if c["creative_id"] in images}
|
|
1085
|
+
request = {"schema_version": 1, "taxonomy_version": taxonomy["taxonomy_version"], "product_context": product_context or None,
|
|
1086
|
+
"creatives": [_classify_input(c, frame_ref(c) if c["creative_id"] in images else None) for c, _ in batch]}
|
|
1087
|
+
if client.semantic_context is not None:
|
|
1088
|
+
# Existing paid labels anchor canonical meanings for later batches.
|
|
1089
|
+
# The stable per-ad key binds the offer/language policy; each batch
|
|
1090
|
+
# receipt also binds the actual (possibly expanded) anchor context.
|
|
1091
|
+
prior_rows = [{**item, "ad_ref": cid} for cid, (item, _) in labels.items()]
|
|
1092
|
+
prior_by_id = {row["creative_id"]: row for row in [*creatives, *cached_only]}
|
|
1093
|
+
prior_labels = [copy.deepcopy({**row, "creative_id": row["ad_ref"], "with_image": row["ad_ref"] in images}) for row in prior_rows if row.get("ad_ref") in prior_by_id]
|
|
1094
|
+
prior_creatives = [copy.deepcopy(prior_by_id[row["creative_id"]]) for row in prior_labels]
|
|
1095
|
+
dictionary = SEMANTICS.apply_semantics(prior_creatives, prior_labels, client.semantic_context)
|
|
1096
|
+
original_counts = {field: len(dictionary[field]) for field in ("angles", "message_families")}
|
|
1097
|
+
for field in original_counts:
|
|
1098
|
+
dictionary[field] = sorted(dictionary[field], key=lambda row: (-len(row.get("members") or []), row["key"]))[:320]
|
|
1099
|
+
# Full observations remain in the final dictionary. Request anchors
|
|
1100
|
+
# obey the bounded public inference context, with visible coverage.
|
|
1101
|
+
while len(json.dumps(dictionary, ensure_ascii=False)) > 220000 and any(dictionary[field] for field in original_counts):
|
|
1102
|
+
field = max(original_counts, key=lambda name: len(json.dumps(dictionary[name], ensure_ascii=False)))
|
|
1103
|
+
dictionary[field].pop()
|
|
1104
|
+
if any(len(dictionary[field]) < original_counts[field] for field in original_counts):
|
|
1105
|
+
coverage.append({"type": "coverage", "source": "semantic_context", "status": "bounded_anchors", "detail": f"Request canonical anchors bounded from {original_counts}; full observations remain in output dictionary", "tier": "C"})
|
|
1106
|
+
request["semantic_context"] = SEMANTICS.build_semantic_context(product_context, client.semantic_context["allowed_languages"], dictionary)
|
|
1107
|
+
try:
|
|
1108
|
+
response, retrieved_on, batch_digest = client.classify(request, frames, taxonomy_sha256)
|
|
1109
|
+
except BudgetExhausted as exc:
|
|
1110
|
+
if "cumulative" not in str(exc):
|
|
1111
|
+
raise
|
|
1112
|
+
for creative, _ in [entry for remaining in batches[batch_position:] for entry in remaining]:
|
|
1113
|
+
coverage.append({"type": "coverage", "creative_id": creative["creative_id"], "domain": creative["domain"], "channel": creative["channel"],
|
|
1114
|
+
"country": creative.get("country"), "source": "classify", "status": "budget_deferred",
|
|
1115
|
+
"detail": str(exc), "tier": "C"})
|
|
1116
|
+
break
|
|
1117
|
+
except CreativePatternsError as exc:
|
|
1118
|
+
if not isinstance(exc, SourceUnavailable) and re.search(r"HTTP (?:404|405|409|503)\b", str(exc)):
|
|
1119
|
+
# The endpoint is not deployed, not configured or on another taxonomy: no batch can succeed.
|
|
1120
|
+
raise CreativePatternsError(f"creative classification is unavailable: {CF._provider_detail(exc)}") from exc
|
|
1121
|
+
if isinstance(exc, CF.OsUnreachable) or (not isinstance(exc, SourceUnavailable) and STOP_CLASSIFYING_RE.search(str(exc))):
|
|
1122
|
+
# Not about this batch: the OS is unreachable, the key is
|
|
1123
|
+
# refused, rate-limited or out of LLM budget, so later batches
|
|
1124
|
+
# would fail the same way. Recording gaps would complete the
|
|
1125
|
+
# section with creatives unlabelled; failing the attempt lets
|
|
1126
|
+
# a rerun continue from the cached answers.
|
|
1127
|
+
raise type(exc)(f"creative classification stopped: {CF._provider_detail(exc)}; rerun later to continue "
|
|
1128
|
+
f"({len(labels)} label(s) so far are cached)") from exc
|
|
1129
|
+
failures += 1
|
|
1130
|
+
if isinstance(exc, SourceUnavailable) and getattr(exc, "batch_digest", None):
|
|
1131
|
+
for creative, key in batch:
|
|
1132
|
+
client._cache_write(client._shared() / f"{key}.json", {"kind": "creative_label", "response": {"unavailable": str(exc)}, "batch": exc.batch_digest})
|
|
1133
|
+
CF._coverage_gap(coverage, "classify", sorted({c["domain"] for c, _ in batch}), f"a classification batch failed: {CF._provider_detail(exc)}")
|
|
1134
|
+
continue
|
|
1135
|
+
meta = {"taxonomy_version": response.get("taxonomy_version"), "taxonomy_sha256": response.get("taxonomy_sha256") or taxonomy_sha256,
|
|
1136
|
+
"model_role": response.get("model_role"), "effective_model": response.get("effective_model"), "classified_on": retrieved_on,
|
|
1137
|
+
"classification_request_context_sha256": (request.get("semantic_context") or {}).get("context_sha256")}
|
|
1138
|
+
by_ref = {c["creative_id"]: (c, k) for c, k in batch}
|
|
1139
|
+
for item in response.get("classifications") or []:
|
|
1140
|
+
if not isinstance(item, dict) or item.get("ad_ref") not in by_ref:
|
|
1141
|
+
continue
|
|
1142
|
+
creative, key = by_ref[item["ad_ref"]]
|
|
1143
|
+
labels[creative["creative_id"]] = (item, meta)
|
|
1144
|
+
client._cache_write(client._shared() / f"{key}.json", {"kind": "creative_label", "response": item, "meta": meta,
|
|
1145
|
+
"batch": batch_digest, "retrieved_on": retrieved_on})
|
|
1146
|
+
if batches and failures == len(batches) and not labels:
|
|
1147
|
+
raise CreativePatternsError("every creative classification batch failed; rerun later (no spend is lost: answers are cached)")
|
|
1148
|
+
if deferred and not labels:
|
|
1149
|
+
raise CreativePatternsError("transient image failures left no usable creative classifications; rerun later to continue from the cache")
|
|
1150
|
+
|
|
1151
|
+
rows: list[dict[str, Any]] = []
|
|
1152
|
+
for creative in [*creatives, *cached_only]:
|
|
1153
|
+
if creative["creative_id"] not in labels or any(r["creative_id"] == creative["creative_id"] for r in rows):
|
|
1154
|
+
continue
|
|
1155
|
+
item, meta = labels[creative["creative_id"]]
|
|
1156
|
+
rows.append({
|
|
1157
|
+
"type": "creative_classification",
|
|
1158
|
+
"creative_id": creative["creative_id"],
|
|
1159
|
+
"domain": creative["domain"],
|
|
1160
|
+
"platform": creative["platform"],
|
|
1161
|
+
"channel": creative["channel"],
|
|
1162
|
+
"fingerprint": creative["fingerprint"],
|
|
1163
|
+
"semantics": item.get("semantics") or {},
|
|
1164
|
+
"classification_request_context_sha256": meta.get("classification_request_context_sha256"),
|
|
1165
|
+
"classification_source_sha256": creative.get("classification_source_sha256"),
|
|
1166
|
+
"classification_input_limits": creative.get("classification_input_limits") or [],
|
|
1167
|
+
"observed_text": item.get("observed_text"),
|
|
1168
|
+
"observed_text_source": item.get("observed_text_source"),
|
|
1169
|
+
"observed_text_sources": item.get("observed_text_sources") or [],
|
|
1170
|
+
"texts_sources": item.get("texts_sources") or {},
|
|
1171
|
+
"observed_text_frame_refs": item.get("observed_text_frame_refs") or [],
|
|
1172
|
+
"country": creative.get("country"),
|
|
1173
|
+
"creative_type": creative.get("creative_type"),
|
|
1174
|
+
"format_kind": creative.get("format_kind"),
|
|
1175
|
+
"labels": item.get("labels") or {},
|
|
1176
|
+
"texts": item.get("texts") or {},
|
|
1177
|
+
"offer": item.get("offer") or {},
|
|
1178
|
+
"confidence": item.get("confidence"),
|
|
1179
|
+
"coerced": item.get("coerced") or [],
|
|
1180
|
+
"with_image": creative["creative_id"] in images,
|
|
1181
|
+
"with_video_frames": False,
|
|
1182
|
+
"with_analyzed_video_frames": bool(creative.get("video_frames")),
|
|
1183
|
+
"video_analysis_ref": (creative.get("video_analysis_receipt") or {}).get("input_sha256"),
|
|
1184
|
+
"input_text_sources": _classify_input(creative, None).get("extra_text_sources") or [],
|
|
1185
|
+
"taxonomy_version": meta.get("taxonomy_version"),
|
|
1186
|
+
"taxonomy_sha256": meta.get("taxonomy_sha256") or taxonomy_sha256,
|
|
1187
|
+
"model_role": meta.get("model_role"),
|
|
1188
|
+
"effective_model": meta.get("effective_model"),
|
|
1189
|
+
"classified_on": meta.get("classified_on"),
|
|
1190
|
+
"market_scope": creative["market_scope"],
|
|
1191
|
+
"tier": "C",
|
|
1192
|
+
})
|
|
1193
|
+
return rows
|
|
1194
|
+
|
|
1195
|
+
|
|
1196
|
+
# ── patterns ────────────────────────────────────────────────────────────────
|
|
1197
|
+
|
|
1198
|
+
def score_patterns(creatives: list[dict[str, Any]], classifications: list[dict[str, Any]], taxonomy: dict[str, Any], long_running_days: int, dictionary: dict | None = None) -> list[dict[str, Any]]:
|
|
1199
|
+
return MODEL.patterns(creatives, classifications, taxonomy, long_running_days, dictionary)
|
|
1200
|
+
|
|
1201
|
+
|
|
1202
|
+
def price_anchors(creatives: list[dict[str, Any]], classifications: list[dict] | None = None) -> list[dict[str, Any]]:
|
|
1203
|
+
return MODEL.promises(creatives, classifications)
|
|
1204
|
+
|
|
1205
|
+
|
|
1206
|
+
def _cell(value: Any, cap: int = 110) -> str:
|
|
1207
|
+
return CF._cell(value, cap)
|
|
1208
|
+
|
|
1209
|
+
|
|
1210
|
+
def _label(value: Any) -> str:
|
|
1211
|
+
return str(value).replace("_", " ") if value not in (None, "") else "—"
|
|
1212
|
+
|
|
1213
|
+
|
|
1214
|
+
def render_section(*, domains: list[str], creatives: list[dict[str, Any]], classifications: list[dict[str, Any]],
|
|
1215
|
+
patterns: list[dict[str, Any]], anchors: list[dict[str, Any]], coverage: list[dict[str, Any]],
|
|
1216
|
+
totals: dict[str, Any], as_of: str, retrieved_on: str, long_running_days: int, taxonomy_version: str, selection: dict | None = None) -> str:
|
|
1217
|
+
body = MODEL.section(creatives, patterns, anchors, coverage, long_running_days, selection)
|
|
1218
|
+
spend = (f"\nClassified {len(classifications)} of {len(creatives)} individual ads; "
|
|
1219
|
+
f"{totals['classify_calls']} classification calls and {totals['classify_prompt_tokens']} prompt tokens; "
|
|
1220
|
+
f"{totals.get('media_calls', 0)} media calls and {totals.get('media_prompt_tokens', 0)} media prompt tokens. "
|
|
1221
|
+
f"Cumulative paid starts: {totals.get('cumulative_paid_starts', 0)}/{totals.get('cumulative_paid_start_limit')}; "
|
|
1222
|
+
f"these are resource counts, not a currency estimate. "
|
|
1223
|
+
f"Cutoff {as_of}; sources retrieved {retrieved_on}; taxonomy {taxonomy_version}.\n")
|
|
1224
|
+
return body + spend
|
|
1225
|
+
|
|
1226
|
+
|
|
1227
|
+
# ── orchestration ───────────────────────────────────────────────────────────
|
|
1228
|
+
|
|
1229
|
+
def frozen_semantic_context(client: OsClient, inputs: RunInputs) -> dict:
|
|
1230
|
+
# Freeze anchors across compatible cutoffs. A dictionary learned during a
|
|
1231
|
+
# run must not change the key of already-paid labels on the next refresh.
|
|
1232
|
+
base = SEMANTICS.build_semantic_context(inputs.product_context, inputs.languages)
|
|
1233
|
+
base_sha = POLICY.canonical_hash({"context": base, "market_languages": inputs.market_languages})
|
|
1234
|
+
shared = client._shared() / "semantic-context.json"
|
|
1235
|
+
snapshot = client.cache_dir / "semantic-context.json"
|
|
1236
|
+
for path in (snapshot, shared):
|
|
1237
|
+
previous = client._cache_read(path)
|
|
1238
|
+
if previous and previous.get("base_sha256") == base_sha:
|
|
1239
|
+
_write_atomic(snapshot, json.dumps(previous, ensure_ascii=False))
|
|
1240
|
+
return previous["context"]
|
|
1241
|
+
record = {"base_sha256": base_sha, "context": base}
|
|
1242
|
+
_write_atomic(shared, json.dumps(record, ensure_ascii=False))
|
|
1243
|
+
_write_atomic(snapshot, json.dumps(record, ensure_ascii=False))
|
|
1244
|
+
return base
|
|
1245
|
+
|
|
1246
|
+
|
|
1247
|
+
def language_inputs(workspace: Path, args: argparse.Namespace, countries: list[str] | None = None) -> tuple[list[str], dict]:
|
|
1248
|
+
intake = CF.typed_intake(workspace)
|
|
1249
|
+
raw = str(_arg(args, "languages", "") or "")
|
|
1250
|
+
market = str(_arg(args, "market_languages", "") or "")
|
|
1251
|
+
languages = ([code.strip().lower() for code in raw.split(",") if code.strip()]
|
|
1252
|
+
if raw else list(intake.get("languages") or []))
|
|
1253
|
+
by_market = {} if raw or market else dict(intake.get("market_languages") or {})
|
|
1254
|
+
for entry in market.split(";"):
|
|
1255
|
+
if not entry.strip():
|
|
1256
|
+
continue
|
|
1257
|
+
if ":" not in entry:
|
|
1258
|
+
raise CreativePatternsError("market languages must use COUNTRY:language,language entries")
|
|
1259
|
+
country, codes = entry.split(":", 1)
|
|
1260
|
+
country = country.strip().upper().replace("UK", "GB")
|
|
1261
|
+
if not re.fullmatch(r"[A-Z]{2}", country):
|
|
1262
|
+
raise CreativePatternsError("market language country must be an ISO code")
|
|
1263
|
+
by_market[country] = [code.strip().lower() for code in codes.split(",") if code.strip()]
|
|
1264
|
+
if not by_market[country]:
|
|
1265
|
+
raise CreativePatternsError("market languages cannot be empty")
|
|
1266
|
+
if by_market:
|
|
1267
|
+
for country in countries or []:
|
|
1268
|
+
if country not in by_market:
|
|
1269
|
+
if not languages:
|
|
1270
|
+
raise CreativePatternsError(f"accepted languages are missing for market {country}; set a global list or that market explicitly")
|
|
1271
|
+
by_market[country] = list(languages)
|
|
1272
|
+
languages = sorted(set(languages + [c for values in by_market.values() for c in values]))
|
|
1273
|
+
SEMANTICS.build_semantic_context("", languages)
|
|
1274
|
+
return languages, by_market
|
|
1275
|
+
|
|
1276
|
+
|
|
1277
|
+
def full_product_context(workspace: Path) -> str:
|
|
1278
|
+
text = CF.IO.context_text(workspace, SECTION_ID)
|
|
1279
|
+
intake = CF.typed_intake(workspace)
|
|
1280
|
+
offer = {key: value for key, value in (intake.get("offer") or {}).items()
|
|
1281
|
+
if value not in (None, "", [])}
|
|
1282
|
+
if offer:
|
|
1283
|
+
text += "\n\nOffer card:\n" + json.dumps(offer, ensure_ascii=False)
|
|
1284
|
+
if len(text) > PRODUCT_CONTEXT_CHARS:
|
|
1285
|
+
raise CreativePatternsError("offer context exceeds 24000 characters; provide a bounded offer card rather than silently truncating it")
|
|
1286
|
+
return text
|
|
1287
|
+
|
|
1288
|
+
|
|
1289
|
+
def hydrate_footprint_media(creatives: list[dict], inputs: RunInputs, taxonomy: dict) -> None:
|
|
1290
|
+
if inputs.runtime is None:
|
|
1291
|
+
return
|
|
1292
|
+
by_native = {}
|
|
1293
|
+
for c in creatives:
|
|
1294
|
+
by_native.setdefault(c["native_ad_id"], []).append(c)
|
|
1295
|
+
for path in CF.IO.cache_entries(inputs.runtime.parent / "channel_footprint/cache"):
|
|
1296
|
+
try:
|
|
1297
|
+
entry = CF.IO.read_cache_entry(path)
|
|
1298
|
+
except (ValueError, OSError, EOFError):
|
|
1299
|
+
continue
|
|
1300
|
+
if not isinstance(entry, dict):
|
|
1301
|
+
continue
|
|
1302
|
+
captured = entry.get("cutoff") or entry.get("retrieved_on")
|
|
1303
|
+
if isinstance(captured, str) and captured[:10] > inputs.as_of.isoformat():
|
|
1304
|
+
continue # Future media must not enrich a historical cutoff.
|
|
1305
|
+
response = entry.get("response") or {}
|
|
1306
|
+
rows = response.get("rows") or response.get("data") or []
|
|
1307
|
+
if not isinstance(rows, list):
|
|
1308
|
+
continue
|
|
1309
|
+
for raw in rows:
|
|
1310
|
+
if not isinstance(raw, dict):
|
|
1311
|
+
continue
|
|
1312
|
+
native = str(raw.get("ad_archive_id") or raw.get("creativeId") or raw.get("adId") or raw.get("id") or "")
|
|
1313
|
+
for c in by_native.get(native, []):
|
|
1314
|
+
if c["platform"] == "meta" and raw.get("snapshot"):
|
|
1315
|
+
recovered = _meta_actor_creative(raw, c["domain"], CF.brand_token(c["domain"]), inputs.as_of,
|
|
1316
|
+
entry.get("retrieved_on") or inputs.as_of.isoformat(), taxonomy)
|
|
1317
|
+
if recovered:
|
|
1318
|
+
for field in ("headline", "body", "extra_texts", "image_url", "video_url", "media_urls", "media_cards", "format_raw", "format_kind"):
|
|
1319
|
+
if recovered.get(field):
|
|
1320
|
+
c[field] = recovered[field]
|
|
1321
|
+
c["creative_type"] = MODEL.creative_type(c)
|
|
1322
|
+
elif c["platform"] in ("google", "youtube"):
|
|
1323
|
+
for field, native_field in (("image_url", "imageUrl"), ("video_url", "videoUrl"), ("link", "adUrl")):
|
|
1324
|
+
if raw.get(native_field):
|
|
1325
|
+
c[field] = raw[native_field]
|
|
1326
|
+
|
|
1327
|
+
|
|
1328
|
+
def footprint_coverage(inputs: RunInputs, creatives: list[dict]) -> list[dict]:
|
|
1329
|
+
document = inputs.footprint or {}
|
|
1330
|
+
countries = document.get("inputs", {}).get("target_countries") or [None]
|
|
1331
|
+
rows = []
|
|
1332
|
+
for rank in document.get("rows", []):
|
|
1333
|
+
if rank.get("type") != "channel_observation":
|
|
1334
|
+
continue
|
|
1335
|
+
channel = rank.get("channel") or rank.get("platform")
|
|
1336
|
+
if not channel:
|
|
1337
|
+
continue
|
|
1338
|
+
for country in [rank["country"]] if rank.get("country") else countries:
|
|
1339
|
+
if inputs.deepened_cells is not None and (country, channel) not in inputs.deepened_cells:
|
|
1340
|
+
rows.extend({"type": "coverage", "domain": domain, "country": country, "channel": channel, "source": "channel_footprint",
|
|
1341
|
+
"status": "not_deepened", "detail": f"Bounded scope inspects the first {inputs.max_channels} observed channel(s) in stable name order "
|
|
1342
|
+
"per market; use --max-channels 0 for every cell.", "rank": rank.get("rank"), "tier": "C"}
|
|
1343
|
+
for domain in inputs.domains)
|
|
1344
|
+
continue
|
|
1345
|
+
for domain in inputs.domains:
|
|
1346
|
+
items = [c for c in creatives if c["channel"] == channel and c["domain"] == domain and c.get("country") == country]
|
|
1347
|
+
gaps = [g for g in document.get("rows", []) if g.get("type") == "coverage" and g.get("domain") == domain
|
|
1348
|
+
and (g.get("channel") or g.get("platform")) == channel and g.get("country") in (country, None)]
|
|
1349
|
+
rows.append({"type": "coverage", "domain": domain, "country": country, "channel": channel,
|
|
1350
|
+
"source": "channel_footprint", "status": "extracted" if items else "not_extracted",
|
|
1351
|
+
"detail": f"{len(items)} native ads in available sample; not a census" if items else
|
|
1352
|
+
"; ".join(str(g.get("detail") or g.get("reason") or g.get("status")) for g in gaps) or "No extractable ads in completed Footprint; channel presence retained",
|
|
1353
|
+
"rank": rank.get("rank"), "tier": "C"})
|
|
1354
|
+
return rows
|
|
1355
|
+
|
|
1356
|
+
|
|
1357
|
+
def prepare_media(client: OsClient, creatives: list[dict], inputs: RunInputs, config: dict, coverage: list[dict], *, analyze_videos: bool = True) -> None:
|
|
1358
|
+
selected_videos: dict[tuple, dict[str, str]] = {}
|
|
1359
|
+
for c in sorted(creatives, key=MODEL.strength_key):
|
|
1360
|
+
c["allowed_languages"] = inputs.market_languages.get(c.get("country"), inputs.languages)
|
|
1361
|
+
urls = [c.get("image_url")] + list(c.get("media_urls") or [])
|
|
1362
|
+
for url in dict.fromkeys(urls):
|
|
1363
|
+
if not isinstance(url, str) or not url.startswith("https://"):
|
|
1364
|
+
continue
|
|
1365
|
+
image = client.fetch_image(url)
|
|
1366
|
+
if image and image.get("file") and inputs.runtime:
|
|
1367
|
+
workspace = VIDEO._workspace(inputs.runtime)
|
|
1368
|
+
path = client._shared() / "images" / image["file"]
|
|
1369
|
+
if path.is_relative_to(workspace):
|
|
1370
|
+
asset = {"path": path.relative_to(workspace).as_posix(), "kind": "preview" if c["creative_type"] == "video" else "image", "sha256": image["sha256"]}
|
|
1371
|
+
if asset not in c.setdefault("media_assets", []):
|
|
1372
|
+
c["media_assets"].append(asset)
|
|
1373
|
+
if c["creative_type"] != "video" or inputs.runtime is None or not analyze_videos:
|
|
1374
|
+
continue
|
|
1375
|
+
key = (c["domain"], c["channel"], c.get("country"))
|
|
1376
|
+
# A market observation or second native ad is not a new video. Prefer
|
|
1377
|
+
# actual bytes, then exact provider video ID, before native-ad fallback.
|
|
1378
|
+
content_id = c.get("video_sha256") or c.get("youtube_video_id") or VIDEO.youtube_id(c.get("video_id")) \
|
|
1379
|
+
or VIDEO.youtube_id(c.get("video_url")) or VIDEO.youtube_id(c.get("link")) \
|
|
1380
|
+
or c.get("video_url") or c.get("native_ad_id") or c["creative_id"]
|
|
1381
|
+
content_id = str(content_id)
|
|
1382
|
+
selected = selected_videos.setdefault(key, {})
|
|
1383
|
+
excluded = c.get("relevance") == "irrelevant" or c.get("language_status") == "excluded"
|
|
1384
|
+
if excluded or (content_id not in selected and len(set(selected.values())) >= inputs.top_videos):
|
|
1385
|
+
c["video_analysis_status"] = "not_selected"
|
|
1386
|
+
c.setdefault("video_gaps", []).append("Irrelevant or excluded-language video" if excluded else "Beyond configured top-N unique videos")
|
|
1387
|
+
else:
|
|
1388
|
+
selected.setdefault(content_id, content_id)
|
|
1389
|
+
try:
|
|
1390
|
+
c.update(VIDEO.preserve_video_media(c, inputs.runtime, config, client=client))
|
|
1391
|
+
c.update(VIDEO.analyze_video(client, c, inputs.runtime, config))
|
|
1392
|
+
# Resolved IDs/hashes let subsequent ads reuse this video's
|
|
1393
|
+
# shared paid receipts, including after the cell reached N.
|
|
1394
|
+
if c.get("video_sha256") or c.get("youtube_video_id"):
|
|
1395
|
+
resolved_id = str(c.get("video_sha256") or c["youtube_video_id"])
|
|
1396
|
+
previous_id = selected[content_id]
|
|
1397
|
+
for alias in list(selected):
|
|
1398
|
+
if selected[alias] == previous_id:
|
|
1399
|
+
selected[alias] = resolved_id
|
|
1400
|
+
selected[resolved_id] = resolved_id
|
|
1401
|
+
except BudgetExhausted as exc:
|
|
1402
|
+
c["video_analysis_status"] = "budget_deferred"
|
|
1403
|
+
c.setdefault("video_gaps", []).append(str(exc))
|
|
1404
|
+
if c.get("video_gaps"):
|
|
1405
|
+
coverage.append({"type": "coverage", "domain": c["domain"], "channel": c["channel"], "country": c.get("country"),
|
|
1406
|
+
"source": "video_analysis", "status": c.get("video_analysis_status") or "unavailable", "detail": "; ".join(c["video_gaps"]), "tier": "C"})
|
|
1407
|
+
|
|
1408
|
+
|
|
1409
|
+
def footprint_input(workspace: Path, policy: dict[str, Any], policy_sha256: str) -> dict[str, Any] | None:
|
|
1410
|
+
"""A v2 footprint is an owned input, never an unverified optional hint."""
|
|
1411
|
+
path = workspace / CHANNEL_FOOTPRINT_EVIDENCE_REL
|
|
1412
|
+
if not path.is_file():
|
|
1413
|
+
return None
|
|
1414
|
+
document = json.loads(path.read_text(encoding="utf-8"))
|
|
1415
|
+
if document.get("schema_version") != 2:
|
|
1416
|
+
raise CreativePatternsError("Channel Footprint must be refreshed to its scoped v2 contract before Creative Patterns")
|
|
1417
|
+
reader = _load_script("research-evidence.py", "pmr_cf_inputs_for_creative_patterns")
|
|
1418
|
+
result = reader.inspect_section(workspace, "channel_footprint")
|
|
1419
|
+
if result.get("status") != "completed":
|
|
1420
|
+
raise CreativePatternsError("Channel Footprint input is pending, stale or unverified: " + str(result.get("reason") or result.get("status")))
|
|
1421
|
+
inputs = document.get("inputs") or {}
|
|
1422
|
+
if inputs.get("effective_policy") != policy or inputs.get("policy_sha256") != policy_sha256:
|
|
1423
|
+
raise CreativePatternsError("Channel Footprint evidence policy changed; finalize it with the current shared taxonomy before Creative Patterns")
|
|
1424
|
+
return document
|
|
1425
|
+
|
|
1426
|
+
|
|
1427
|
+
def footprint_creatives(document: dict[str, Any], as_of: date, taxonomy: dict[str, Any]) -> list[dict[str, Any]]:
|
|
1428
|
+
"""Reuse decided ads with per-market lineage, without re-admitting unrelated ads."""
|
|
1429
|
+
context = (document.get("inputs") or {}).get("competitor_context") or {}
|
|
1430
|
+
policy, _ = POLICY.policy_from_taxonomy(taxonomy)
|
|
1431
|
+
selected_countries = set((document.get("inputs") or {}).get("target_countries") or [])
|
|
1432
|
+
result = []
|
|
1433
|
+
platforms = {"google_ads": "google", "youtube_ads": "youtube", "meta_ads": "meta", "linkedin_ads": "linkedin", "reddit_ads": "reddit", "tiktok_ads": "tiktok", "display_native": "google"}
|
|
1434
|
+
for ad in document.get("rows") or []:
|
|
1435
|
+
if ad.get("type") != "channel_ad":
|
|
1436
|
+
continue
|
|
1437
|
+
base_channel = ad.get("channel") or ad.get("platform")
|
|
1438
|
+
if base_channel not in platforms:
|
|
1439
|
+
continue
|
|
1440
|
+
domain_context = context.get(ad.get("domain"), {}) if isinstance(context, dict) else {}
|
|
1441
|
+
role = ad.get("competitor_tier") or (domain_context.get("competitor_tier") if isinstance(domain_context, dict) else None) or "unknown"
|
|
1442
|
+
for observation in ad.get("observations") or []:
|
|
1443
|
+
channel = base_channel
|
|
1444
|
+
if channel == "google_ads":
|
|
1445
|
+
placement = str(observation.get("placement") or "").lower()
|
|
1446
|
+
if placement in {"youtube", "youtube_ads"}:
|
|
1447
|
+
channel = "youtube_ads"
|
|
1448
|
+
elif placement in {"display", "display_ads", "display_native"}:
|
|
1449
|
+
channel = "display_native"
|
|
1450
|
+
platform = platforms.get(channel, str(channel).removesuffix("_ads"))
|
|
1451
|
+
country = observation.get("country")
|
|
1452
|
+
if country and country not in selected_countries:
|
|
1453
|
+
continue
|
|
1454
|
+
native_date_scope = observation.get("date_scope") or "unknown"
|
|
1455
|
+
date_country = observation.get("date_country")
|
|
1456
|
+
date_scope = native_date_scope
|
|
1457
|
+
if date_scope == "country" and (not country or date_country != country):
|
|
1458
|
+
date_scope = "unknown"
|
|
1459
|
+
first = observation.get("first_shown")
|
|
1460
|
+
last = observation.get("last_shown")
|
|
1461
|
+
activity = observation.get("activity_status", observation.get("status"))
|
|
1462
|
+
activity_scope = observation.get("activity_scope") or "unknown"
|
|
1463
|
+
activity_country = observation.get("activity_country")
|
|
1464
|
+
# Activity observed in this country's query (global provider flag) or
|
|
1465
|
+
# native country activity both establish current use in the market.
|
|
1466
|
+
same_activity_country = bool(country and activity_scope in ("country", "global") and activity_country == country)
|
|
1467
|
+
capture = observation.get("observed_on") or observation.get("last_observed") or document.get("retrieved_on") or as_of.isoformat()
|
|
1468
|
+
capture = str(capture).split("..")[-1][:10]
|
|
1469
|
+
running = (True if activity == "active" else
|
|
1470
|
+
False if activity in ("stopped", "inactive", "ended", "archived") else None) if same_activity_country else None
|
|
1471
|
+
same_delivery_scope = ((date_scope == "global" and activity_scope == "global")
|
|
1472
|
+
or (date_scope == "country" and activity_scope == "country" and same_activity_country))
|
|
1473
|
+
counted_last = last
|
|
1474
|
+
if activity == "active" and same_activity_country and same_delivery_scope and first and (not last or str(last)[:10] > capture):
|
|
1475
|
+
# Counted through the footprint capture, as Channel Footprint
|
|
1476
|
+
# does: a later section cannot extend delivery because time
|
|
1477
|
+
# elapsed, and a scheduled future end adds no days.
|
|
1478
|
+
counted_last = capture
|
|
1479
|
+
last = last or capture
|
|
1480
|
+
# Listing observations deliberately have no delivery age or flag.
|
|
1481
|
+
served = _days(first, counted_last) if first and counted_last and date_scope in ("country", "global") else None
|
|
1482
|
+
if date_scope in ("country", "global") and counted_last and str(counted_last)[:10] > capture:
|
|
1483
|
+
served = None # Future delivery is not historical proof at source capture.
|
|
1484
|
+
native_id = str(ad.get("native_ad_id") or ad.get("ad_id"))
|
|
1485
|
+
scope_key = country or "global"
|
|
1486
|
+
identity = f"{native_id}:{channel}:{scope_key}:{date_scope}"
|
|
1487
|
+
creative = _creative(
|
|
1488
|
+
domain=ad["domain"], platform=platform, source=str(observation.get("source") or ad.get("source") or "channel_footprint"),
|
|
1489
|
+
ad_id=identity, advertiser=ad.get("advertiser"), headline=ad.get("headline"), body=ad.get("ad_text"),
|
|
1490
|
+
landing_url=ad.get("landing_url"), format_raw=str(ad.get("format") or "unknown").upper(), image_url=ad.get("image_url"),
|
|
1491
|
+
first_seen=first, last_seen=last, served_days=served, running=running,
|
|
1492
|
+
age_basis="platform_dates" if served is not None else "none", market_scope=observation.get("market_scope") or "global",
|
|
1493
|
+
verified_by=ad.get("verified_by") or "channel_footprint_identity", link=ad.get("source_url"), taxonomy=_google_taxonomy(taxonomy, platform),
|
|
1494
|
+
extra={"country": country, "countries": [country] if country else [], "competitor_tier": role,
|
|
1495
|
+
"relevance": ad.get("relevance") or "unknown", "relevance_reason": ad.get("relevance_reason"), "duration_scope": date_scope,
|
|
1496
|
+
"video_url": ad.get("video_url"), "media_urls": ad.get("media_urls") or [],
|
|
1497
|
+
"media_cards": ad.get("media_cards") or [], "format_kind": _alias(_google_taxonomy(taxonomy, platform), "format_kind", platform, str(ad.get("format") or "unknown").upper()),
|
|
1498
|
+
"footprint_ad_id": ad.get("ad_id"), "native_ad_id": native_id, "channel": channel,
|
|
1499
|
+
"first_observed": observation.get("first_observed"), "last_observed": observation.get("last_observed"),
|
|
1500
|
+
"retrieved_on": capture,
|
|
1501
|
+
"date_scope": native_date_scope, "date_country": date_country,
|
|
1502
|
+
"activity_scope": activity_scope, "activity_country": activity_country,
|
|
1503
|
+
"native_activity_status": activity, "placement": observation.get("placement")},
|
|
1504
|
+
)
|
|
1505
|
+
creative["source_members"] = [{k: creative.get(k) for k in (
|
|
1506
|
+
"creative_id", "domain", "country", "countries", "channel", "competitor_tier", "relevance", "duration_scope",
|
|
1507
|
+
"footprint_ad_id", "native_ad_id", "served_days", "running_at_capture", "first_seen", "last_seen",
|
|
1508
|
+
"date_scope", "date_country", "activity_scope", "activity_country", "native_activity_status", "placement") }]
|
|
1509
|
+
result.append(creative)
|
|
1510
|
+
return result
|
|
1511
|
+
|
|
1512
|
+
|
|
1513
|
+
@dataclass
|
|
1514
|
+
class RunInputs:
|
|
1515
|
+
domains: list[str]
|
|
1516
|
+
domain_sources: dict[str, str]
|
|
1517
|
+
product_domain: str
|
|
1518
|
+
sources: list[str]
|
|
1519
|
+
country: str
|
|
1520
|
+
eu_country: str
|
|
1521
|
+
as_of: date
|
|
1522
|
+
long_running_days: int
|
|
1523
|
+
product_context: str
|
|
1524
|
+
linkedin_slugs: dict[str, str] = field(default_factory=dict)
|
|
1525
|
+
runtime: Path | None = None
|
|
1526
|
+
footprint: dict[str, Any] | None = None
|
|
1527
|
+
effective_policy: dict[str, Any] = field(default_factory=dict)
|
|
1528
|
+
policy_sha256: str = ""
|
|
1529
|
+
restrict_sources: bool = False
|
|
1530
|
+
languages: list[str] = field(default_factory=list)
|
|
1531
|
+
market_languages: dict[str, list[str]] = field(default_factory=dict)
|
|
1532
|
+
top_videos: int = 5
|
|
1533
|
+
max_channels: int = 0 # express scope: deepen this many candidate channels per market; 0 keeps every cell
|
|
1534
|
+
deepened_cells: set[tuple[str | None, str]] | None = None
|
|
1535
|
+
|
|
1536
|
+
|
|
1537
|
+
def candidate_cells(footprint: dict[str, Any], max_channels: int) -> set[tuple[str | None, str]] | None:
|
|
1538
|
+
"""Bound an inspection sample by stable channel name, not business merit.
|
|
1539
|
+
None means every observed comparable cell is deepened."""
|
|
1540
|
+
if not max_channels:
|
|
1541
|
+
return None
|
|
1542
|
+
rows = footprint.get("rows") or []
|
|
1543
|
+
channel = lambda row: row.get("channel") or row.get("platform") # Footprint presence rows name it `platform`
|
|
1544
|
+
current = {(r.get("country"), channel(r)) for r in rows if r.get("type") == "library_presence"
|
|
1545
|
+
and r.get("relevance") == "relevant" and channel(r)}
|
|
1546
|
+
ranks = {(r.get("country"), channel(r)): r.get("rank", 999) for r in rows if r.get("type") == "channel_observation"}
|
|
1547
|
+
cells: set[tuple[str | None, str]] = set()
|
|
1548
|
+
for country in {country for country, _ in current}:
|
|
1549
|
+
channels = sorted((name for market, name in current if market == country), key=str)
|
|
1550
|
+
cells.update((country, channel) for channel in channels[:max_channels])
|
|
1551
|
+
return cells
|
|
1552
|
+
|
|
1553
|
+
|
|
1554
|
+
def run_pipeline(client: OsClient, inputs: RunInputs, config: dict[str, Any], taxonomy: dict[str, Any], taxonomy_sha256: str) -> dict[str, Any]:
|
|
1555
|
+
actors = config["actors"]
|
|
1556
|
+
coverage: list[dict[str, Any]] = []
|
|
1557
|
+
raw: list[dict[str, Any]] = []
|
|
1558
|
+
listings: dict[str, Any] = {}
|
|
1559
|
+
def partial_source(source, domains, fn):
|
|
1560
|
+
try:
|
|
1561
|
+
return CF._attempt(client, coverage, source, domains, fn)
|
|
1562
|
+
except BudgetExhausted as exc:
|
|
1563
|
+
if "cumulative" not in str(exc):
|
|
1564
|
+
raise
|
|
1565
|
+
for domain in domains:
|
|
1566
|
+
coverage.append({"type": "coverage", "domain": domain, "source": source, "status": "budget_deferred", "detail": str(exc), "tier": "C"})
|
|
1567
|
+
return None
|
|
1568
|
+
if inputs.footprint is not None:
|
|
1569
|
+
raw = [c for c in footprint_creatives(inputs.footprint, inputs.as_of, taxonomy) if c["domain"] in inputs.domains]
|
|
1570
|
+
inputs.deepened_cells = candidate_cells(inputs.footprint, inputs.max_channels)
|
|
1571
|
+
if inputs.deepened_cells is not None:
|
|
1572
|
+
raw = [c for c in raw if (c.get("country"), c["channel"]) in inputs.deepened_cells]
|
|
1573
|
+
if inputs.restrict_sources:
|
|
1574
|
+
families = {"google_ads": "google_transparency", "youtube_ads": "google_transparency", "display_native": "google_transparency",
|
|
1575
|
+
"meta_ads": "meta", "linkedin_ads": "linkedin", "reddit_ads": "reddit"}
|
|
1576
|
+
raw = [c for c in raw if families.get(c["channel"]) in inputs.sources]
|
|
1577
|
+
hydrate_footprint_media(raw, inputs, taxonomy)
|
|
1578
|
+
else:
|
|
1579
|
+
if "google_transparency" in inputs.sources:
|
|
1580
|
+
for domain in inputs.domains:
|
|
1581
|
+
found = partial_source("google_transparency", [domain], lambda d=domain: collect_google_transparency(client, actors["google_ads"], [d], inputs.as_of, taxonomy, coverage))
|
|
1582
|
+
raw.extend(found or [])
|
|
1583
|
+
for domain in inputs.domains:
|
|
1584
|
+
if "meta" in inputs.sources:
|
|
1585
|
+
found = partial_source("meta", [domain], lambda d=domain: collect_meta(client, actors["meta_ads"], d, inputs.as_of, inputs.eu_country, taxonomy))
|
|
1586
|
+
if found:
|
|
1587
|
+
meta_rows, us_rows = found
|
|
1588
|
+
raw.extend(meta_rows)
|
|
1589
|
+
listings[domain] = {"meta_us_rows": us_rows}
|
|
1590
|
+
if "google_search" in inputs.sources:
|
|
1591
|
+
found = partial_source("google_search", [domain], lambda d=domain: collect_google_search(client, d, inputs.country, inputs.as_of, taxonomy))
|
|
1592
|
+
raw.extend(found or [])
|
|
1593
|
+
if "linkedin" in inputs.sources:
|
|
1594
|
+
found = partial_source("linkedin", [domain], lambda d=domain: collect_linkedin(client, actors["linkedin_ads"], d, inputs.linkedin_slugs.get(d), taxonomy))
|
|
1595
|
+
raw.extend(found or [])
|
|
1596
|
+
if "reddit" in inputs.sources:
|
|
1597
|
+
found = partial_source("reddit", [domain], lambda d=domain: collect_reddit(client, actors["reddit_ads"], d, taxonomy))
|
|
1598
|
+
raw.extend(found or [])
|
|
1599
|
+
# Meta (and Reddit) media URLs are signed and expire within days:
|
|
1600
|
+
# download this competitor's images now, not after every collector.
|
|
1601
|
+
for creative in raw:
|
|
1602
|
+
if creative["domain"] != domain:
|
|
1603
|
+
continue
|
|
1604
|
+
if isinstance(creative.get("image_url"), str) and creative["image_url"].startswith("https://"):
|
|
1605
|
+
client.fetch_image(creative["image_url"])
|
|
1606
|
+
|
|
1607
|
+
coverage.extend(client.budget_gaps)
|
|
1608
|
+
|
|
1609
|
+
raw = cap_at_cutoff(raw, inputs.as_of)
|
|
1610
|
+
creatives = deduplicate(raw, inputs.long_running_days)
|
|
1611
|
+
# Native identity and scope were reconciled above; copy never defines identity.
|
|
1612
|
+
by_id: dict[str, dict[str, Any]] = {}
|
|
1613
|
+
for c in sorted(creatives, key=MODEL.strength_key):
|
|
1614
|
+
by_id.setdefault(c["creative_id"], c)
|
|
1615
|
+
creatives = list(by_id.values())
|
|
1616
|
+
if inputs.runtime is not None and inputs.footprint is None:
|
|
1617
|
+
# Earlier cutoffs only; the corpus of this cutoff is written below.
|
|
1618
|
+
creatives = merge_snapshots(creatives, load_snapshots(inputs.runtime, inputs.as_of), inputs.as_of, inputs.long_running_days,
|
|
1619
|
+
inputs.domains, listings)
|
|
1620
|
+
creatives.sort(key=lambda c: (c["domain"], c["platform"], c["creative_id"]))
|
|
1621
|
+
stored = creatives
|
|
1622
|
+
if inputs.footprint is not None:
|
|
1623
|
+
coverage.extend(footprint_coverage(inputs, stored))
|
|
1624
|
+
else:
|
|
1625
|
+
for domain in inputs.domains:
|
|
1626
|
+
for source in inputs.sources:
|
|
1627
|
+
if not any(c["domain"] == domain and c["source"] in SOURCE_ROW_SOURCES[source] for c in raw):
|
|
1628
|
+
coverage.append({"type": "coverage", "domain": domain, "source": source, "status": "no_data", "detail": "no verified creative", "tier": "C"})
|
|
1629
|
+
# Copy/preview classification establishes exclusions before large downloads.
|
|
1630
|
+
prepare_media(client, stored, inputs, config, coverage, analyze_videos=False)
|
|
1631
|
+
client.semantic_context = frozen_semantic_context(client, inputs)
|
|
1632
|
+
selected = select_for_classification([c for c in stored if not c.get("vanished_after")])
|
|
1633
|
+
stopped_rows = [c for c in stored if c.get("vanished_after")]
|
|
1634
|
+
classifications = classify_corpus(client, selected, taxonomy, taxonomy_sha256, inputs.product_context, coverage,
|
|
1635
|
+
cached_only=stopped_rows) if (selected or stopped_rows) else []
|
|
1636
|
+
SEMANTICS.apply_semantics(stored, classifications, client.semantic_context)
|
|
1637
|
+
before_media = {c["creative_id"]: classification_source_hash(c) for c in stored}
|
|
1638
|
+
prepare_media(client, stored, inputs, config, coverage)
|
|
1639
|
+
enriched = [c for c in stored if not c.get("vanished_after")
|
|
1640
|
+
and classification_source_hash(c) != before_media[c["creative_id"]]]
|
|
1641
|
+
if enriched:
|
|
1642
|
+
changed_ids = {c["creative_id"] for c in enriched}
|
|
1643
|
+
# A preview classification cannot claim it observed subsequently added
|
|
1644
|
+
# speech/frames. Only a successful enriched classification replaces it.
|
|
1645
|
+
classifications = [cl for cl in classifications if cl["creative_id"] not in changed_ids]
|
|
1646
|
+
classifications += classify_corpus(client, select_for_classification(enriched), taxonomy, taxonomy_sha256,
|
|
1647
|
+
inputs.product_context, coverage)
|
|
1648
|
+
dictionary = SEMANTICS.apply_semantics(stored, classifications, client.semantic_context)
|
|
1649
|
+
if inputs.runtime is not None:
|
|
1650
|
+
write_snapshot(inputs.runtime, inputs.as_of, [c for c in stored if not c.get("vanished_after")], listings, inputs.domains, inputs.sources)
|
|
1651
|
+
for c in stored:
|
|
1652
|
+
for member in c.get("source_members") or []:
|
|
1653
|
+
member["language_allowed"] = c.get("language_allowed")
|
|
1654
|
+
member["language_status"] = c.get("language_status")
|
|
1655
|
+
by_creative = {c["creative_id"]: c for c in stored}
|
|
1656
|
+
for cl in classifications:
|
|
1657
|
+
c = by_creative[cl["creative_id"]]
|
|
1658
|
+
labels = cl.get("labels") or {}
|
|
1659
|
+
detail = labels.get("message_shape") if c.get("creative_type") == "text" else "+".join(sorted(labels.get("visual_treatment") or []))
|
|
1660
|
+
c["format_detail"] = detail or "unspecified"
|
|
1661
|
+
for name in ("language_allowed", "language_status", "country", "creative_type", "format_kind", "format_detail", "competitor_tier", "relevance", "duration_scope", "source_members"):
|
|
1662
|
+
cl[name] = by_creative[cl["creative_id"]].get(name)
|
|
1663
|
+
classified_ids = {c["creative_id"] for c in classifications}
|
|
1664
|
+
for c in stored:
|
|
1665
|
+
if c.get("relevance") != "irrelevant" and c.get("language_status") != "excluded" and c["creative_id"] not in classified_ids:
|
|
1666
|
+
coverage.append({"type": "coverage", "domain": c["domain"], "country": c.get("country"), "channel": c["channel"], "source": "classify", "status": "unclassified", "detail": "No usable classification; see media and budget gaps", "tier": "C"})
|
|
1667
|
+
patterns = score_patterns(stored, classifications, taxonomy, inputs.long_running_days, dictionary)
|
|
1668
|
+
anchors = price_anchors(stored, classifications)
|
|
1669
|
+
totals = client.ledger_totals()
|
|
1670
|
+
retrieved_on = totals["first_retrieved_on"] if totals["first_retrieved_on"] == totals["last_retrieved_on"] else f"{totals['first_retrieved_on']}..{totals['last_retrieved_on']}"
|
|
1671
|
+
if inputs.footprint is not None:
|
|
1672
|
+
retrieved_on = inputs.footprint.get("retrieved_on") or inputs.as_of.isoformat()
|
|
1673
|
+
evidence = {
|
|
1674
|
+
"schema_version": EVIDENCE_SCHEMA_VERSION,
|
|
1675
|
+
"section_id": SECTION_ID,
|
|
1676
|
+
"coverage_status": "partial" if any(r.get("status") in ("budget_deferred", "unclassified", "not_extracted", "unavailable", "media_gap", "truncated_input") for r in coverage) else "sample_completed",
|
|
1677
|
+
"angle_dictionary": dictionary,
|
|
1678
|
+
"generated_by": Path(__file__).name,
|
|
1679
|
+
"retrieved_on": retrieved_on,
|
|
1680
|
+
"inputs": {"domains": inputs.domains, "domain_sources": inputs.domain_sources, "product_domain": inputs.product_domain,
|
|
1681
|
+
"sources": inputs.sources, "country": inputs.country, "eu_country": inputs.eu_country, "as_of": inputs.as_of.isoformat(),
|
|
1682
|
+
"effective_policy": inputs.effective_policy,
|
|
1683
|
+
"policy_sha256": inputs.policy_sha256, "target_countries": (inputs.footprint or {}).get("inputs", {}).get("target_countries", sorted({inputs.country, inputs.eu_country})),
|
|
1684
|
+
"collection_mode": "channel_footprint_reuse" if inputs.footprint is not None else "standalone",
|
|
1685
|
+
"accepted_languages": inputs.languages, "market_languages": inputs.market_languages,
|
|
1686
|
+
"top_videos_per_competitor_channel_market": inputs.top_videos,
|
|
1687
|
+
"deepened_cells": None if inputs.deepened_cells is None else sorted([country, channel] for country, channel in inputs.deepened_cells),
|
|
1688
|
+
"semantic_context_sha256": client.semantic_context["context_sha256"],
|
|
1689
|
+
"footprint_document_sha256": POLICY.canonical_hash(inputs.footprint) if inputs.footprint is not None else None,
|
|
1690
|
+
"taxonomy_version": taxonomy["taxonomy_version"],
|
|
1691
|
+
"taxonomy_sha256": taxonomy_sha256},
|
|
1692
|
+
# Patterns first by score, then every other type interleaved by
|
|
1693
|
+
# competitor: any sample of the file (the AI projection keeps the
|
|
1694
|
+
# first rows of each type) covers the competitors, not the first one.
|
|
1695
|
+
"rows": patterns + _interleave_by_domain(stored) + _interleave_by_domain(classifications)
|
|
1696
|
+
+ _interleave_by_domain(anchors) + _interleave_by_domain(coverage),
|
|
1697
|
+
"totals": {k: v for k, v in totals.items() if k not in ("first_retrieved_on", "last_retrieved_on")},
|
|
1698
|
+
}
|
|
1699
|
+
section = render_section(domains=inputs.domains, creatives=stored, classifications=classifications, patterns=patterns, anchors=anchors,
|
|
1700
|
+
coverage=coverage, totals=totals, as_of=inputs.as_of.isoformat(), retrieved_on=retrieved_on,
|
|
1701
|
+
long_running_days=inputs.long_running_days, taxonomy_version=taxonomy["taxonomy_version"], selection=MODEL.SELECTION.build_selection(evidence))
|
|
1702
|
+
return {"evidence": evidence, "section_markdown": section, "totals": totals,
|
|
1703
|
+
"attempt": {"actor_runs": client.actor_runs, "meta_searches": client.meta_searches, "spyfu_calls": client.spyfu_calls,
|
|
1704
|
+
"classify_calls": client.classify_calls, "media_calls": client.media_calls, "image_fetches": client.image_fetches, "cache_hits": client.cache_hits}}
|
|
1705
|
+
|
|
1706
|
+
|
|
1707
|
+
def publish_previews(rows: list[dict], workspace: Path, selection: dict | None = None, countries: list[str] | None = None) -> tuple[Path, ...]:
|
|
1708
|
+
"""Make previews portable in the research artifact folder, hash-verified: the
|
|
1709
|
+
top selection plus every row a campaign pack may cite as its original."""
|
|
1710
|
+
selection = selection or MODEL.SELECTION.build_selection({"rows": rows})
|
|
1711
|
+
selected = {(ref["country"], ref["channel"], ref["creative_type"], ref["domain"], ref["creative_id"]) for ref in selection["selected_creatives"]}
|
|
1712
|
+
publishable = [row for row in rows if row.get("type") == "creative"
|
|
1713
|
+
and (not countries or row.get("country") in countries) and (
|
|
1714
|
+
(row.get("country"), row.get("channel"), row.get("creative_type", "unknown"), row.get("domain"), row.get("creative_id")) in selected
|
|
1715
|
+
or MODEL.SELECTION.citable(row, countries))]
|
|
1716
|
+
published = set()
|
|
1717
|
+
for row in publishable:
|
|
1718
|
+
assets = row.get("media_assets") or []
|
|
1719
|
+
previews = [a for a in assets if a.get("kind") in ("image", "preview")]
|
|
1720
|
+
if not previews:
|
|
1721
|
+
previews = [a for a in assets if a.get("kind") == "frame"][:1]
|
|
1722
|
+
for asset in previews:
|
|
1723
|
+
try:
|
|
1724
|
+
ref = Path(str(asset["path"]))
|
|
1725
|
+
source = (workspace / ref).resolve()
|
|
1726
|
+
if ref.is_absolute() or ".." in ref.parts or not source.is_relative_to(workspace):
|
|
1727
|
+
raise ValueError("preview path escapes workspace")
|
|
1728
|
+
if source.stat().st_size > VIDEO.MAX_FRAME_BYTES:
|
|
1729
|
+
raise ValueError("preview exceeds the bounded image size")
|
|
1730
|
+
content = source.read_bytes()
|
|
1731
|
+
digest = hashlib.sha256(content).hexdigest()
|
|
1732
|
+
if asset.get("sha256") and asset["sha256"] != digest:
|
|
1733
|
+
raise ValueError("preview hash mismatch")
|
|
1734
|
+
mime = CF.sniff_image(content)
|
|
1735
|
+
if mime is None:
|
|
1736
|
+
raise ValueError("preview is not a supported image")
|
|
1737
|
+
extension = {"image/jpeg": "jpg", "image/png": "png", "image/webp": "webp"}[mime]
|
|
1738
|
+
dest = workspace / _artifact_path("egbe-docs/research/market/assets/creative-patterns") / f"{digest}.{extension}"
|
|
1739
|
+
if not dest.resolve().is_relative_to(workspace):
|
|
1740
|
+
raise ValueError("published preview path escapes workspace")
|
|
1741
|
+
# The cached bytes themselves, hard-linked when on one filesystem.
|
|
1742
|
+
CF.IO.link_or_copy(source, dest)
|
|
1743
|
+
if hashlib.sha256(dest.read_bytes()).hexdigest() != digest:
|
|
1744
|
+
raise ValueError("published preview changed while it was linked")
|
|
1745
|
+
asset.setdefault("source_path", asset["path"])
|
|
1746
|
+
asset.update(path=dest.relative_to(workspace).as_posix(), sha256=digest)
|
|
1747
|
+
published.add(dest.relative_to(workspace))
|
|
1748
|
+
except (OSError, ValueError, KeyError) as exc:
|
|
1749
|
+
asset["publication_status"] = "unavailable"
|
|
1750
|
+
rows.append({"type": "coverage", "source": "preview_publication", "domain": row["domain"], "country": row.get("country"), "channel": row["channel"], "creative_id": row.get("creative_id"), "status": "media_gap", "detail": str(exc), "tier": "C"})
|
|
1751
|
+
return tuple(sorted(published))
|
|
1752
|
+
|
|
1753
|
+
|
|
1754
|
+
def _interleave_by_domain(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
1755
|
+
groups: dict[str, list[dict[str, Any]]] = {}
|
|
1756
|
+
for row in rows:
|
|
1757
|
+
groups.setdefault(str(row.get("domain") or ""), []).append(row)
|
|
1758
|
+
out: list[dict[str, Any]] = []
|
|
1759
|
+
while groups:
|
|
1760
|
+
for domain in sorted(list(groups)):
|
|
1761
|
+
out.append(groups[domain].pop(0))
|
|
1762
|
+
if not groups[domain]:
|
|
1763
|
+
del groups[domain]
|
|
1764
|
+
return out
|
|
1765
|
+
|
|
1766
|
+
|
|
1767
|
+
def _arg(args: argparse.Namespace, name: str, default: Any) -> Any:
|
|
1768
|
+
value = getattr(args, name, None)
|
|
1769
|
+
return default if value is None else value
|
|
1770
|
+
|
|
1771
|
+
|
|
1772
|
+
def _write_atomic(path: Path, content: str) -> None:
|
|
1773
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
1774
|
+
tmp = path.with_suffix(path.suffix + ".tmp")
|
|
1775
|
+
tmp.write_text(content, encoding="utf-8")
|
|
1776
|
+
tmp.replace(path)
|
|
1777
|
+
|
|
1778
|
+
|
|
1779
|
+
def resolve_as_of(runtime: Path, override: str, refresh: bool) -> date:
|
|
1780
|
+
"""The cutoff is fixed at the first attempt so that every rerun asks the
|
|
1781
|
+
providers the same questions and replays from the cache. --as-of picks a
|
|
1782
|
+
cutoff; --refresh starts today's cutoff, but only once the current one has
|
|
1783
|
+
completed: a refresh that stopped on a budget resumes, it never buys the
|
|
1784
|
+
same calls again under a new date."""
|
|
1785
|
+
marker = runtime / "as-of.json"
|
|
1786
|
+
try:
|
|
1787
|
+
state = json.loads(marker.read_text(encoding="utf-8"))
|
|
1788
|
+
current = date.fromisoformat(state["as_of"])
|
|
1789
|
+
completed = bool(state.get("completed"))
|
|
1790
|
+
except (OSError, ValueError, KeyError, json.JSONDecodeError):
|
|
1791
|
+
current, completed = None, False
|
|
1792
|
+
if override:
|
|
1793
|
+
if not _DATE_RE.match(override):
|
|
1794
|
+
raise CreativePatternsError("--as-of must be YYYY-MM-DD")
|
|
1795
|
+
chosen = date.fromisoformat(override)
|
|
1796
|
+
elif current is None:
|
|
1797
|
+
chosen = date.today()
|
|
1798
|
+
elif refresh and completed:
|
|
1799
|
+
chosen = date.today()
|
|
1800
|
+
else:
|
|
1801
|
+
if refresh and current != date.today():
|
|
1802
|
+
print(f"Creative Patterns: cutoff {current.isoformat()} has not completed yet; resuming it instead of starting a new one.", file=sys.stderr)
|
|
1803
|
+
chosen = current
|
|
1804
|
+
if chosen != current:
|
|
1805
|
+
completed = False
|
|
1806
|
+
_write_atomic(marker, json.dumps({"as_of": chosen.isoformat(), "completed": completed}))
|
|
1807
|
+
return chosen
|
|
1808
|
+
|
|
1809
|
+
|
|
1810
|
+
def mark_cutoff_completed(runtime: Path, as_of: date) -> None:
|
|
1811
|
+
_write_atomic(runtime / "as-of.json", json.dumps({"as_of": as_of.isoformat(), "completed": True}))
|
|
1812
|
+
|
|
1813
|
+
|
|
1814
|
+
def linkedin_slugs(workspace: Path) -> dict[str, str]:
|
|
1815
|
+
"""Company page slugs that Channel Footprint read from competitor sites."""
|
|
1816
|
+
path = workspace / CHANNEL_FOOTPRINT_EVIDENCE_REL
|
|
1817
|
+
try:
|
|
1818
|
+
rows = json.loads(path.read_text(encoding="utf-8")).get("rows") or []
|
|
1819
|
+
except (OSError, json.JSONDecodeError, AttributeError):
|
|
1820
|
+
return {}
|
|
1821
|
+
return {r["domain"]: r["linkedin_company"] for r in rows if isinstance(r, dict) and r.get("type") == "pixel_scan" and r.get("linkedin_company")}
|
|
1822
|
+
|
|
1823
|
+
|
|
1824
|
+
def run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
|
|
1825
|
+
workspace = workspace.resolve()
|
|
1826
|
+
io = _load_script("research-artifact-io.py", "pmr_artifact_io")
|
|
1827
|
+
with io.section_attempt(workspace, SECTION_ID, context_file=getattr(args, "context_file", None)):
|
|
1828
|
+
return _run(workspace, args)
|
|
1829
|
+
|
|
1830
|
+
|
|
1831
|
+
def _run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
|
|
1832
|
+
runtime = workspace / RUNTIME_REL
|
|
1833
|
+
runtime.mkdir(parents=True, exist_ok=True)
|
|
1834
|
+
api_key = (os.environ.get("OS_API_KEY") or "").strip()
|
|
1835
|
+
if not api_key:
|
|
1836
|
+
raise CreativePatternsError("OS_API_KEY is required")
|
|
1837
|
+
os_url = (os.environ.get("OS_URL") or DEFAULT_OS_URL).strip() or DEFAULT_OS_URL
|
|
1838
|
+
config = CF.load_sources_config(Path(_arg(args, "sources_config", "")) if _arg(args, "sources_config", "") else None)
|
|
1839
|
+
taxonomy, taxonomy_sha256 = load_taxonomy()
|
|
1840
|
+
effective_policy, policy_sha256 = POLICY.policy_from_taxonomy(taxonomy)
|
|
1841
|
+
footprint = footprint_input(workspace, effective_policy, policy_sha256)
|
|
1842
|
+
raw_sources = str(_arg(args, "sources", "") or "")
|
|
1843
|
+
chosen = [s.strip().lower() for s in raw_sources.split(",") if s.strip()] if raw_sources.strip() else list(ALL_SOURCES)
|
|
1844
|
+
unknown = [s for s in chosen if s not in ALL_SOURCES]
|
|
1845
|
+
if unknown:
|
|
1846
|
+
raise CreativePatternsError(f"unknown sources: {', '.join(unknown)} (known: {', '.join(ALL_SOURCES)})")
|
|
1847
|
+
sources = [s for s in ALL_SOURCES if s in chosen]
|
|
1848
|
+
|
|
1849
|
+
max_competitors = max(1, int(_arg(args, "max_competitors", MAX_COMPETITORS)))
|
|
1850
|
+
product_domain = registrable_domain(str(_arg(args, "product_domain", "") or ""))
|
|
1851
|
+
explicit = [d for d in CF.parse_domains(_arg(args, "domains", ""), MAX_EXPLICIT_DOMAINS) if d != product_domain]
|
|
1852
|
+
evidence_mod = _load_script("research-evidence.py", "pmr_evidence_for_creative_patterns")
|
|
1853
|
+
tiered = evidence_mod.competitor_set_inputs(workspace, consumer=SECTION_ID, cap=MAX_COMPETITORS)
|
|
1854
|
+
rejected = evidence_mod.competitor_set_excluded_domains(workspace) - set(explicit)
|
|
1855
|
+
discovered = evidence_mod.keyword_inputs(workspace, consumer=SECTION_ID, row_type="competitor_domain", field="domain", cap=MAX_COMPETITORS, exclude=rejected)
|
|
1856
|
+
domains: list[str] = []
|
|
1857
|
+
domain_sources: dict[str, str] = {}
|
|
1858
|
+
for source, candidates in (("explicit", explicit), ("competitor_set", tiered), ("keyword_economics", discovered)):
|
|
1859
|
+
for domain in candidates:
|
|
1860
|
+
domain = registrable_domain(domain)
|
|
1861
|
+
if len(domains) < max_competitors and domain and domain != product_domain and domain not in domains:
|
|
1862
|
+
domains.append(domain)
|
|
1863
|
+
domain_sources[domain] = source
|
|
1864
|
+
if footprint is not None:
|
|
1865
|
+
footprint_domains = footprint["inputs"]["domains"]
|
|
1866
|
+
if any(d not in footprint_domains for d in explicit):
|
|
1867
|
+
raise CreativePatternsError("explicit competitor is outside the completed Channel Footprint; collect and finalize it there first")
|
|
1868
|
+
domains = list(dict.fromkeys(explicit + footprint_domains))[:max_competitors]
|
|
1869
|
+
domain_sources = {d: "channel_footprint" for d in domains}
|
|
1870
|
+
if not domains:
|
|
1871
|
+
raise CreativePatternsError("no competitor domains available: pass --domains, or complete the Competitor Set (or Keyword Economics) section first")
|
|
1872
|
+
target_markets = (footprint or {}).get("inputs", {}).get("target_countries") or [str(_arg(args, "country", "US") or "US").upper(), str(_arg(args, "eu_country", "DE") or "DE").upper()]
|
|
1873
|
+
languages, market_languages = language_inputs(workspace, args, target_markets)
|
|
1874
|
+
if not languages:
|
|
1875
|
+
raise CreativePatternsError("accepted languages must be explicitly supplied in intake or --languages before paid calls")
|
|
1876
|
+
for name in ("top_videos", "max_channels", "max_media_calls", "max_total_paid_starts", "max_actor_runs", "max_meta_searches", "max_spyfu_calls", "max_classify_calls", "max_image_fetches"):
|
|
1877
|
+
value = _arg(args, name, None)
|
|
1878
|
+
if value is not None and int(value) < 0:
|
|
1879
|
+
raise CreativePatternsError(f"--{name.replace('_', '-')} cannot be negative")
|
|
1880
|
+
inputs = RunInputs(
|
|
1881
|
+
domains=domains, domain_sources=domain_sources, product_domain=product_domain, sources=sources,
|
|
1882
|
+
country=str(_arg(args, "country", "US") or "US").upper(), eu_country=str(_arg(args, "eu_country", "DE") or "DE").upper(),
|
|
1883
|
+
as_of=resolve_as_of(runtime, str(_arg(args, "as_of", "") or ""), bool(_arg(args, "refresh", False))),
|
|
1884
|
+
long_running_days=None,
|
|
1885
|
+
footprint=footprint, effective_policy=effective_policy, policy_sha256=policy_sha256,
|
|
1886
|
+
restrict_sources=bool(str(_arg(args, "sources", "") or "").strip()),
|
|
1887
|
+
product_context=full_product_context(workspace),
|
|
1888
|
+
languages=languages, market_languages=market_languages, top_videos=int(_arg(args, "top_videos", 5)),
|
|
1889
|
+
max_channels=int(_arg(args, "max_channels", 0) or 0),
|
|
1890
|
+
linkedin_slugs=linkedin_slugs(workspace),
|
|
1891
|
+
runtime=runtime,
|
|
1892
|
+
)
|
|
1893
|
+
client = OsClient(
|
|
1894
|
+
# One cache namespace per cutoff: a new cutoff asks every provider again.
|
|
1895
|
+
os_url=os_url, api_key=api_key, cache_dir=runtime / "cache" / inputs.as_of.isoformat(), shared_dir=runtime / "cache" / "shared",
|
|
1896
|
+
cutoff=inputs.as_of.isoformat(), key_scope=f"{SECTION_ID}:{inputs.as_of.isoformat()}:",
|
|
1897
|
+
max_actor_runs=int(_arg(args, "max_actor_runs", MAX_ACTOR_RUNS_PER_ATTEMPT)),
|
|
1898
|
+
max_meta_searches=int(_arg(args, "max_meta_searches", MAX_META_SEARCHES_PER_ATTEMPT)),
|
|
1899
|
+
max_spyfu_calls=int(_arg(args, "max_spyfu_calls", MAX_SPYFU_CALLS_PER_ATTEMPT)),
|
|
1900
|
+
max_total_paid_starts=int(_arg(args, "max_total_paid_starts", 240)),
|
|
1901
|
+
cumulative_budget_path=runtime / "budget.json",
|
|
1902
|
+
max_media_calls=int(_arg(args, "max_media_calls", 100)),
|
|
1903
|
+
max_classify_calls=int(_arg(args, "max_classify_calls", MAX_CLASSIFY_CALLS_PER_ATTEMPT)),
|
|
1904
|
+
max_image_fetches=int(_arg(args, "max_image_fetches", CF.MAX_IMAGE_FETCHES_PER_ATTEMPT)),
|
|
1905
|
+
)
|
|
1906
|
+
existing_budget = client._cache_read(client.cumulative_budget_path)
|
|
1907
|
+
if existing_budget is None and client.cumulative_budget_path.is_file():
|
|
1908
|
+
raise CreativePatternsError("invalid cumulative budget ledger; do not reset its accepted limit")
|
|
1909
|
+
budget = existing_budget or {"schema_version": 1, "limit": client.max_total_paid_starts, "paid_starts": []}
|
|
1910
|
+
budget["limit"] = min(int(budget["limit"]), client.max_total_paid_starts)
|
|
1911
|
+
budget["spent_starts"] = len(budget["paid_starts"])
|
|
1912
|
+
_write_atomic(client.cumulative_budget_path, json.dumps(budget, indent=1))
|
|
1913
|
+
outcome = run_pipeline(client, inputs, config, taxonomy, taxonomy_sha256)
|
|
1914
|
+
selection = MODEL.SELECTION.build_selection(outcome["evidence"], workspace)
|
|
1915
|
+
published_previews = publish_previews(outcome["evidence"]["rows"], workspace, selection, outcome["evidence"]["inputs"]["target_countries"])
|
|
1916
|
+
if any(r.get("status") == "media_gap" for r in outcome["evidence"]["rows"]):
|
|
1917
|
+
outcome["evidence"]["coverage_status"] = "partial"
|
|
1918
|
+
# Regenerate the markdown with portable preview refs after materialization.
|
|
1919
|
+
rows = outcome["evidence"]["rows"]
|
|
1920
|
+
outcome["section_markdown"] = render_section(domains=inputs.domains, creatives=[r for r in rows if r["type"] == "creative"],
|
|
1921
|
+
classifications=[r for r in rows if r["type"] == "creative_classification"], patterns=[r for r in rows if r["type"] == "pattern_score"],
|
|
1922
|
+
anchors=[r for r in rows if r["type"] == "price_anchor"], coverage=[r for r in rows if r["type"] == "coverage"], totals=outcome["totals"],
|
|
1923
|
+
as_of=inputs.as_of.isoformat(), retrieved_on=outcome["evidence"]["retrieved_on"], long_running_days=inputs.long_running_days, taxonomy_version=taxonomy["taxonomy_version"], selection=selection)
|
|
1924
|
+
if outcome["totals"]["provider_calls"] == 0 and inputs.footprint is None and not any(r.get("status") == "budget_deferred" for r in outcome["evidence"]["rows"]):
|
|
1925
|
+
raise CreativePatternsError("no provider answered for any competitor; the section cannot complete without one successful call")
|
|
1926
|
+
|
|
1927
|
+
selection = MODEL.SELECTION.build_selection(outcome["evidence"], workspace)
|
|
1928
|
+
selection_json = json.dumps(selection, indent=1, ensure_ascii=False)
|
|
1929
|
+
selection_sha256 = hashlib.sha256(selection_json.encode("utf-8")).hexdigest()
|
|
1930
|
+
_write_atomic(workspace / MODEL.SELECTION.SELECTION_REL, selection_json)
|
|
1931
|
+
outcome["evidence"]["selection"] = selection
|
|
1932
|
+
outcome["evidence"]["inputs"].update(creative_selection_path=MODEL.SELECTION.SELECTION_REL.as_posix(), creative_selection_sha256=selection_sha256)
|
|
1933
|
+
evidence_json = json.dumps(outcome["evidence"], indent=1, ensure_ascii=False)
|
|
1934
|
+
_write_atomic(workspace / EVIDENCE_REL, evidence_json)
|
|
1935
|
+
_write_atomic(runtime / "result.md", outcome["section_markdown"])
|
|
1936
|
+
upsert = _load_script("upsert-markdown-section.py", "pmr_upsert_markdown_section")
|
|
1937
|
+
report_path = workspace / REPORT_REL
|
|
1938
|
+
report_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1939
|
+
upsert.upsert_markdown_section(report_path, SECTION_HEADING, outcome["section_markdown"])
|
|
1940
|
+
persist = _load_script("persist-research-report.py", "pmr_persist_research_report")
|
|
1941
|
+
persistence = persist.persist_research_report(
|
|
1942
|
+
workspace, commit_message=f"research: update {SECTION_HEADING.lower()} section",
|
|
1943
|
+
extra_paths=(Path(_artifact_path("egbe-docs/research/market/evidence/creative-patterns.json")), Path(_artifact_path("egbe-docs/research/market/evidence/creative-selection.v1.json")), *published_previews),
|
|
1944
|
+
)
|
|
1945
|
+
totals, attempt = outcome["totals"], outcome["attempt"]
|
|
1946
|
+
state = {
|
|
1947
|
+
"schema_version": SCHEMA_VERSION,
|
|
1948
|
+
"section_id": SECTION_ID,
|
|
1949
|
+
"status": "completed",
|
|
1950
|
+
"coverage_status": outcome["evidence"]["coverage_status"],
|
|
1951
|
+
"creative_selection_path": MODEL.SELECTION.SELECTION_REL.as_posix(),
|
|
1952
|
+
"creative_selection_sha256": selection_sha256,
|
|
1953
|
+
"intake_sha256": hashlib.sha256((workspace / _artifact_path("egbe-docs/research/market/intake.json")).read_bytes()).hexdigest() if (workspace / _artifact_path("egbe-docs/research/market/intake.json")).is_file() else None,
|
|
1954
|
+
"media_calls": totals["media_calls"], "reused_media_calls": totals["reused_media_calls"],
|
|
1955
|
+
"media_prompt_tokens": totals["media_prompt_tokens"], "media_completion_tokens": totals["media_completion_tokens"],
|
|
1956
|
+
"cumulative_paid_starts": totals["cumulative_paid_starts"], "cumulative_paid_start_limit": totals["cumulative_paid_start_limit"],
|
|
1957
|
+
"domains": inputs.domains,
|
|
1958
|
+
"domain_sources": inputs.domain_sources,
|
|
1959
|
+
"sources": inputs.sources,
|
|
1960
|
+
"country": inputs.country,
|
|
1961
|
+
"eu_country": inputs.eu_country,
|
|
1962
|
+
"as_of": inputs.as_of.isoformat(),
|
|
1963
|
+
"taxonomy_version": taxonomy["taxonomy_version"],
|
|
1964
|
+
"effective_policy": effective_policy, "policy_sha256": policy_sha256,
|
|
1965
|
+
"channel_footprint_sha256": hashlib.sha256((workspace / CHANNEL_FOOTPRINT_EVIDENCE_REL).read_bytes()).hexdigest() if footprint is not None else None,
|
|
1966
|
+
"taxonomy_sha256": taxonomy_sha256,
|
|
1967
|
+
"evidence_path": EVIDENCE_REL.as_posix(),
|
|
1968
|
+
"evidence_sha256": hashlib.sha256(evidence_json.encode("utf-8")).hexdigest(),
|
|
1969
|
+
"result_path": (RUNTIME_REL / "result.md").as_posix(),
|
|
1970
|
+
"result_sha256": hashlib.sha256(outcome["section_markdown"].encode("utf-8")).hexdigest(),
|
|
1971
|
+
"provider_calls": totals["provider_calls"],
|
|
1972
|
+
"actor_runs": totals["actor_runs"],
|
|
1973
|
+
"meta_searches": totals["meta_searches"],
|
|
1974
|
+
"spyfu_calls": totals["spyfu_calls"],
|
|
1975
|
+
"classify_calls": totals["classify_calls"],
|
|
1976
|
+
"reused_classify_calls": totals["reused_classify_calls"],
|
|
1977
|
+
"image_fetches": totals["image_fetches"],
|
|
1978
|
+
"rejected_calls": totals["rejected_calls"],
|
|
1979
|
+
"observations": totals["observations"],
|
|
1980
|
+
"classify_prompt_tokens": totals["classify_prompt_tokens"],
|
|
1981
|
+
"classify_completion_tokens": totals["classify_completion_tokens"],
|
|
1982
|
+
"creatives": sum(1 for r in outcome["evidence"]["rows"] if r["type"] == "creative"),
|
|
1983
|
+
"classifications": sum(1 for r in outcome["evidence"]["rows"] if r["type"] == "creative_classification"),
|
|
1984
|
+
"patterns": sum(1 for r in outcome["evidence"]["rows"] if r["type"] == "pattern_score"),
|
|
1985
|
+
"attempt_actor_runs": attempt["actor_runs"],
|
|
1986
|
+
"attempt_meta_searches": attempt["meta_searches"],
|
|
1987
|
+
"attempt_spyfu_calls": attempt["spyfu_calls"],
|
|
1988
|
+
"attempt_classify_calls": attempt["classify_calls"],
|
|
1989
|
+
"attempt_image_fetches": attempt["image_fetches"],
|
|
1990
|
+
"attempt_cache_hits": attempt["cache_hits"],
|
|
1991
|
+
"report_commit_sha": persistence.commit_sha,
|
|
1992
|
+
"publication_mode": persistence.publication_mode,
|
|
1993
|
+
"research_context": _load_script("research-artifact-io.py", "pmr_input_io").context_binding(workspace, SECTION_ID),
|
|
1994
|
+
"error": None,
|
|
1995
|
+
}
|
|
1996
|
+
_write_atomic(runtime / "request.json", json.dumps(state, indent=1))
|
|
1997
|
+
_write_atomic(runtime / "semantic-dictionary.json", json.dumps({**outcome["evidence"]["angle_dictionary"],
|
|
1998
|
+
"as_of": inputs.as_of.isoformat(), "product_sha256": hashlib.sha256(inputs.product_context.encode()).hexdigest(),
|
|
1999
|
+
"languages": inputs.languages}, ensure_ascii=False))
|
|
2000
|
+
mark_cutoff_completed(runtime, inputs.as_of)
|
|
2001
|
+
return state
|
|
2002
|
+
|
|
2003
|
+
|
|
2004
|
+
def main() -> int:
|
|
2005
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
2006
|
+
parser.add_argument("--context-file", type=Path, help="Explicit context for this study; no project files are discovered implicitly.")
|
|
2007
|
+
parser.add_argument("--workspace", type=Path, required=True)
|
|
2008
|
+
parser.add_argument("--domains", default="", help="Comma-separated competitor domains; merged ahead of the Competitor Set and Keyword Economics domains.")
|
|
2009
|
+
parser.add_argument("--product-domain", default="", help="The product's own domain; never collected as a competitor.")
|
|
2010
|
+
parser.add_argument("--sources", default="", help=f"Comma-separated subset of {', '.join(ALL_SOURCES)} (default: all).")
|
|
2011
|
+
parser.add_argument("--sources-config", default="", help="Path to an alternative actor config (default: the Channel Footprint sources config).")
|
|
2012
|
+
parser.add_argument("--max-competitors", type=int, default=MAX_COMPETITORS)
|
|
2013
|
+
parser.add_argument("--country", default="US", help="Market for SpyFu Google search ads (default US).")
|
|
2014
|
+
parser.add_argument("--eu-country", default="DE", help="EU country for official Meta delivery data (default DE).")
|
|
2015
|
+
parser.add_argument("--as-of", default="", help="Cutoff date YYYY-MM-DD (default: the first attempt's date, kept across reruns).")
|
|
2016
|
+
parser.add_argument("--refresh", action="store_true", help="Start a new cutoff at today's date (new provider calls).")
|
|
2017
|
+
parser.add_argument("--max-actor-runs", type=int, default=MAX_ACTOR_RUNS_PER_ATTEMPT)
|
|
2018
|
+
parser.add_argument("--max-meta-searches", type=int, default=MAX_META_SEARCHES_PER_ATTEMPT)
|
|
2019
|
+
parser.add_argument("--max-spyfu-calls", type=int, default=MAX_SPYFU_CALLS_PER_ATTEMPT, help="SpyFu ad-history lookups per attempt (two per competitor).")
|
|
2020
|
+
parser.add_argument("--max-classify-calls", type=int, default=MAX_CLASSIFY_CALLS_PER_ATTEMPT, help="Classification calls per attempt (up to ten creatives each).")
|
|
2021
|
+
parser.add_argument("--max-image-fetches", type=int, default=CF.MAX_IMAGE_FETCHES_PER_ATTEMPT)
|
|
2022
|
+
parser.add_argument("--languages", default="", help="Explicit accepted language codes, e.g. en,es")
|
|
2023
|
+
parser.add_argument("--market-languages", default="", help="Per-market language codes: US:en,es;GB:en")
|
|
2024
|
+
parser.add_argument("--top-videos", type=int, default=5)
|
|
2025
|
+
parser.add_argument("--max-channels", type=int, default=0, help="Express scope: deepen only this many candidate channels per market (0: every cell).")
|
|
2026
|
+
parser.add_argument("--max-media-calls", type=int, default=100)
|
|
2027
|
+
parser.add_argument("--max-total-paid-starts", type=int, default=240, help="Durable section-wide cap; reruns do not reset it")
|
|
2028
|
+
args = parser.parse_args()
|
|
2029
|
+
try:
|
|
2030
|
+
state = run(args.workspace, args)
|
|
2031
|
+
except CreativePatternsError as exc:
|
|
2032
|
+
print(f"Creative Patterns could not complete: {exc}", file=sys.stderr)
|
|
2033
|
+
return 1
|
|
2034
|
+
except ValueError as exc:
|
|
2035
|
+
print(f"Creative Patterns could not complete: {exc}", file=sys.stderr)
|
|
2036
|
+
return 1
|
|
2037
|
+
print(json.dumps({k: state[k] for k in ("status", "domains", "creatives", "classifications", "patterns", "provider_calls")}, indent=1))
|
|
2038
|
+
return 0
|
|
2039
|
+
|
|
2040
|
+
|
|
2041
|
+
if __name__ == "__main__":
|
|
2042
|
+
raise SystemExit(main())
|