@egbe-ai/opencode 0.0.0-stage → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +124 -2
- package/index.js +5 -0
- package/package.json +55 -4
- package/plugin.js +230 -0
- package/session-log.js +315 -0
- package/skills/.egbe-catalog/manifest.json +1 -0
- package/skills/egbe-os-analytics/SKILL.md +499 -0
- package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
- package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
- package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
- package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
- package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
- package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
- package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
- package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
- package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
- package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
- package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
- package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
- package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
- package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
- package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
- package/skills/egbe-os-db/SKILL.md +174 -0
- package/skills/egbe-os-deploy/SKILL.md +412 -0
- package/skills/egbe-os-design/SKILL.md +182 -0
- package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
- package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
- package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
- package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
- package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
- package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
- package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
- package/skills/egbe-os-destination-readiness/references/report.md +116 -0
- package/skills/egbe-os-domains/SKILL.md +189 -0
- package/skills/egbe-os-errors/SKILL.md +259 -0
- package/skills/egbe-os-git/SKILL.md +275 -0
- package/skills/egbe-os-git/references/issues.md +33 -0
- package/skills/egbe-os-google-ads/SKILL.md +163 -0
- package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
- package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
- package/skills/egbe-os-google-ads/references/conversions.md +153 -0
- package/skills/egbe-os-google-ads/references/delivery.md +128 -0
- package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
- package/skills/egbe-os-google-ads/references/reporting.md +105 -0
- package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
- package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
- package/skills/egbe-os-google-auth/SKILL.md +285 -0
- package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
- package/skills/egbe-os-images/SKILL.md +163 -0
- package/skills/egbe-os-images/references/stock-images.md +21 -0
- package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
- package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
- package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
- package/skills/egbe-os-llm/SKILL.md +160 -0
- package/skills/egbe-os-mail/SKILL.md +180 -0
- package/skills/egbe-os-me/SKILL.md +119 -0
- package/skills/egbe-os-meta-ads/SKILL.md +114 -0
- package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
- package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
- package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
- package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
- package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
- package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
- package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
- package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
- package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
- package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
- package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
- package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
- package/skills/egbe-os-notifications/SKILL.md +78 -0
- package/skills/egbe-os-openai-ads/SKILL.md +149 -0
- package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
- package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
- package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
- package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
- package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
- package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
- package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
- package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
- package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
- package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
- package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
- package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
- package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
- package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
- package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
- package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
- package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
- package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
- package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
- package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
- package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
- package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
- package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
- package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
- package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
- package/skills/egbe-os-payments/SKILL.md +215 -0
- package/skills/egbe-os-payments/agents/openai.yaml +4 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
- package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
- package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
- package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
- package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
- package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
- package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
- package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
- package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
- package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
- package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
- package/skills/egbe-os-playbook/SKILL.md +79 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
- package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
- package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
- package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
- package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
- package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
- package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
- package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
- package/skills/egbe-os-playbook/playbooks/product.md +67 -0
- package/skills/egbe-os-playbook/references/catalog.md +127 -0
- package/skills/egbe-os-playbook/references/changes.md +167 -0
- package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
- package/skills/egbe-os-product-auth/SKILL.md +151 -0
- package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
- package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
- package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
- package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
- package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
- package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
- package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
- package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
- package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
- package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
- package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
- package/skills/egbe-os-product-build/SKILL.md +111 -0
- package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-build/references/capabilities.md +25 -0
- package/skills/egbe-os-product-build/references/engineering.md +72 -0
- package/skills/egbe-os-product-build/references/execution.md +55 -0
- package/skills/egbe-os-product-build/references/foundations.md +93 -0
- package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
- package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
- package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
- package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
- package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
- package/skills/egbe-os-product-market-research/SKILL.md +159 -0
- package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
- package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
- package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
- package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
- package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
- package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
- package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
- package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
- package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
- package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
- package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
- package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
- package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
- package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
- package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
- package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
- package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
- package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
- package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
- package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
- package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
- package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
- package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
- package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
- package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
- package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
- package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
- package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
- package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
- package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
- package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
- package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
- package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
- package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
- package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
- package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
- package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
- package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
- package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
- package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
- package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
- package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
- package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
- package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
- package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
- package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
- package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
- package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
- package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
- package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
- package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
- package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
- package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
- package/skills/egbe-os-product-report/SKILL.md +150 -0
- package/skills/egbe-os-product-report/references/artifacts.md +84 -0
- package/skills/egbe-os-product-report/references/brief.md +34 -0
- package/skills/egbe-os-product-report/references/daily.md +76 -0
- package/skills/egbe-os-product-report/references/sources.md +68 -0
- package/skills/egbe-os-product-routines/SKILL.md +166 -0
- package/skills/egbe-os-product-routines/references/catalog.md +22 -0
- package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
- package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
- package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
- package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
- package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
- package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
- package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
- package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
- package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
- package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
- package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
- package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
- package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
- package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
- package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
- package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
- package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
- package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
- package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
- package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
- package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
- package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
- package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
- package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
- package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
- package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
- package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
- package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
- package/skills/egbe-os-report-ui/SKILL.md +133 -0
- package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
- package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
- package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
- package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
- package/skills/egbe-os-report-ui/references/components.md +209 -0
- package/skills/egbe-os-research/SKILL.md +125 -0
- package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
- package/skills/egbe-os-scrape/SKILL.md +160 -0
- package/skills/egbe-os-search/SKILL.md +182 -0
- package/skills/egbe-os-search/references/google-places.md +81 -0
- package/skills/egbe-os-support/SKILL.md +149 -0
- package/skills/egbe-os-telemetry/SKILL.md +344 -0
- package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
- package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
- package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
- package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
- package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
- package/skills/egbe-os-tools/SKILL.md +180 -0
- package/skills/egbe-os-tools/references/theona.md +26 -0
- package/skills/egbe-os-wallet/SKILL.md +181 -0
- package/skills/egbe-setup/EGBE.md +484 -0
- package/skills/egbe-setup/SKILL.md +116 -0
- package/skills/egbe-setup/references/project.md +80 -0
- package/state.js +37 -0
- package/update.js +361 -0
|
@@ -0,0 +1,2258 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Run the deterministic Channel Footprint section: where competitors buy traffic.
|
|
3
|
+
|
|
4
|
+
Ad libraries say whether a competitor advertises on a platform; nothing in the
|
|
5
|
+
other measured sections says WHERE its traffic comes from or which platforms
|
|
6
|
+
it pays. This runner answers that per competitor from five kinds of
|
|
7
|
+
observation, each cheap and reproducible:
|
|
8
|
+
|
|
9
|
+
- tracking tags on the competitor's own pages (home page plus its Google
|
|
10
|
+
Tag Manager containers): ad-platform pixels, native-ad networks, review
|
|
11
|
+
marketplace trackers and affiliate platforms reveal where conversions are
|
|
12
|
+
bought, including platforms without any public ad library;
|
|
13
|
+
- ad-library presence and age on Google, Meta (US page search plus the
|
|
14
|
+
official EU delivery data), LinkedIn, TikTok (EU library) and Reddit,
|
|
15
|
+
through OS scrape actors and the official Meta endpoint;
|
|
16
|
+
- the public traffic-source split and monthly visit estimate (a Similarweb
|
|
17
|
+
actor, or the SimilarWeb tools provider once it is configured);
|
|
18
|
+
- partner signals from web search: affiliate program pages, coupon and
|
|
19
|
+
promo-code listings, sponsorship mentions;
|
|
20
|
+
- page history from the Wayback Machine: domain age and how often the home
|
|
21
|
+
page changed over the last two years.
|
|
22
|
+
|
|
23
|
+
The staged runner preserves a normalized ad corpus, validates context-bound
|
|
24
|
+
agent relevance decisions, and reports scoped activity, duration, coverage and
|
|
25
|
+
comparison roles without selecting a business priority. No LLM runs
|
|
26
|
+
inside the runner. Cache replays add no spend; durable cumulative provider
|
|
27
|
+
starts and per-attempt budgets bound collection and sample expansion.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import sys as _artifact_sys
|
|
33
|
+
from pathlib import Path as _ArtifactPath
|
|
34
|
+
_artifact_sys.path.insert(0, str(_ArtifactPath(__file__).resolve().parent))
|
|
35
|
+
from egbe_artifacts import artifact_path as _artifact_path
|
|
36
|
+
|
|
37
|
+
import runpy as _egbe_runpy
|
|
38
|
+
from pathlib import Path as _EgbePath
|
|
39
|
+
_egbe_adapter = _EgbePath(__file__).with_name("connection_identity.py")
|
|
40
|
+
_connection_headers = _egbe_runpy.run_path(str(_egbe_adapter))["connection_headers"] if _egbe_adapter.is_file() else lambda _url: {}
|
|
41
|
+
|
|
42
|
+
import argparse
|
|
43
|
+
import base64
|
|
44
|
+
import hashlib
|
|
45
|
+
import html as html_lib
|
|
46
|
+
import http.client
|
|
47
|
+
import importlib.util
|
|
48
|
+
import json
|
|
49
|
+
import os
|
|
50
|
+
import re
|
|
51
|
+
import sys
|
|
52
|
+
import time
|
|
53
|
+
import urllib.error
|
|
54
|
+
import urllib.parse
|
|
55
|
+
import urllib.request
|
|
56
|
+
from dataclasses import dataclass, field
|
|
57
|
+
from datetime import date, datetime, timedelta, timezone
|
|
58
|
+
from pathlib import Path
|
|
59
|
+
from types import ModuleType
|
|
60
|
+
from typing import Any, Callable
|
|
61
|
+
|
|
62
|
+
SCHEMA_VERSION = 2
|
|
63
|
+
SECTION_ID = "channel_footprint"
|
|
64
|
+
SECTION_HEADING = "Channel footprint"
|
|
65
|
+
DISPLAY_NAME = "Channel Footprint"
|
|
66
|
+
|
|
67
|
+
DEFAULT_OS_URL = "https://os.hz.egbe.dev"
|
|
68
|
+
RUNTIME_REL = Path(_artifact_path("egbe-docs/research/market/work")) / SECTION_ID
|
|
69
|
+
EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/channel-footprint.json"))
|
|
70
|
+
REPORT_REL = Path(_artifact_path("egbe-docs/research/market/full-report.md"))
|
|
71
|
+
SOURCES_CONFIG_REL = Path("references/config/channel-footprint-sources.v1.json")
|
|
72
|
+
WAYBACK_CDX_URL = "https://web.archive.org/cdx/search/cdx"
|
|
73
|
+
|
|
74
|
+
MAX_COMPETITORS = 10
|
|
75
|
+
MAX_EXPLICIT_DOMAINS = 10
|
|
76
|
+
MAX_ACTOR_RUNS_PER_ATTEMPT = 12
|
|
77
|
+
MAX_META_SEARCHES_PER_ATTEMPT = 10
|
|
78
|
+
MAX_SEARCH_CALLS_PER_ATTEMPT = 30
|
|
79
|
+
MAX_FETCHES_PER_ATTEMPT = 40
|
|
80
|
+
MAX_CDX_CALLS_PER_ATTEMPT = 25
|
|
81
|
+
# One SpyFu domain-stats lookup per competitor (12 months = 12 rows, ~$0.006).
|
|
82
|
+
MAX_SPYFU_CALLS_PER_ATTEMPT = 12
|
|
83
|
+
SPYFU_MONTHS = 12
|
|
84
|
+
FETCH_MAX_BYTES = 2_000_000
|
|
85
|
+
FETCH_TIMEOUT_SECONDS = 25.0
|
|
86
|
+
CDX_TIMEOUT_SECONDS = 60.0
|
|
87
|
+
CDX_PAUSE_SECONDS = 1.0
|
|
88
|
+
CDX_ROW_LIMIT = 500
|
|
89
|
+
GTM_CONTAINERS_PER_SITE = 3
|
|
90
|
+
HISTORY_MONTHS = 24
|
|
91
|
+
BRAVE_COUNT = 10
|
|
92
|
+
PARTNER_URLS_PER_KIND = 3
|
|
93
|
+
ACTOR_RETRY_DELAY_SECONDS = 20.0
|
|
94
|
+
# The OS host refuses connections for about 40 s while a gateway on it
|
|
95
|
+
# redeploys (observed live 2026-10-02); one retry after this delay covers it.
|
|
96
|
+
OS_UNREACHABLE_RETRY_DELAY_SECONDS = 45.0
|
|
97
|
+
FREE_RETRY_DELAY_SECONDS = 5.0
|
|
98
|
+
DEFAULT_COUNTRY = "US"
|
|
99
|
+
DEFAULT_EU_COUNTRY = "DE"
|
|
100
|
+
# Product relevance by the creative classifier's semantic verdict: small
|
|
101
|
+
# batches keep each answer bounded; a decided sample may expand once more
|
|
102
|
+
# per round, so a few rounds settle a collect.
|
|
103
|
+
MAX_IMAGE_FETCHES_PER_ATTEMPT = 500
|
|
104
|
+
MAX_IMAGE_BYTES = 4 * 1024 * 1024
|
|
105
|
+
IMAGE_TIMEOUT_SECONDS = 30.0
|
|
106
|
+
RELEVANCE_PATH = "/v1/media/classify-creative"
|
|
107
|
+
RELEVANCE_BATCH = 4
|
|
108
|
+
MAX_RELEVANCE_CALLS_PER_ATTEMPT = 40
|
|
109
|
+
RELEVANCE_TIMEOUT_SECONDS = 330.0
|
|
110
|
+
MODEL_RELEVANCE_ROUNDS = 3
|
|
111
|
+
PRESENT_SHARE_PCT = 0.0 # Any positive modeled share is an observation, not a success threshold.
|
|
112
|
+
|
|
113
|
+
ALL_SOURCES = ("pixels", "history", "traffic", "google", "spyfu", "meta", "linkedin", "partners", "tiktok", "reddit")
|
|
114
|
+
CHANNELS = (
|
|
115
|
+
"google_ads", "youtube_ads", "meta_ads", "linkedin_ads", "tiktok_ads", "reddit_ads", "microsoft_ads", "x_ads",
|
|
116
|
+
"pinterest_ads", "snap_ads", "quora_ads", "display_native", "review_marketplace", "affiliate", "paid_search", "paid_social",
|
|
117
|
+
)
|
|
118
|
+
CHANNEL_LABELS = {
|
|
119
|
+
"google_ads": "Google Ads", "youtube_ads": "YouTube (video)", "meta_ads": "Meta", "linkedin_ads": "LinkedIn",
|
|
120
|
+
"tiktok_ads": "TikTok", "reddit_ads": "Reddit", "microsoft_ads": "Microsoft Ads", "x_ads": "X",
|
|
121
|
+
"pinterest_ads": "Pinterest", "snap_ads": "Snapchat", "quora_ads": "Quora", "display_native": "Display / native",
|
|
122
|
+
"review_marketplace": "Review marketplaces", "affiliate": "Affiliate / partners", "paid_search": "Paid search (category)", "paid_social": "Paid social (category)",
|
|
123
|
+
}
|
|
124
|
+
TAG_MANAGER_SIGNATURES: tuple[tuple[str, str], ...] = (
|
|
125
|
+
("google_tag_manager", r"googletagmanager\.com/gtm\.js|\bGTM-[A-Z0-9]{4,10}\b"),
|
|
126
|
+
("segment", r"cdn\.segment\.com/analytics\.js|\banalytics\.load\s*\("),
|
|
127
|
+
("tealium", r"tags\.tiqcdn\.com"),
|
|
128
|
+
("adobe_launch", r"assets\.adobedtm\.com"),
|
|
129
|
+
)
|
|
130
|
+
LINKEDIN_COMPANY_RE = re.compile(r"linkedin\.com/(?:company|school|showcase)/([A-Za-z0-9._%-]+)", re.IGNORECASE)
|
|
131
|
+
AFFILIATE_NETWORK_DOMAINS = {
|
|
132
|
+
"impact.com", "partnerstack.com", "shareasale.com", "cj.com", "awin.com", "rakutenadvertising.com",
|
|
133
|
+
"rewardful.com", "firstpromoter.com", "tapfiliate.com", "refersion.com", "getrewardful.com",
|
|
134
|
+
"affiliateprogramdb.com", "lasso.net", "affpaying.com", "offervault.com",
|
|
135
|
+
}
|
|
136
|
+
COUPON_DOMAINS = {
|
|
137
|
+
"retailmenot.com", "joinhoney.com", "coupons.com", "dealspotr.com", "couponfollow.com", "slickdeals.net",
|
|
138
|
+
"wethrift.com", "knoji.com", "couponbirds.com", "coupert.com", "simplycodes.com", "capitaloneshopping.com",
|
|
139
|
+
"rakuten.com", "groupon.com", "couponcabin.com", "offers.com", "promocodes.com", "hotdeals.com",
|
|
140
|
+
}
|
|
141
|
+
TRAFFIC_CHANNEL_MAP = {
|
|
142
|
+
"paid_search": "paid_search", "search_paid": "paid_search", "paid_social": "paid_social", "social_paid": "paid_social",
|
|
143
|
+
"display": "display_native", "display_ads": "display_native", "displayads": "display_native",
|
|
144
|
+
"paid_referrals": "display_native", "affiliates": "affiliate", "affiliate": "affiliate",
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
# (tag id, channel, regex over page HTML and tag-manager container JS)
|
|
148
|
+
TAG_SIGNATURES: tuple[tuple[str, str, str], ...] = (
|
|
149
|
+
("meta_pixel", "meta_ads", r"connect\.facebook\.net/[^\"']*/fbevents\.js|\bfbq\s*\("),
|
|
150
|
+
("google_ads_tag", "google_ads", r"googleads\.g\.doubleclick\.net|googleadservices\.com/pagead/conversion|\bAW-\d{6,}"),
|
|
151
|
+
("reddit_pixel", "reddit_ads", r"redditstatic\.com/ads/pixel\.js|\brdt\s*\(\s*['\"]init"),
|
|
152
|
+
("linkedin_insight", "linkedin_ads", r"snap\.licdn\.com/li\.lms-analytics/insight\.min\.js|_linkedin_partner_id"),
|
|
153
|
+
("tiktok_pixel", "tiktok_ads", r"analytics\.tiktok\.com/i18n/pixel/events\.js|\bttq\.load\s*\("),
|
|
154
|
+
("microsoft_uet", "microsoft_ads", r"bat\.bing\.com/bat\.js|\buetq\b"),
|
|
155
|
+
("x_pixel", "x_ads", r"static\.ads-twitter\.com/uwt\.js|\btwq\s*\(\s*['\"](?:config|init)"),
|
|
156
|
+
("pinterest_tag", "pinterest_ads", r"s\.pinimg\.com/ct/core\.js|\bpintrk\s*\("),
|
|
157
|
+
("snap_pixel", "snap_ads", r"sc-static\.net/scevent\.min\.js|\bsnaptr\s*\("),
|
|
158
|
+
("quora_pixel", "quora_ads", r"a\.quora\.com/qevents\.js|\bqp\s*\(\s*['\"]init"),
|
|
159
|
+
("taboola", "display_native", r"cdn\.taboola\.com/libtrc|\b_tfa\b"),
|
|
160
|
+
("outbrain", "display_native", r"amplify\.outbrain\.com/cp/obtp\.js|\bobApi\s*\("),
|
|
161
|
+
("criteo", "display_native", r"static\.criteo\.net/js/ld/ld\.js"),
|
|
162
|
+
("capterra_tracker", "review_marketplace", r"ct\.capterra\.com/capterra_tracker"),
|
|
163
|
+
("g2_tracker", "review_marketplace", r"tracking\.g2crowd\.com|g2crowd\.com/attribution"),
|
|
164
|
+
("partnerstack", "affiliate", r"js\.partnerstack\.com|\bgrsm\.io\b|growsumo"),
|
|
165
|
+
("impact", "affiliate", r"utt\.impactcdn\.com|impactradius"),
|
|
166
|
+
("rewardful", "affiliate", r"r\.wdfl\.co|\brewardful\b"),
|
|
167
|
+
("firstpromoter", "affiliate", r"cdn\.firstpromoter\.com|\bfpr\s*\(\s*['\"]init"),
|
|
168
|
+
("tapfiliate", "affiliate", r"script\.tapfiliate\.com"),
|
|
169
|
+
("refersion", "affiliate", r"refersion\.com/tracker"),
|
|
170
|
+
("shareasale", "affiliate", r"shareasale\.com/sale\.cfm|static\.shareasale\.com/json/shareasale\.js"),
|
|
171
|
+
)
|
|
172
|
+
GTM_ID_RE = re.compile(r"\bGTM-[A-Z0-9]{4,10}\b")
|
|
173
|
+
PARTNER_KINDS: tuple[tuple[str, str], ...] = (
|
|
174
|
+
# (kind, query template); precision rules live in `_partner_hit`.
|
|
175
|
+
("affiliate_program", '"{brand}" affiliate program'),
|
|
176
|
+
("coupon_listing", '"{brand}" promo code coupon'),
|
|
177
|
+
("sponsorship", '"sponsored by {brand}" OR "{brand} sponsored"'),
|
|
178
|
+
)
|
|
179
|
+
_DOMAIN_RE = re.compile(r"^[a-z0-9][a-z0-9.-]{0,251}\.[a-z]{2,24}$")
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class ChannelFootprintError(RuntimeError):
|
|
183
|
+
"""Terminal runner failure; runtime state is preserved for resume."""
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class SourceUnavailable(ChannelFootprintError):
|
|
187
|
+
"""A provider rejected the request itself (invalid input, unsupported
|
|
188
|
+
actor): a fact about the source, cached so reruns never re-spend."""
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
class BudgetExhausted(ChannelFootprintError):
|
|
192
|
+
"""The per-attempt budget of one metered or free call type is spent;
|
|
193
|
+
always propagates so a rerun resumes from the cache."""
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
class OsUnreachable(ChannelFootprintError):
|
|
197
|
+
"""The request never reached the OS gateway (connection refused, DNS,
|
|
198
|
+
TLS, a connect timeout), so nothing was started or billed. Retried once;
|
|
199
|
+
then it fails the attempt instead of becoming a coverage gap, because it
|
|
200
|
+
says nothing about the source and a rerun resumes from the cache."""
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
# Consecutive failures of one supplementary source within an attempt after
|
|
204
|
+
# which its remaining domains are skipped instead of started (a provider that
|
|
205
|
+
# is down must not burn the paid actor budget one domain at a time).
|
|
206
|
+
SOURCE_BREAKER_FAILURES = 2
|
|
207
|
+
# Hosts that are not a competitor's destination but the platform itself or a
|
|
208
|
+
# store/link hub: the advertiser name decides for those rows.
|
|
209
|
+
NEUTRAL_DESTINATION_HOSTS = {
|
|
210
|
+
# the platforms themselves and their messaging / short hosts
|
|
211
|
+
"facebook.com", "instagram.com", "fb.me", "fb.com", "fb.watch", "fb.gg", "m.me", "ig.me", "messenger.com",
|
|
212
|
+
"whatsapp.com", "wa.me", "t.me", "reddit.com", "redd.it", "linkedin.com", "lnkd.in", "tiktok.com", "vm.tiktok.com",
|
|
213
|
+
"youtu.be", "youtube.com", "t.co", "x.com", "twitter.com",
|
|
214
|
+
# stores and link hubs
|
|
215
|
+
"apps.apple.com", "itunes.apple.com", "play.google.com", "linktr.ee", "lnk.to", "lnk.bio",
|
|
216
|
+
# shorteners and attribution/deep-link hosts (the real destination is hidden)
|
|
217
|
+
"bit.ly", "tinyurl.com", "ow.ly", "buff.ly", "hubs.ly", "hubs.la", "amzn.to", "rebrand.ly", "cutt.ly",
|
|
218
|
+
"onelink.me", "app.link", "adjust.com", "adj.st", "go.link", "smart.link", "page.link",
|
|
219
|
+
}
|
|
220
|
+
# Click-tracking wrappers whose `u` query parameter carries the real address.
|
|
221
|
+
REDIRECT_WRAPPER_HOSTS = {"l.facebook.com", "lm.facebook.com", "l.instagram.com", "l.messenger.com", "google.com", "www.google.com"}
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _load_script(filename: str, module_name: str) -> ModuleType:
|
|
225
|
+
path = Path(__file__).resolve().parent / filename
|
|
226
|
+
spec = importlib.util.spec_from_file_location(module_name, path)
|
|
227
|
+
if spec is None or spec.loader is None:
|
|
228
|
+
raise ChannelFootprintError(f"cannot load bundled script {filename}")
|
|
229
|
+
module = importlib.util.module_from_spec(spec)
|
|
230
|
+
sys.modules[spec.name] = module
|
|
231
|
+
previous = sys.dont_write_bytecode
|
|
232
|
+
sys.dont_write_bytecode = True
|
|
233
|
+
try:
|
|
234
|
+
spec.loader.exec_module(module)
|
|
235
|
+
finally:
|
|
236
|
+
sys.dont_write_bytecode = previous
|
|
237
|
+
return module
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
# Cache entries are stored gzip-compressed; plain entries of older runs replay too.
|
|
241
|
+
IO = _load_script("research-artifact-io.py", "pmr_channel_footprint_io")
|
|
242
|
+
_KEYWORD_ECONOMICS: ModuleType | None = None
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _keyword_economics() -> ModuleType:
|
|
246
|
+
global _KEYWORD_ECONOMICS
|
|
247
|
+
if _KEYWORD_ECONOMICS is None:
|
|
248
|
+
_KEYWORD_ECONOMICS = _load_script("run-keyword-economics.py", "pmr_keyword_economics_for_channel_footprint")
|
|
249
|
+
return _KEYWORD_ECONOMICS
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def registrable_domain(value: str) -> str:
|
|
253
|
+
return _keyword_economics().registrable_domain(value)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def brand_token(domain: str) -> str:
|
|
257
|
+
return _keyword_economics().brand_token(domain)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def load_sources_config(path: Path | None = None) -> dict[str, Any]:
|
|
261
|
+
config_path = path or (Path(__file__).resolve().parent.parent / SOURCES_CONFIG_REL)
|
|
262
|
+
try:
|
|
263
|
+
raw = json.loads(config_path.read_text(encoding="utf-8"))
|
|
264
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
265
|
+
raise ChannelFootprintError(f"cannot read the sources config {config_path.name}: {exc}") from exc
|
|
266
|
+
if raw.get("schema_version") != 1 or not isinstance(raw.get("actors"), dict):
|
|
267
|
+
raise ChannelFootprintError("sources config must have schema_version 1 and an actors object")
|
|
268
|
+
for key, spec in raw["actors"].items():
|
|
269
|
+
if not isinstance(spec, dict) or not str(spec.get("actor") or "").strip():
|
|
270
|
+
raise ChannelFootprintError(f"sources config actor `{key}` has no actor id")
|
|
271
|
+
registry = raw.get("channels") or {}
|
|
272
|
+
if not isinstance(registry, dict):
|
|
273
|
+
raise ChannelFootprintError("sources config channels must be an object of channel labels")
|
|
274
|
+
global CHANNELS
|
|
275
|
+
for channel, label in registry.items():
|
|
276
|
+
if not re.fullmatch(r"[a-z][a-z0-9_]+", channel) or not isinstance(label, str) or not label:
|
|
277
|
+
raise ChannelFootprintError("invalid channel registry entry")
|
|
278
|
+
CHANNEL_LABELS[channel] = label
|
|
279
|
+
CHANNELS = tuple(dict.fromkeys((*CHANNELS, *registry)))
|
|
280
|
+
return raw
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
# ── OS client: metered actors, Meta library, web search; free fetches ──────
|
|
284
|
+
|
|
285
|
+
@dataclass
|
|
286
|
+
class OsClient:
|
|
287
|
+
os_url: str
|
|
288
|
+
api_key: str
|
|
289
|
+
cache_dir: Path
|
|
290
|
+
max_actor_runs: int = MAX_ACTOR_RUNS_PER_ATTEMPT
|
|
291
|
+
max_meta_searches: int = MAX_META_SEARCHES_PER_ATTEMPT
|
|
292
|
+
max_search_calls: int = MAX_SEARCH_CALLS_PER_ATTEMPT
|
|
293
|
+
max_fetches: int = MAX_FETCHES_PER_ATTEMPT
|
|
294
|
+
max_cdx_calls: int = MAX_CDX_CALLS_PER_ATTEMPT
|
|
295
|
+
max_spyfu_calls: int = MAX_SPYFU_CALLS_PER_ATTEMPT
|
|
296
|
+
actor_runs: int = 0
|
|
297
|
+
meta_searches: int = 0
|
|
298
|
+
search_calls: int = 0
|
|
299
|
+
fetches: int = 0
|
|
300
|
+
cdx_calls: int = 0
|
|
301
|
+
spyfu_calls: int = 0
|
|
302
|
+
cache_hits: int = 0
|
|
303
|
+
cache_writes: int = 0
|
|
304
|
+
touched: set[str] = field(default_factory=set)
|
|
305
|
+
# Consecutive live non-budget failures per supplementary source in this attempt.
|
|
306
|
+
source_failures: dict[str, int] = field(default_factory=dict)
|
|
307
|
+
request: Callable[..., dict[str, Any]] | None = None
|
|
308
|
+
fetch: Callable[[str], tuple[int, str]] | None = None
|
|
309
|
+
max_image_fetches: int = MAX_IMAGE_FETCHES_PER_ATTEMPT
|
|
310
|
+
image_fetches: int = 0
|
|
311
|
+
max_relevance_calls: int = MAX_RELEVANCE_CALLS_PER_ATTEMPT
|
|
312
|
+
relevance_calls: int = 0
|
|
313
|
+
fetch_bytes: Callable[[str], tuple[int, bytes]] | None = None
|
|
314
|
+
# The cutoff this client collects for: spend is reported per cutoff, and
|
|
315
|
+
# answers reused from an earlier cutoff are counted apart.
|
|
316
|
+
cutoff: str = ""
|
|
317
|
+
sleep: Callable[[float], None] | None = None
|
|
318
|
+
# Prefix of the wire Idempotency-Key. Sections that send byte-identical
|
|
319
|
+
# payloads (Creative Patterns reuses these actor inputs, and at every new
|
|
320
|
+
# cutoff) set their own scope so the gateway never merges their starts.
|
|
321
|
+
key_scope: str = ""
|
|
322
|
+
max_total_paid_starts: int | None = None
|
|
323
|
+
cumulative_budget_path: Path | None = None
|
|
324
|
+
|
|
325
|
+
def _http(self, path: str, payload: dict[str, Any], timeout: float, *, idempotency_key: str | None = None) -> dict[str, Any]:
|
|
326
|
+
url = f"{self.os_url.rstrip('/')}{path}"
|
|
327
|
+
headers = {"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json", "Accept": "application/json"}
|
|
328
|
+
if idempotency_key:
|
|
329
|
+
headers["Idempotency-Key"] = idempotency_key
|
|
330
|
+
req = urllib.request.Request(url, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST")
|
|
331
|
+
try:
|
|
332
|
+
for _name, _value in _connection_headers(req.full_url).items():
|
|
333
|
+
req.add_unredirected_header(_name, _value)
|
|
334
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
335
|
+
envelope = json.loads(resp.read().decode("utf-8"))
|
|
336
|
+
except urllib.error.HTTPError as exc:
|
|
337
|
+
detail = exc.read().decode("utf-8", errors="replace")[:500]
|
|
338
|
+
raise ChannelFootprintError(f"OS {path} failed: HTTP {exc.code} {detail}") from exc
|
|
339
|
+
except urllib.error.URLError as exc:
|
|
340
|
+
raise OsUnreachable(f"OS {path} unreachable: {exc.reason}") from exc
|
|
341
|
+
except (OSError, http.client.HTTPException, ValueError) as exc:
|
|
342
|
+
# A read timeout or a dropped connection after the request was
|
|
343
|
+
# sent surfaces outside URLError; it is a provider failure too.
|
|
344
|
+
raise ChannelFootprintError(f"OS {path} failed in transit: {type(exc).__name__}: {str(exc)[:200]}") from exc
|
|
345
|
+
data = envelope.get("data")
|
|
346
|
+
if envelope.get("code") not in (None, 200) or not isinstance(data, dict):
|
|
347
|
+
raise ChannelFootprintError(f"OS {path} returned no data: {str(envelope)[:300]}")
|
|
348
|
+
return data
|
|
349
|
+
|
|
350
|
+
def _fetch_url(self, url: str) -> tuple[int, str]:
|
|
351
|
+
"""Plain GET of a public page; the status and a bounded body."""
|
|
352
|
+
req = urllib.request.Request(url, headers={
|
|
353
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
|
|
354
|
+
"Accept": "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8",
|
|
355
|
+
"Accept-Language": "en-US,en;q=0.9",
|
|
356
|
+
})
|
|
357
|
+
timeout = CDX_TIMEOUT_SECONDS if url.startswith(WAYBACK_CDX_URL) else FETCH_TIMEOUT_SECONDS
|
|
358
|
+
try:
|
|
359
|
+
for _name, _value in _connection_headers(req.full_url).items():
|
|
360
|
+
req.add_unredirected_header(_name, _value)
|
|
361
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
362
|
+
body = resp.read(FETCH_MAX_BYTES + 1)
|
|
363
|
+
status = int(getattr(resp, "status", 200) or 200)
|
|
364
|
+
except urllib.error.HTTPError as exc:
|
|
365
|
+
return exc.code, ""
|
|
366
|
+
except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
|
|
367
|
+
raise ChannelFootprintError(f"fetch of {url} failed: {getattr(exc, 'reason', exc)}") from exc
|
|
368
|
+
return status, body[:FETCH_MAX_BYTES].decode("utf-8", errors="replace")
|
|
369
|
+
|
|
370
|
+
def _cache_read(self, cache_file: Path) -> dict[str, Any] | None:
|
|
371
|
+
try:
|
|
372
|
+
return IO.read_cache_entry(cache_file)
|
|
373
|
+
except (OSError, ValueError, EOFError):
|
|
374
|
+
IO.drop_cache_entry(cache_file)
|
|
375
|
+
return None
|
|
376
|
+
|
|
377
|
+
def _cache_write(self, cache_file: Path, entry: dict[str, Any]) -> None:
|
|
378
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
379
|
+
IO.write_cache_entry(cache_file, entry)
|
|
380
|
+
self.cache_writes += 1
|
|
381
|
+
|
|
382
|
+
def _journal_start(self, digest: str) -> None:
|
|
383
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
384
|
+
with (self.cache_dir / "starts.log").open("a", encoding="utf-8") as handle:
|
|
385
|
+
handle.write(f"{date.today().isoformat()} {digest}\n")
|
|
386
|
+
|
|
387
|
+
def _prior_start_count(self, digest: str) -> int:
|
|
388
|
+
try:
|
|
389
|
+
lines = (self.cache_dir / "starts.log").read_text(encoding="utf-8").splitlines()
|
|
390
|
+
except OSError:
|
|
391
|
+
return 0
|
|
392
|
+
return sum(1 for line in lines if line.endswith(f" {digest}"))
|
|
393
|
+
|
|
394
|
+
def _reserve_paid_start(self, kind: str, digest: str) -> None:
|
|
395
|
+
"""A durable total cap survives every retry, failed call and restart."""
|
|
396
|
+
if self.max_total_paid_starts is None:
|
|
397
|
+
return
|
|
398
|
+
path = self.cumulative_budget_path or self.cache_dir.parent / "budget.json"
|
|
399
|
+
try:
|
|
400
|
+
budget = json.loads(path.read_text())
|
|
401
|
+
except FileNotFoundError:
|
|
402
|
+
try:
|
|
403
|
+
legacy = [line for line in (self.cache_dir / "starts.log").read_text().splitlines() if line]
|
|
404
|
+
except OSError:
|
|
405
|
+
legacy = []
|
|
406
|
+
budget = {"schema_version": 1, "limit": self.max_total_paid_starts, "paid_starts": [{"kind": "legacy_actor", "digest": line.split()[-1], "started_on": line.split()[0]} for line in legacy]}
|
|
407
|
+
_write_atomic(path, json.dumps(budget, indent=1))
|
|
408
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
409
|
+
raise ChannelFootprintError(f"invalid cumulative budget ledger: {exc}") from exc
|
|
410
|
+
if budget.get("schema_version") != 1 or type(budget.get("limit")) is not int or budget["limit"] < 0 or not isinstance(budget.get("paid_starts"), list):
|
|
411
|
+
raise ChannelFootprintError("invalid cumulative budget ledger shape")
|
|
412
|
+
# A new invocation cannot silently raise the previously accepted cap.
|
|
413
|
+
maximum = min(int(budget["limit"]), self.max_total_paid_starts)
|
|
414
|
+
starts = budget.get("paid_starts", [])
|
|
415
|
+
if len(starts) >= maximum:
|
|
416
|
+
raise BudgetExhausted(f"cumulative paid-start budget exhausted ({maximum}); remaining checks are budget_deferred")
|
|
417
|
+
starts.append({"kind": kind, "digest": digest, "started_on": date.today().isoformat()})
|
|
418
|
+
budget.update(limit=maximum, paid_starts=starts, spent_starts=len(starts), money_estimate=None)
|
|
419
|
+
_write_atomic(path, json.dumps(budget, indent=1))
|
|
420
|
+
|
|
421
|
+
def _metered(self, kind: str, path: str, payload: dict[str, Any], timeout: float, *, budget_attr: str, budget_max: int, label: str, paid: bool, retry_delay: float,
|
|
422
|
+
cache_payload: dict[str, Any] | None = None) -> tuple[dict[str, Any], str]:
|
|
423
|
+
cache_payload = payload if cache_payload is None else cache_payload
|
|
424
|
+
key_raw = json.dumps({"kind": kind, "path": path, "payload": cache_payload}, sort_keys=True)
|
|
425
|
+
digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
|
|
426
|
+
cache_file = self.cache_dir / f"{digest}.json"
|
|
427
|
+
self.touched.add(digest)
|
|
428
|
+
entry = self._cache_read(cache_file)
|
|
429
|
+
if entry is not None:
|
|
430
|
+
self.cache_hits += 1
|
|
431
|
+
response = entry.get("response")
|
|
432
|
+
return (response if isinstance(response, dict) else {}), entry.get("retrieved_on") or date.today().isoformat()
|
|
433
|
+
|
|
434
|
+
transport = self.request or self._http
|
|
435
|
+
attempts = 0
|
|
436
|
+
wire_key = None
|
|
437
|
+
while True:
|
|
438
|
+
if getattr(self, budget_attr) >= budget_max:
|
|
439
|
+
progress = (
|
|
440
|
+
f"{self.cache_writes} new answer(s) cached in this attempt; rerun to continue from the cache"
|
|
441
|
+
if self.cache_writes
|
|
442
|
+
else "nothing new was cached in this attempt: a failing per-domain source costs up to "
|
|
443
|
+
f"{2 * SOURCE_BREAKER_FAILURES} starts before it is skipped, so raise the budget or narrow --sources"
|
|
444
|
+
)
|
|
445
|
+
raise BudgetExhausted(f"{label} budget exhausted ({budget_max} per attempt); {progress}")
|
|
446
|
+
self._reserve_paid_start(kind, digest)
|
|
447
|
+
setattr(self, budget_attr, getattr(self, budget_attr) + 1)
|
|
448
|
+
if paid:
|
|
449
|
+
# Paid starts count BEFORE the transport call (fail closed)
|
|
450
|
+
# and are journaled durably: a failed start may still bill.
|
|
451
|
+
# Every start gets its own key, including an in-attempt retry:
|
|
452
|
+
# the gateway released the failed start's wallet hold, and a
|
|
453
|
+
# reused key would run the actor without being able to settle.
|
|
454
|
+
wire_key = f"{self.key_scope}{digest}-{self._prior_start_count(digest)}"
|
|
455
|
+
self._journal_start(digest)
|
|
456
|
+
try:
|
|
457
|
+
response = transport(path, payload, timeout, idempotency_key=wire_key)
|
|
458
|
+
break
|
|
459
|
+
except ChannelFootprintError as exc:
|
|
460
|
+
if (kind == "actor" and re.search(r"HTTP 400\b", str(exc))) or (
|
|
461
|
+
kind == "spyfu" and re.search(r"HTTP 4(?:00|22)\b", str(exc))
|
|
462
|
+
):
|
|
463
|
+
# The provider rejected this input (observed live: an
|
|
464
|
+
# actor URL list without schemes; SpyFu answers 400 for an
|
|
465
|
+
# unsupported market); cache the rejection so replays stay
|
|
466
|
+
# byte-stable and never pay for the same refusal again.
|
|
467
|
+
response = {"unavailable": str(exc)[:400]}
|
|
468
|
+
break
|
|
469
|
+
unreachable = isinstance(exc, OsUnreachable)
|
|
470
|
+
if attempts >= 1 or not (unreachable or re.search(r"HTTP 5\d\d", str(exc))):
|
|
471
|
+
raise
|
|
472
|
+
attempts += 1
|
|
473
|
+
(self.sleep or time.sleep)(max(retry_delay, OS_UNREACHABLE_RETRY_DELAY_SECONDS) if unreachable else retry_delay)
|
|
474
|
+
retrieved_on = date.today().isoformat()
|
|
475
|
+
self._cache_write(cache_file, {"kind": kind, "path": path, "payload": cache_payload, "response": response, "retrieved_on": retrieved_on})
|
|
476
|
+
return (response if isinstance(response, dict) else {}), retrieved_on
|
|
477
|
+
|
|
478
|
+
def _shared(self) -> Path:
|
|
479
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
480
|
+
return self.cache_dir
|
|
481
|
+
|
|
482
|
+
def _download(self, url: str) -> tuple[int, bytes]:
|
|
483
|
+
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36", "Accept": "image/avif,image/webp,image/png,image/jpeg,*/*;q=0.8"})
|
|
484
|
+
try:
|
|
485
|
+
with urllib.request.urlopen(req, timeout=IMAGE_TIMEOUT_SECONDS) as resp:
|
|
486
|
+
return int(getattr(resp, "status", 200) or 200), resp.read(MAX_IMAGE_BYTES + 1)
|
|
487
|
+
except urllib.error.HTTPError as exc:
|
|
488
|
+
return exc.code, b""
|
|
489
|
+
except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
|
|
490
|
+
raise ChannelFootprintError(f"image download failed: {getattr(exc, 'reason', exc)}") from exc
|
|
491
|
+
|
|
492
|
+
def fetch_image(self, url: str) -> dict[str, Any] | None:
|
|
493
|
+
"""Free, cached download of one ad image. Bytes are kept under the
|
|
494
|
+
shared cache by content hash, so a rerun can classify again after the
|
|
495
|
+
platform's signed URL has expired. Returns the image, None when the
|
|
496
|
+
image is definitively unusable (not a JPEG, PNG or WebP, too large,
|
|
497
|
+
HTTP 4xx; cached as such), or {"transient": ...} for a rate limit,
|
|
498
|
+
server error or network failure (not cached; retried next attempt)."""
|
|
499
|
+
digest = hashlib.sha256(json.dumps({"kind": "image", "url": url}, sort_keys=True).encode("utf-8")).hexdigest()
|
|
500
|
+
cache_file = self._shared() / f"{digest}.json"
|
|
501
|
+
self.touched.add(digest)
|
|
502
|
+
entry = self._cache_read(cache_file)
|
|
503
|
+
if entry is not None:
|
|
504
|
+
response = entry.get("response") if isinstance(entry.get("response"), dict) else {}
|
|
505
|
+
blob = self._shared() / "images" / str(response.get("file") or "")
|
|
506
|
+
if not response.get("sha256") or blob.is_file():
|
|
507
|
+
self.cache_hits += 1
|
|
508
|
+
return response if response.get("sha256") else None
|
|
509
|
+
if self.image_fetches >= self.max_image_fetches:
|
|
510
|
+
raise BudgetExhausted(f"image download budget exhausted ({self.max_image_fetches} per attempt); rerun to continue from the cache")
|
|
511
|
+
self.image_fetches += 1
|
|
512
|
+
try:
|
|
513
|
+
status, content = (self.fetch_bytes or self._download)(url)
|
|
514
|
+
except ChannelFootprintError as exc:
|
|
515
|
+
return {"transient": str(exc)[:200]}
|
|
516
|
+
if _transient_status(status):
|
|
517
|
+
return {"transient": f"HTTP {status}"}
|
|
518
|
+
media_type = sniff_image(content) if status == 200 and 0 < len(content) <= MAX_IMAGE_BYTES else None
|
|
519
|
+
response: dict[str, Any] = {"status": status}
|
|
520
|
+
if media_type:
|
|
521
|
+
sha = hashlib.sha256(content).hexdigest()
|
|
522
|
+
ext = media_type.split("/", 1)[1]
|
|
523
|
+
(self._shared() / "images").mkdir(parents=True, exist_ok=True)
|
|
524
|
+
# Atomic: the same bytes may be hard-linked into published assets.
|
|
525
|
+
tmp = self._shared() / "images" / f".{sha}.{ext}.tmp"
|
|
526
|
+
tmp.write_bytes(content)
|
|
527
|
+
tmp.replace(self._shared() / "images" / f"{sha}.{ext}")
|
|
528
|
+
response.update(sha256=sha, media_type=media_type, file=f"{sha}.{ext}", bytes=len(content))
|
|
529
|
+
else:
|
|
530
|
+
response["unusable"] = "not a JPEG, PNG or WebP image under 4 MB" if status == 200 else f"HTTP {status}"
|
|
531
|
+
self._cache_write(cache_file, {"kind": "image", "url": url, "response": response, "cutoff": self.cutoff,
|
|
532
|
+
"retrieved_on": date.today().isoformat()})
|
|
533
|
+
return response if media_type else None
|
|
534
|
+
|
|
535
|
+
def image_bytes(self, image: dict[str, Any]) -> bytes:
|
|
536
|
+
return (self._shared() / "images" / image["file"]).read_bytes()
|
|
537
|
+
|
|
538
|
+
def classify_relevance(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
539
|
+
"""One cached classifier call (an AI upstream call), reserved in the
|
|
540
|
+
durable budget like a provider start. The cache key covers frame
|
|
541
|
+
hashes, never the image bytes."""
|
|
542
|
+
key = {**payload, "frames": [{k: v for k, v in frame.items() if k != "content_base64"} for frame in payload.get("frames", [])]}
|
|
543
|
+
response, _ = self._metered("relevance", RELEVANCE_PATH, payload, RELEVANCE_TIMEOUT_SECONDS, budget_attr="relevance_calls",
|
|
544
|
+
budget_max=self.max_relevance_calls, label="relevance classification", paid=False,
|
|
545
|
+
retry_delay=FREE_RETRY_DELAY_SECONDS, cache_payload=key)
|
|
546
|
+
return response
|
|
547
|
+
|
|
548
|
+
def run_actor(self, actor: str, actor_input: dict[str, Any], limit: int) -> tuple[list[dict[str, Any]], str]:
|
|
549
|
+
payload = {"actor": actor, "input": actor_input, "sync": True, "limit": limit}
|
|
550
|
+
response, retrieved_on = self._metered(
|
|
551
|
+
"actor", "/scrape/apify/run", payload, 330.0,
|
|
552
|
+
budget_attr="actor_runs", budget_max=self.max_actor_runs, label="actor-run", paid=True, retry_delay=ACTOR_RETRY_DELAY_SECONDS,
|
|
553
|
+
)
|
|
554
|
+
if response.get("unavailable"):
|
|
555
|
+
raise SourceUnavailable(f"actor {actor} rejected the request: {response['unavailable']}")
|
|
556
|
+
rows = response.get("rows")
|
|
557
|
+
return ([r for r in rows if isinstance(r, dict)] if isinstance(rows, list) else []), retrieved_on
|
|
558
|
+
|
|
559
|
+
def meta_library_search(self, payload: dict[str, Any]) -> tuple[list[dict[str, Any]], str]:
|
|
560
|
+
response, retrieved_on = self._metered(
|
|
561
|
+
"meta-ad-library", "/tools/meta-ad-library/search", payload, 90.0,
|
|
562
|
+
budget_attr="meta_searches", budget_max=self.max_meta_searches, label="meta-ad-library", paid=False, retry_delay=FREE_RETRY_DELAY_SECONDS,
|
|
563
|
+
)
|
|
564
|
+
rows = response.get("data")
|
|
565
|
+
return ([r for r in rows if isinstance(r, dict)] if isinstance(rows, list) else []), retrieved_on
|
|
566
|
+
|
|
567
|
+
def spyfu_domain_stats(self, domain: str, country: str, months: int) -> tuple[dict[str, Any], str]:
|
|
568
|
+
"""SpyFu's modeled Google Ads budget, paid clicks and paid keywords per
|
|
569
|
+
month. Billed per returned row (one per month) to the shared account;
|
|
570
|
+
the gateway's daily `spyfu:rows` limit bounds it per claw."""
|
|
571
|
+
response, retrieved_on = self._metered(
|
|
572
|
+
"spyfu", "/tools/spyfu/domain-stats", {"domain": domain, "country_code": country, "months": months}, 90.0,
|
|
573
|
+
budget_attr="spyfu_calls", budget_max=self.max_spyfu_calls, label="SpyFu lookup", paid=False, retry_delay=FREE_RETRY_DELAY_SECONDS,
|
|
574
|
+
)
|
|
575
|
+
if response.get("unavailable"):
|
|
576
|
+
raise SourceUnavailable(f"SpyFu rejected the request: {response['unavailable']}")
|
|
577
|
+
return response, retrieved_on
|
|
578
|
+
|
|
579
|
+
def brave(self, query: str, count: int, country: str) -> tuple[dict[str, Any], str]:
|
|
580
|
+
return self._metered(
|
|
581
|
+
"search", "/search/web/brave", {"query": query, "count": count, "country": country}, 60.0,
|
|
582
|
+
budget_attr="search_calls", budget_max=self.max_search_calls, label="web search", paid=False, retry_delay=FREE_RETRY_DELAY_SECONDS,
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
def fetch_page(self, url: str, *, budget_attr: str = "fetches", budget_max: int | None = None, label: str = "page fetch") -> tuple[dict[str, Any], str]:
|
|
586
|
+
"""Free, cached GET of a public page or API; failures are cached as a
|
|
587
|
+
status so a blocked site does not get hammered on every rerun."""
|
|
588
|
+
kind = "cdx" if budget_attr == "cdx_calls" else "fetch"
|
|
589
|
+
key_raw = json.dumps({"kind": kind, "url": url}, sort_keys=True)
|
|
590
|
+
digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
|
|
591
|
+
cache_file = self.cache_dir / f"{digest}.json"
|
|
592
|
+
self.touched.add(digest)
|
|
593
|
+
entry = self._cache_read(cache_file)
|
|
594
|
+
if entry is not None:
|
|
595
|
+
cached = entry.get("response") if isinstance(entry.get("response"), dict) else {}
|
|
596
|
+
if _transient_status(cached.get("status")):
|
|
597
|
+
# An older runner cached rate limits and server errors; such
|
|
598
|
+
# an entry is a miss, never a replayable answer.
|
|
599
|
+
IO.drop_cache_entry(cache_file)
|
|
600
|
+
else:
|
|
601
|
+
self.cache_hits += 1
|
|
602
|
+
return cached, entry.get("retrieved_on") or date.today().isoformat()
|
|
603
|
+
maximum = self.max_fetches if budget_max is None else budget_max
|
|
604
|
+
if getattr(self, budget_attr) >= maximum:
|
|
605
|
+
raise BudgetExhausted(f"{label} budget exhausted ({maximum} per attempt); {self.cache_writes} new answer(s) cached in this attempt; rerun to continue from the cache")
|
|
606
|
+
setattr(self, budget_attr, getattr(self, budget_attr) + 1)
|
|
607
|
+
transport = self.fetch or self._fetch_url
|
|
608
|
+
retrieved_on = date.today().isoformat()
|
|
609
|
+
try:
|
|
610
|
+
status, body = transport(url)
|
|
611
|
+
response: dict[str, Any] = {"status": status, "body": body}
|
|
612
|
+
except ChannelFootprintError as exc:
|
|
613
|
+
# A timeout or connection error is transient: report it for this
|
|
614
|
+
# attempt but do not cache it, so a rerun tries again (bounded by
|
|
615
|
+
# the per-attempt budget).
|
|
616
|
+
return {"status": None, "body": "", "error": str(exc)[:200]}, retrieved_on
|
|
617
|
+
if _transient_status(status):
|
|
618
|
+
# Rate limits and server errors are transient as well: not cached,
|
|
619
|
+
# not an observation.
|
|
620
|
+
return {**response, "error": f"HTTP {status}"}, retrieved_on
|
|
621
|
+
self._cache_write(cache_file, {"kind": kind, "url": url, "response": response, "retrieved_on": retrieved_on})
|
|
622
|
+
return response, retrieved_on
|
|
623
|
+
|
|
624
|
+
def ledger_totals(self) -> dict[str, Any]:
|
|
625
|
+
"""Durable provenance over entries THIS execution touched. A cached
|
|
626
|
+
actor rejection is a journaled start, not a successful observation."""
|
|
627
|
+
counts = {"actor": 0, "meta-ad-library": 0, "search": 0, "spyfu": 0, "fetch": 0, "cdx": 0, "relevance": 0}
|
|
628
|
+
rejected = 0
|
|
629
|
+
rejected_spyfu = 0
|
|
630
|
+
dates: list[str] = []
|
|
631
|
+
if self.cache_dir.is_dir():
|
|
632
|
+
for digest in sorted(self.touched):
|
|
633
|
+
try:
|
|
634
|
+
entry = IO.read_cache_entry(self.cache_dir / f"{digest}.json")
|
|
635
|
+
except (OSError, ValueError, EOFError):
|
|
636
|
+
continue
|
|
637
|
+
if not isinstance(entry, dict):
|
|
638
|
+
continue
|
|
639
|
+
kind = entry.get("kind")
|
|
640
|
+
response = entry.get("response") if isinstance(entry.get("response"), dict) else {}
|
|
641
|
+
if response.get("unavailable"):
|
|
642
|
+
if kind == "spyfu":
|
|
643
|
+
rejected_spyfu += 1
|
|
644
|
+
else:
|
|
645
|
+
rejected += 1
|
|
646
|
+
elif kind in ("fetch", "cdx") and _transient_status(response.get("status")):
|
|
647
|
+
continue # an older cache entry of a rate limit or server error is not an observation
|
|
648
|
+
elif kind in counts:
|
|
649
|
+
counts[kind] += 1
|
|
650
|
+
if isinstance(entry.get("retrieved_on"), str):
|
|
651
|
+
dates.append(entry["retrieved_on"])
|
|
652
|
+
today = date.today().isoformat()
|
|
653
|
+
try:
|
|
654
|
+
starts_recorded = sum(1 for line in (self.cache_dir / "starts.log").read_text(encoding="utf-8").splitlines() if line)
|
|
655
|
+
except OSError:
|
|
656
|
+
starts_recorded = 0
|
|
657
|
+
return {
|
|
658
|
+
"actor_runs": counts["actor"],
|
|
659
|
+
"meta_searches": counts["meta-ad-library"],
|
|
660
|
+
"search_calls": counts["search"],
|
|
661
|
+
"fetches": counts["fetch"],
|
|
662
|
+
"cdx_calls": counts["cdx"],
|
|
663
|
+
"spyfu_calls": counts["spyfu"],
|
|
664
|
+
"relevance_calls": counts["relevance"],
|
|
665
|
+
"provider_calls": counts["actor"] + counts["meta-ad-library"] + counts["search"] + counts["spyfu"],
|
|
666
|
+
"rejected_actor_runs": rejected,
|
|
667
|
+
"rejected_spyfu_calls": rejected_spyfu,
|
|
668
|
+
# Every successful observation, metered or free: the completion
|
|
669
|
+
# proof for selections that use only free sources.
|
|
670
|
+
"observations": counts["actor"] + counts["meta-ad-library"] + counts["search"] + counts["spyfu"] + counts["fetch"] + counts["cdx"],
|
|
671
|
+
"actor_starts_recorded": starts_recorded,
|
|
672
|
+
"first_retrieved_on": min(dates) if dates else today,
|
|
673
|
+
"last_retrieved_on": max(dates) if dates else today,
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
# ── inputs ──────────────────────────────────────────────────────────────────
|
|
678
|
+
|
|
679
|
+
def parse_domains(raw: str, cap: int) -> list[str]:
|
|
680
|
+
domains: list[str] = []
|
|
681
|
+
for term in (raw or "").split(","):
|
|
682
|
+
domain = registrable_domain(term)
|
|
683
|
+
if domain and _DOMAIN_RE.match(domain) and domain not in domains:
|
|
684
|
+
domains.append(domain)
|
|
685
|
+
return domains[:cap]
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
DOMAIN_SOURCES = frozenset({"pixels", "history", "traffic", "linkedin", "reddit"}) # one check per domain, not per country
|
|
689
|
+
EXPANDABLE_SOURCES = frozenset({"google", "meta", "tiktok", "linkedin"}) # a direct sample may be expanded once
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
def check_starts(source: str, country: str | None, pixel_actor_fallback: bool = False) -> dict[str, int]:
|
|
693
|
+
"""Paid provider starts of one check, by provider kind."""
|
|
694
|
+
if source == "partners":
|
|
695
|
+
return {"search": 3}
|
|
696
|
+
if source == "spyfu":
|
|
697
|
+
return {"spyfu": 1}
|
|
698
|
+
if source == "meta":
|
|
699
|
+
return {"actor": 1, "meta-ad-library": 1} if country in EU_COUNTRIES else {"actor": 1}
|
|
700
|
+
if source == "tiktok":
|
|
701
|
+
return {"actor": 1} if country in EU_COUNTRIES else {}
|
|
702
|
+
if source == "pixels":
|
|
703
|
+
return {"actor": 1} if pixel_actor_fallback else {}
|
|
704
|
+
return {"actor": 1} if source in {"traffic", "google", "linkedin", "reddit"} else {}
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def collection_plan(competitors: list[tuple[str, str]], sources: list[str], target_countries: list[str], *,
|
|
708
|
+
pixel_actor_fallback: bool = False) -> list[dict[str, Any]]:
|
|
709
|
+
"""Every (domain, tier) x source x country check with its upper bound of
|
|
710
|
+
paid provider starts. Every scoped domain is collected regardless of role."""
|
|
711
|
+
plan = []
|
|
712
|
+
for domain, role in competitors:
|
|
713
|
+
for source in sources:
|
|
714
|
+
for country in ([None] if source in DOMAIN_SOURCES else target_countries):
|
|
715
|
+
starts = check_starts(source, country, pixel_actor_fallback)
|
|
716
|
+
plan.append({"domain": domain, "competitor_tier": role, "source": source, "country": country,
|
|
717
|
+
"max_expansion_factor": 2 if source in EXPANDABLE_SOURCES else 1,
|
|
718
|
+
"provider_starts": starts, "provider_starts_upper_bound": sum(starts.values())})
|
|
719
|
+
return plan
|
|
720
|
+
|
|
721
|
+
|
|
722
|
+
def sniff_image(content: bytes) -> str | None:
|
|
723
|
+
if content[:3] == b"\xff\xd8\xff":
|
|
724
|
+
return "image/jpeg"
|
|
725
|
+
if content[:8] == b"\x89PNG\r\n\x1a\n":
|
|
726
|
+
return "image/png"
|
|
727
|
+
if content[:4] == b"RIFF" and content[8:12] == b"WEBP":
|
|
728
|
+
return "image/webp"
|
|
729
|
+
return None
|
|
730
|
+
|
|
731
|
+
|
|
732
|
+
def typed_intake(workspace: Path) -> dict[str, Any]:
|
|
733
|
+
"""Use the existing typed-input validator; no prose or project discovery."""
|
|
734
|
+
loader = _load_script("run-competitor-set.py", "pmr_typed_intake_for_channels")
|
|
735
|
+
try:
|
|
736
|
+
return loader.load_intake(workspace / _artifact_path("egbe-docs/research/market/intake.json")) or {}
|
|
737
|
+
except loader.CompetitorSetError as exc:
|
|
738
|
+
raise ValueError(str(exc)) from exc
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
def relevance_context(workspace: Path, args: argparse.Namespace) -> tuple[str, list[str]]:
|
|
742
|
+
"""The offer the classifier judges ads against, and the audience languages."""
|
|
743
|
+
parts = [IO.context_text(workspace, SECTION_ID)]
|
|
744
|
+
intake = typed_intake(workspace)
|
|
745
|
+
offer = {key: value for key, value in (intake.get("offer") or {}).items()
|
|
746
|
+
if value not in (None, "", [])}
|
|
747
|
+
if offer:
|
|
748
|
+
parts.append("Offer card: " + json.dumps(offer, ensure_ascii=False))
|
|
749
|
+
text = "\n\n".join(part for part in parts if part)
|
|
750
|
+
if len(text) > 24000:
|
|
751
|
+
raise ChannelFootprintError("relevance context exceeds 24000 characters; supply bounded context")
|
|
752
|
+
explicit = str(_arg(args, "languages", "") or "")
|
|
753
|
+
languages = ([code.strip().lower() for code in explicit.split(",") if code.strip()]
|
|
754
|
+
if explicit else list(intake.get("languages") or []))
|
|
755
|
+
_load_script("creative-semantics.py", "pmr_relevance_context_validator").build_semantic_context(text, languages)
|
|
756
|
+
return text, languages
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
def model_relevance(client: "OsClient", corpus: list[dict[str, Any]], pending: list[str], decisions: dict[str, Any], context_hash: str,
|
|
760
|
+
product_context: str, languages: list[str]) -> dict[str, Any] | None:
|
|
761
|
+
"""Decide pending ads by the classifier's relevance verdict from their text
|
|
762
|
+
or their downloaded image; an ad with neither stays pending for review.
|
|
763
|
+
None: nothing new was decided (nothing to judge, or the budget stopped)."""
|
|
764
|
+
platforms = set(_taxonomy()["platforms"])
|
|
765
|
+
judged = []
|
|
766
|
+
for ad in (ad for ad in corpus if ad["ad_id"] in pending and str(ad["platform"]).removesuffix("_ads") in platforms):
|
|
767
|
+
text = (ad.get("ad_text") or "").strip() # a CTA label alone is no copy: the offer is in the picture
|
|
768
|
+
image = client.fetch_image(ad["image_url"]) if not text and ad.get("image_url") else None
|
|
769
|
+
if text or (image and image.get("sha256")):
|
|
770
|
+
judged.append((ad, image if image and image.get("sha256") else None))
|
|
771
|
+
if not judged:
|
|
772
|
+
return None
|
|
773
|
+
semantic = _load_script("creative-semantics.py", "pmr_semantics_for_footprint").build_semantic_context(product_context, languages)
|
|
774
|
+
new: list[dict[str, Any]] = []
|
|
775
|
+
for start in range(0, len(judged), RELEVANCE_BATCH):
|
|
776
|
+
batch = {ad["ad_id"]: ad for ad, _ in judged[start:start + RELEVANCE_BATCH]}
|
|
777
|
+
frames = [{"ref": f"{ad['ad_id']}/image", "timestamp_seconds": 0, "media_type": image["media_type"], "sha256": image["sha256"],
|
|
778
|
+
"content_base64": base64.b64encode(client.image_bytes(image)).decode("ascii")} for ad, image in judged[start:start + RELEVANCE_BATCH] if image]
|
|
779
|
+
creatives = [{"ad_ref": ad_id, "platform": str(ad["platform"]).removesuffix("_ads"), "advertiser": str(ad.get("advertiser") or ad["domain"])[:200],
|
|
780
|
+
"body": (ad.get("ad_text") or None) and str(ad["ad_text"])[:4000], "cta_label": (ad.get("cta") or None) and str(ad["cta"])[:100],
|
|
781
|
+
"landing_url": (ad.get("landing_url") or None) and str(ad["landing_url"])[:2000],
|
|
782
|
+
"frame_refs": [frame["ref"] for frame in frames if frame["ref"] == f"{ad_id}/image"]} for ad_id, ad in batch.items()]
|
|
783
|
+
try:
|
|
784
|
+
data = client.classify_relevance({"schema_version": 1, "taxonomy_version": _taxonomy()["taxonomy_version"], "semantic_context": semantic,
|
|
785
|
+
"creatives": creatives, "frames": frames})
|
|
786
|
+
except BudgetExhausted:
|
|
787
|
+
break # the rest stays pending; a rerun continues from the cache
|
|
788
|
+
for item in data.get("classifications") or []:
|
|
789
|
+
ad = batch.get(item.get("ad_ref"))
|
|
790
|
+
if ad is None:
|
|
791
|
+
continue
|
|
792
|
+
verdict = (item.get("semantics") or {}).get("relevance") or {}
|
|
793
|
+
status = verdict.get("status") if verdict.get("status") in {"relevant", "irrelevant"} else "unknown"
|
|
794
|
+
excerpt = f' Excerpt: "{verdict["excerpt"]}".' if verdict.get("excerpt") else ""
|
|
795
|
+
source = "ad_text" if (ad.get("ad_text") or "").strip() else "image"
|
|
796
|
+
new.append({"ad_id": ad["ad_id"], "candidate_sha256": ad["candidate_sha256"], "relevance": status,
|
|
797
|
+
"reason": f"{verdict.get('reason') or 'The classifier gave no reason.'}{excerpt} (classifier role {data.get('model_role')})",
|
|
798
|
+
"evidence_refs": [f"{ad['ad_id']}:{source}", *([ad["landing_url"]] if ad.get("landing_url") else [])], "decided_by": "model"})
|
|
799
|
+
if not new:
|
|
800
|
+
return None
|
|
801
|
+
decided = {d["ad_id"] for d in new}
|
|
802
|
+
return {"schema_version": 1, "product_context_sha256": context_hash,
|
|
803
|
+
"decisions": [d for d in decisions.get("decisions", []) if d.get("ad_id") not in decided] + new}
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def _taxonomy() -> dict[str, Any]:
|
|
807
|
+
return json.loads((Path(__file__).resolve().parent.parent / "references/config/creative-taxonomy.v1.json").read_text(encoding="utf-8"))
|
|
808
|
+
|
|
809
|
+
|
|
810
|
+
def collection_budget(plan: list[dict[str, Any]], config: dict[str, Any]) -> dict[str, Any]:
|
|
811
|
+
"""Persistent shared cap, including expansion and the configured retry reserve.
|
|
812
|
+
|
|
813
|
+
Starts are not allocated to provider kinds in the ledger. Failed uncached
|
|
814
|
+
requests can consume the pool across resumed attempts, so each active kind's
|
|
815
|
+
conservative upper bound is the whole pool, rather than a promised share.
|
|
816
|
+
"""
|
|
817
|
+
reserve = int(config.get("budget", {}).get("retry_reserve_factor", 1))
|
|
818
|
+
if reserve < 0:
|
|
819
|
+
raise ChannelFootprintError("retry reserve factor must be nonnegative")
|
|
820
|
+
base, expanded = {}, {}
|
|
821
|
+
for check in plan:
|
|
822
|
+
for kind, starts in check["provider_starts"].items():
|
|
823
|
+
base[kind] = base.get(kind, 0) + starts
|
|
824
|
+
expanded[kind] = expanded.get(kind, 0) + starts * check["max_expansion_factor"]
|
|
825
|
+
limit = max(1, sum(expanded.values()) * (1 + reserve))
|
|
826
|
+
return {"base": base, "expanded": expanded, "upper": {kind: limit for kind in expanded},
|
|
827
|
+
"retry_reserve_factor": reserve, "cumulative_limit": limit}
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
def paid_start_budget(plan: list[dict[str, Any]], config: dict[str, Any]) -> dict[str, int]:
|
|
831
|
+
"""Aggregate runner budget projected from the shared provider calculation."""
|
|
832
|
+
budget = collection_budget(plan, config)
|
|
833
|
+
return {"base": sum(budget["base"].values()), "bounded": sum(budget["expanded"].values()),
|
|
834
|
+
"retry_reserve_factor": budget["retry_reserve_factor"], "cumulative_limit": budget["cumulative_limit"]}
|
|
835
|
+
|
|
836
|
+
|
|
837
|
+
def parse_sources(raw: str) -> list[str]:
|
|
838
|
+
if not (raw or "").strip():
|
|
839
|
+
return list(ALL_SOURCES)
|
|
840
|
+
chosen = [s.strip().lower() for s in raw.split(",") if s.strip()]
|
|
841
|
+
unknown = [s for s in chosen if s not in ALL_SOURCES]
|
|
842
|
+
if unknown:
|
|
843
|
+
raise ChannelFootprintError(f"unknown sources: {', '.join(unknown)} (known: {', '.join(ALL_SOURCES)})")
|
|
844
|
+
return [s for s in ALL_SOURCES if s in chosen]
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
def domains_from_competitor_set(workspace: Path, cap: int) -> list[str]:
|
|
848
|
+
evidence = _load_script("research-evidence.py", "pmr_evidence_for_channel_footprint")
|
|
849
|
+
return evidence.competitor_set_inputs(workspace, consumer=SECTION_ID, cap=cap)
|
|
850
|
+
|
|
851
|
+
|
|
852
|
+
def domains_from_keyword_evidence(workspace: Path, cap: int, exclude=frozenset()) -> list[str]:
|
|
853
|
+
evidence = _load_script("research-evidence.py", "pmr_evidence_for_channel_footprint_kwe")
|
|
854
|
+
return evidence.keyword_inputs(workspace, consumer=SECTION_ID, row_type="competitor_domain", field="domain", cap=cap, exclude=exclude)
|
|
855
|
+
|
|
856
|
+
|
|
857
|
+
def domains_excluded_by_competitor_set(workspace: Path) -> set[str]:
|
|
858
|
+
"""Domains the completed Competitor Set rejected; never refilled automatically."""
|
|
859
|
+
evidence = _load_script("research-evidence.py", "pmr_evidence_for_channel_footprint_excl")
|
|
860
|
+
return evidence.competitor_set_excluded_domains(workspace)
|
|
861
|
+
|
|
862
|
+
|
|
863
|
+
@dataclass
|
|
864
|
+
class RunInputs:
|
|
865
|
+
domains: list[str]
|
|
866
|
+
domain_sources: dict[str, str]
|
|
867
|
+
product_domain: str
|
|
868
|
+
sources: list[str]
|
|
869
|
+
country: str
|
|
870
|
+
eu_country: str
|
|
871
|
+
pixel_actor_fallback: bool
|
|
872
|
+
target_countries: list[str] = field(default_factory=list)
|
|
873
|
+
competitor_context: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
874
|
+
product_context_sha256: str = ""
|
|
875
|
+
effective_policy: dict[str, Any] = field(default_factory=dict)
|
|
876
|
+
relevance_decisions: dict[str, Any] = field(default_factory=dict)
|
|
877
|
+
stage: str = "all"
|
|
878
|
+
# Sample expansions recorded by the previous compatible collect.
|
|
879
|
+
prior_expansions: list[dict[str, Any]] = field(default_factory=list)
|
|
880
|
+
|
|
881
|
+
|
|
882
|
+
# ── helpers ─────────────────────────────────────────────────────────────────
|
|
883
|
+
|
|
884
|
+
def _days_between(start_iso: str | None, end_iso: str | None) -> int | None:
|
|
885
|
+
try:
|
|
886
|
+
start = date.fromisoformat(str(start_iso)[:10])
|
|
887
|
+
end = date.fromisoformat(str(end_iso)[:10])
|
|
888
|
+
except (ValueError, TypeError):
|
|
889
|
+
return None
|
|
890
|
+
return max(0, (end - start).days)
|
|
891
|
+
|
|
892
|
+
|
|
893
|
+
def _iso_from_epoch(value: Any) -> str | None:
|
|
894
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
895
|
+
if isinstance(value, str) and value.strip():
|
|
896
|
+
digits = value.strip()
|
|
897
|
+
if digits.isdigit():
|
|
898
|
+
value = int(digits)
|
|
899
|
+
else:
|
|
900
|
+
return digits[:10]
|
|
901
|
+
else:
|
|
902
|
+
return None
|
|
903
|
+
try:
|
|
904
|
+
seconds = float(value)
|
|
905
|
+
if seconds > 10_000_000_000:
|
|
906
|
+
seconds /= 1000.0
|
|
907
|
+
return datetime.fromtimestamp(seconds, tz=timezone.utc).date().isoformat()
|
|
908
|
+
except (OverflowError, OSError, ValueError):
|
|
909
|
+
return None
|
|
910
|
+
|
|
911
|
+
|
|
912
|
+
_GENERIC_NAME_TOKENS = {
|
|
913
|
+
"inc", "llc", "ltd", "co", "corp", "corporation", "gmbh", "app", "apps", "software", "hq", "official",
|
|
914
|
+
"team", "teams", "io", "ai", "com", "labs", "technologies", "technology", "tech", "the", "limited", "plc", "sas", "srl",
|
|
915
|
+
# product-tier pages a brand runs under its own name
|
|
916
|
+
"pro", "plus", "premium", "business", "enterprise", "for", "education", "edu", "learn", "academy",
|
|
917
|
+
}
|
|
918
|
+
# Handle prefixes platforms put in front of an advertiser name.
|
|
919
|
+
_NAME_PREFIX_RE = re.compile(r"^\s*(?:u/|r/|@)+", re.IGNORECASE)
|
|
920
|
+
|
|
921
|
+
|
|
922
|
+
def _name_matches(name: Any, brand: str) -> bool:
|
|
923
|
+
"""Advertiser identity: the name IS the brand (plus generic corporate or
|
|
924
|
+
product-tier suffixes), not merely a name that contains the brand token:
|
|
925
|
+
'Mural Painting Co' is a namesake of mural.co, 'Lucid Software' and
|
|
926
|
+
'Codecademy Pro' are the brands."""
|
|
927
|
+
if not brand:
|
|
928
|
+
return False
|
|
929
|
+
tokens = [t for t in re.split(r"[^a-z0-9]+", _NAME_PREFIX_RE.sub("", str(name or "")).lower()) if t]
|
|
930
|
+
if not tokens:
|
|
931
|
+
return False
|
|
932
|
+
joined = "".join(tokens)
|
|
933
|
+
if joined == brand:
|
|
934
|
+
return True
|
|
935
|
+
for size in range(1, len(tokens) + 1):
|
|
936
|
+
if "".join(tokens[:size]) == brand:
|
|
937
|
+
return all(t in _GENERIC_NAME_TOKENS for t in tokens[size:])
|
|
938
|
+
if len("".join(tokens[:size])) > len(brand):
|
|
939
|
+
break
|
|
940
|
+
return False
|
|
941
|
+
|
|
942
|
+
|
|
943
|
+
def _mentions(text: Any, brand: str) -> bool:
|
|
944
|
+
"""Free text (a search result title or address) that names the brand."""
|
|
945
|
+
return bool(brand) and brand in re.sub(r"[^a-z0-9]", "", str(text or "").lower())
|
|
946
|
+
|
|
947
|
+
|
|
948
|
+
def _chunks(items: list[str], size: int) -> list[list[str]]:
|
|
949
|
+
size = max(1, int(size))
|
|
950
|
+
return [items[i:i + size] for i in range(0, len(items), size)]
|
|
951
|
+
|
|
952
|
+
|
|
953
|
+
def _transient_status(status: Any) -> bool:
|
|
954
|
+
return isinstance(status, int) and not isinstance(status, bool) and (status == 429 or status >= 500)
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
def _provider_detail(exc: BaseException) -> str:
|
|
958
|
+
"""The provider's message without the per-attempt counter that
|
|
959
|
+
`_call_with_context` appends, so evidence stays byte-stable on replay.
|
|
960
|
+
Only our own wrapper is unwrapped; a transport error's cause (urllib)
|
|
961
|
+
would lose the gateway's message."""
|
|
962
|
+
inner = exc.__cause__ if isinstance(exc.__cause__, ChannelFootprintError) else exc
|
|
963
|
+
return str(inner).split(" — this attempt had already started", 1)[0][:240]
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
def _coverage_gap(coverage: list[dict[str, Any]], source: str, domains: list[str], detail: str, status: str = "unavailable") -> None:
|
|
967
|
+
coverage.extend({"type": "coverage", "domain": d, "source": source, "status": status, "detail": detail[:240], "tier": "C"} for d in domains)
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def _attempt(client: OsClient, coverage: list[dict[str, Any]], source: str, domains: list[str], fn: Callable[[], Any]) -> Any:
|
|
971
|
+
"""One supplementary provider call. A failure other than budget
|
|
972
|
+
exhaustion becomes a coverage gap for THESE domains only; results already
|
|
973
|
+
collected for other domains stay and later domains are still attempted.
|
|
974
|
+
After SOURCE_BREAKER_FAILURES consecutive failures of the same source the
|
|
975
|
+
remaining domains are skipped without starting more paid runs. Budget
|
|
976
|
+
exhaustion and an unreachable OS always propagate so a rerun resumes from
|
|
977
|
+
the cache."""
|
|
978
|
+
failures = client.source_failures.get(source, 0)
|
|
979
|
+
if failures >= SOURCE_BREAKER_FAILURES:
|
|
980
|
+
_coverage_gap(coverage, source, domains, f"skipped after {failures} consecutive {source} failures in this attempt; rerun to retry")
|
|
981
|
+
return None
|
|
982
|
+
hits_before = client.cache_hits
|
|
983
|
+
try:
|
|
984
|
+
result = fn()
|
|
985
|
+
except BudgetExhausted as exc:
|
|
986
|
+
if "cumulative" not in str(exc):
|
|
987
|
+
raise
|
|
988
|
+
_coverage_gap(coverage, source, domains, str(exc), status="budget_deferred")
|
|
989
|
+
return None
|
|
990
|
+
except OsUnreachable:
|
|
991
|
+
raise
|
|
992
|
+
except ChannelFootprintError as exc:
|
|
993
|
+
# Only a LIVE failure feeds the breaker: a rejection replayed from the
|
|
994
|
+
# cache costs nothing, and counting it would skip the remaining
|
|
995
|
+
# domains on every rerun forever.
|
|
996
|
+
if client.cache_hits == hits_before:
|
|
997
|
+
client.source_failures[source] = failures + 1
|
|
998
|
+
_coverage_gap(coverage, source, domains, _provider_detail(exc))
|
|
999
|
+
return None
|
|
1000
|
+
client.source_failures[source] = 0
|
|
1001
|
+
return result
|
|
1002
|
+
|
|
1003
|
+
|
|
1004
|
+
def _call_with_context(client: OsClient, subject: str, label: str, fn: Callable[[], Any]) -> Any:
|
|
1005
|
+
try:
|
|
1006
|
+
return fn()
|
|
1007
|
+
except ChannelFootprintError as exc:
|
|
1008
|
+
# Keep the subclass: a cached provider rejection stays recognisable.
|
|
1009
|
+
raise type(exc)(
|
|
1010
|
+
f"{label} for `{subject}` failed: {exc} — this attempt had already started "
|
|
1011
|
+
f"{client.actor_runs}/{client.max_actor_runs} actor run(s); see GET /me/limits?resource=apify:runs "
|
|
1012
|
+
"for remaining daily quota."
|
|
1013
|
+
) from exc
|
|
1014
|
+
|
|
1015
|
+
|
|
1016
|
+
def _cumulative_answer(coverage: list[dict[str, Any]] | None, source: str, domains: list[str], country: str | None, fn: Callable[[], Any]) -> Any:
|
|
1017
|
+
"""Catch total-cap exhaustion at the smallest provider request unit.
|
|
1018
|
+
|
|
1019
|
+
Completed chunks stay in the collector's result. Attempt exhaustion
|
|
1020
|
+
still stops and resumes from the provider cache.
|
|
1021
|
+
"""
|
|
1022
|
+
try:
|
|
1023
|
+
return fn()
|
|
1024
|
+
except BudgetExhausted as exc:
|
|
1025
|
+
if "cumulative" not in str(exc):
|
|
1026
|
+
raise
|
|
1027
|
+
if coverage is not None:
|
|
1028
|
+
coverage.extend({"type": "coverage", "domain": d, "source": source, "country": country, "status": "budget_deferred", "detail": str(exc), "tier": "C"} for d in domains)
|
|
1029
|
+
return None
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
def usable_destination(value: Any, _depth: int = 0) -> str:
|
|
1033
|
+
"""The registrable domain of an ad destination when the value is really
|
|
1034
|
+
an address (not display text such as 'Learn more') and not the platform
|
|
1035
|
+
itself, a messaging or short host, a store, a link hub or a shortener;
|
|
1036
|
+
otherwise '' so the advertiser name decides. Click-tracking wrappers
|
|
1037
|
+
(l.facebook.com/l.php?u=...) are unwrapped to the address they carry."""
|
|
1038
|
+
text = str(value or "").strip()
|
|
1039
|
+
if not text or " " in text or "." not in text:
|
|
1040
|
+
return ""
|
|
1041
|
+
lowered = text.lower()
|
|
1042
|
+
host = lowered
|
|
1043
|
+
query = ""
|
|
1044
|
+
if "//" in lowered:
|
|
1045
|
+
parsed = urllib.parse.urlparse(lowered)
|
|
1046
|
+
host = parsed.hostname or ""
|
|
1047
|
+
query = parsed.query
|
|
1048
|
+
host = host.split("/", 1)[0].split("?", 1)[0].removeprefix("www.").rstrip(".")
|
|
1049
|
+
if host.split(":", 1)[0] in REDIRECT_WRAPPER_HOSTS or ("www." + host) in REDIRECT_WRAPPER_HOSTS:
|
|
1050
|
+
wrapped = urllib.parse.parse_qs(query).get("u") or urllib.parse.parse_qs(query).get("q") or []
|
|
1051
|
+
if wrapped and _depth < 2:
|
|
1052
|
+
return usable_destination(urllib.parse.unquote(wrapped[0]), _depth + 1)
|
|
1053
|
+
return ""
|
|
1054
|
+
domain = registrable_domain(text)
|
|
1055
|
+
if not domain or not _DOMAIN_RE.match(domain):
|
|
1056
|
+
return ""
|
|
1057
|
+
if domain in NEUTRAL_DESTINATION_HOSTS or host.split(":", 1)[0] in NEUTRAL_DESTINATION_HOSTS:
|
|
1058
|
+
return ""
|
|
1059
|
+
return domain
|
|
1060
|
+
|
|
1061
|
+
|
|
1062
|
+
def meta_search_url(brand: str, country: str, active_status: str = "all") -> str:
|
|
1063
|
+
"""The public Ad Library page search. `all` also lists stopped ads that
|
|
1064
|
+
Meta still keeps (observed 2026-10-02: 58 of 71 US results for four
|
|
1065
|
+
global brands were inactive), so a sample of 30 is mostly old ads;
|
|
1066
|
+
`active` lists running ads only."""
|
|
1067
|
+
if active_status not in ("all", "active"):
|
|
1068
|
+
raise ValueError(f"unsupported Meta active_status {active_status!r}")
|
|
1069
|
+
return (
|
|
1070
|
+
f"https://www.facebook.com/ads/library/?active_status={active_status}&ad_type=all"
|
|
1071
|
+
f"&country={urllib.parse.quote(country)}&q=%22{urllib.parse.quote(brand)}%22&search_type=keyword_unordered&media_type=all"
|
|
1072
|
+
)
|
|
1073
|
+
|
|
1074
|
+
|
|
1075
|
+
# ── pixels ──────────────────────────────────────────────────────────────────
|
|
1076
|
+
|
|
1077
|
+
def detect_tags(blobs: list[str]) -> list[dict[str, str]]:
|
|
1078
|
+
"""Match the versioned signature list over page HTML and container JS."""
|
|
1079
|
+
text = "\n".join(html_lib.unescape(b or "") for b in blobs)
|
|
1080
|
+
found: list[dict[str, str]] = []
|
|
1081
|
+
for tag, channel, pattern in TAG_SIGNATURES:
|
|
1082
|
+
if re.search(pattern, text, re.IGNORECASE):
|
|
1083
|
+
found.append({"tag": tag, "channel": channel})
|
|
1084
|
+
return found
|
|
1085
|
+
|
|
1086
|
+
|
|
1087
|
+
def detect_tag_managers(blobs: list[str]) -> list[str]:
|
|
1088
|
+
text = "\n".join(html_lib.unescape(b or "") for b in blobs)
|
|
1089
|
+
return [name for name, pattern in TAG_MANAGER_SIGNATURES if re.search(pattern, text, re.IGNORECASE)]
|
|
1090
|
+
|
|
1091
|
+
|
|
1092
|
+
def linkedin_company_slug(body: str) -> str | None:
|
|
1093
|
+
"""The competitor's own LinkedIn page link (footer, schema markup)."""
|
|
1094
|
+
for match in LINKEDIN_COMPANY_RE.finditer(html_lib.unescape(body or "")):
|
|
1095
|
+
slug = match.group(1).strip("/").lower()
|
|
1096
|
+
if slug and slug not in {"company", "school", "showcase", "login", "signup"}:
|
|
1097
|
+
return slug
|
|
1098
|
+
return None
|
|
1099
|
+
|
|
1100
|
+
|
|
1101
|
+
def scan_site(client: OsClient, domain: str) -> dict[str, Any]:
|
|
1102
|
+
"""Home page plus up to three Google Tag Manager containers."""
|
|
1103
|
+
page, _ = client.fetch_page(f"https://{domain}/")
|
|
1104
|
+
status = page.get("status")
|
|
1105
|
+
body = str(page.get("body") or "")
|
|
1106
|
+
if status is None:
|
|
1107
|
+
fetch_status = "error"
|
|
1108
|
+
elif int(status) >= 400 or (int(status) < 300 and not body.strip()):
|
|
1109
|
+
fetch_status = "blocked" if int(status) in (401, 403, 429, 503) else "error"
|
|
1110
|
+
else:
|
|
1111
|
+
fetch_status = "ok"
|
|
1112
|
+
containers = list(dict.fromkeys(GTM_ID_RE.findall(body)))[:GTM_CONTAINERS_PER_SITE] if fetch_status == "ok" else []
|
|
1113
|
+
blobs = [body]
|
|
1114
|
+
for container in containers:
|
|
1115
|
+
gtm, _ = client.fetch_page(f"https://www.googletagmanager.com/gtm.js?id={container}")
|
|
1116
|
+
blobs.append(str(gtm.get("body") or ""))
|
|
1117
|
+
tags = detect_tags(blobs) if fetch_status == "ok" else []
|
|
1118
|
+
managers = detect_tag_managers(blobs) if fetch_status == "ok" else []
|
|
1119
|
+
return {
|
|
1120
|
+
"type": "pixel_scan",
|
|
1121
|
+
"domain": domain,
|
|
1122
|
+
"fetch_status": fetch_status,
|
|
1123
|
+
"http_status": status,
|
|
1124
|
+
"via": "html",
|
|
1125
|
+
"gtm_containers": containers,
|
|
1126
|
+
"tag_managers": managers,
|
|
1127
|
+
"tags_detected": [t["tag"] for t in tags],
|
|
1128
|
+
"channels_detected": sorted({t["channel"] for t in tags}),
|
|
1129
|
+
"linkedin_company": linkedin_company_slug(body) if fetch_status == "ok" else None,
|
|
1130
|
+
"tier": "A" if fetch_status == "ok" else "C",
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
|
|
1134
|
+
def scan_sites_with_actor(client: OsClient, actor: str, domains: list[str]) -> dict[str, dict[str, Any]]:
|
|
1135
|
+
"""Headless fallback for sites whose HTML cannot be fetched directly."""
|
|
1136
|
+
rows, _ = _call_with_context(client, ", ".join(domains), "pixel detector actor", lambda: client.run_actor(
|
|
1137
|
+
actor, {"startUrls": [f"https://{d}/" for d in domains], "crawlPages": False, "maxPages": 1}, limit=len(domains) * 2,
|
|
1138
|
+
))
|
|
1139
|
+
scans: dict[str, dict[str, Any]] = {}
|
|
1140
|
+
for row in rows:
|
|
1141
|
+
if row.get("type") not in (None, "page"):
|
|
1142
|
+
continue
|
|
1143
|
+
domain = registrable_domain(str(row.get("url") or ""))
|
|
1144
|
+
if domain not in domains:
|
|
1145
|
+
continue
|
|
1146
|
+
tools = [t for t in (row.get("tools") or []) if isinstance(t, dict) and t.get("detected", True)]
|
|
1147
|
+
names = [str(t.get("tool") or "") for t in tools]
|
|
1148
|
+
tags = detect_tags([" ".join(names + [json.dumps(t.get("details") or {}) for t in tools])])
|
|
1149
|
+
# Tool names from the detector are mapped through the same channel
|
|
1150
|
+
# vocabulary by keyword; unknown tools are kept as names only.
|
|
1151
|
+
for name in names:
|
|
1152
|
+
lowered = name.lower()
|
|
1153
|
+
for keyword, tag, channel in (
|
|
1154
|
+
("facebook pixel", "meta_pixel", "meta_ads"), ("meta pixel", "meta_pixel", "meta_ads"),
|
|
1155
|
+
("google ads", "google_ads_tag", "google_ads"), ("reddit", "reddit_pixel", "reddit_ads"),
|
|
1156
|
+
("linkedin insight", "linkedin_insight", "linkedin_ads"), ("tiktok", "tiktok_pixel", "tiktok_ads"),
|
|
1157
|
+
("bing", "microsoft_uet", "microsoft_ads"), ("microsoft", "microsoft_uet", "microsoft_ads"),
|
|
1158
|
+
("twitter", "x_pixel", "x_ads"), ("pinterest", "pinterest_tag", "pinterest_ads"),
|
|
1159
|
+
("snap", "snap_pixel", "snap_ads"), ("quora", "quora_pixel", "quora_ads"),
|
|
1160
|
+
("taboola", "taboola", "display_native"), ("outbrain", "outbrain", "display_native"),
|
|
1161
|
+
("criteo", "criteo", "display_native"), ("capterra", "capterra_tracker", "review_marketplace"),
|
|
1162
|
+
("partnerstack", "partnerstack", "affiliate"), ("impact", "impact", "affiliate"),
|
|
1163
|
+
("rewardful", "rewardful", "affiliate"), ("firstpromoter", "firstpromoter", "affiliate"),
|
|
1164
|
+
):
|
|
1165
|
+
if keyword in lowered and tag not in {t["tag"] for t in tags}:
|
|
1166
|
+
tags.append({"tag": tag, "channel": channel})
|
|
1167
|
+
scans[domain] = {
|
|
1168
|
+
"type": "pixel_scan",
|
|
1169
|
+
"domain": domain,
|
|
1170
|
+
"fetch_status": "ok",
|
|
1171
|
+
"http_status": row.get("httpStatus"),
|
|
1172
|
+
"via": "actor",
|
|
1173
|
+
"gtm_containers": [],
|
|
1174
|
+
"tag_managers": [n.lower().replace(" ", "_") for n in names if any(k in n.lower() for k in ("tag manager", "segment", "tealium", "adobe launch"))][:4],
|
|
1175
|
+
"tags_detected": [t["tag"] for t in tags],
|
|
1176
|
+
"channels_detected": sorted({t["channel"] for t in tags}),
|
|
1177
|
+
"linkedin_company": None,
|
|
1178
|
+
"tools_reported": names[:40],
|
|
1179
|
+
"tier": "A",
|
|
1180
|
+
}
|
|
1181
|
+
return scans
|
|
1182
|
+
|
|
1183
|
+
|
|
1184
|
+
# ── page history (Wayback CDX) ──────────────────────────────────────────────
|
|
1185
|
+
|
|
1186
|
+
def _cdx_rows(response: dict[str, Any]) -> list[list[str]]:
|
|
1187
|
+
try:
|
|
1188
|
+
value = json.loads(str(response.get("body") or "[]"))
|
|
1189
|
+
except json.JSONDecodeError:
|
|
1190
|
+
return []
|
|
1191
|
+
return [r for r in value[1:] if isinstance(r, list)] if isinstance(value, list) and value else []
|
|
1192
|
+
|
|
1193
|
+
|
|
1194
|
+
def collect_history(client: OsClient, domain: str, today: date) -> dict[str, Any]:
|
|
1195
|
+
"""Distinct home-page versions in the window (CDX `collapse=digest`,
|
|
1196
|
+
so a changed page counts once and an unchanged monthly crawl does not)
|
|
1197
|
+
plus the first archived date. A home page that only redirects to `www`
|
|
1198
|
+
has no 200 rows of its own, so `www.` is tried when the bare host is empty."""
|
|
1199
|
+
since = (today - timedelta(days=30 * HISTORY_MONTHS)).strftime("%Y%m")
|
|
1200
|
+
|
|
1201
|
+
def fetch(host: str, *, recent: bool) -> dict[str, Any]:
|
|
1202
|
+
target = urllib.parse.quote(host + "/", safe="")
|
|
1203
|
+
url = (
|
|
1204
|
+
f"{WAYBACK_CDX_URL}?url={target}&output=json&fl=timestamp,digest&filter=statuscode:200"
|
|
1205
|
+
f"&collapse=digest&from={since}&limit={CDX_ROW_LIMIT}"
|
|
1206
|
+
if recent
|
|
1207
|
+
else f"{WAYBACK_CDX_URL}?url={target}&output=json&fl=timestamp&filter=statuscode:200&limit=1"
|
|
1208
|
+
)
|
|
1209
|
+
before = client.cache_hits
|
|
1210
|
+
response, _ = client.fetch_page(url, budget_attr="cdx_calls", budget_max=client.max_cdx_calls, label="Wayback CDX")
|
|
1211
|
+
if client.cache_hits == before:
|
|
1212
|
+
# The archive rate-limits bursts; space out live calls.
|
|
1213
|
+
(client.sleep or time.sleep)(CDX_PAUSE_SECONDS)
|
|
1214
|
+
return response
|
|
1215
|
+
|
|
1216
|
+
host = domain
|
|
1217
|
+
recent = fetch(host, recent=True)
|
|
1218
|
+
if recent.get("status") == 200 and not _cdx_rows(recent):
|
|
1219
|
+
host = f"www.{domain}"
|
|
1220
|
+
recent = fetch(host, recent=True)
|
|
1221
|
+
first = fetch(host, recent=False)
|
|
1222
|
+
rows = _cdx_rows(recent)
|
|
1223
|
+
versions = {str(r[1]) for r in rows if len(r) > 1}
|
|
1224
|
+
months = {str(r[0])[:6] for r in rows if r}
|
|
1225
|
+
first_rows = _cdx_rows(first)
|
|
1226
|
+
first_ts = str(first_rows[0][0]) if first_rows and first_rows[0] else None
|
|
1227
|
+
available = recent.get("status") == 200 and first.get("status") == 200
|
|
1228
|
+
return {
|
|
1229
|
+
"type": "page_history",
|
|
1230
|
+
"domain": domain,
|
|
1231
|
+
"source": "wayback",
|
|
1232
|
+
"status": "ok" if available else "unavailable",
|
|
1233
|
+
"detail": "" if available else str(recent.get("error") or first.get("error") or f"HTTP {recent.get('status')}/{first.get('status')}")[:160],
|
|
1234
|
+
"archived_host": host,
|
|
1235
|
+
"first_archived": f"{first_ts[:4]}-{first_ts[4:6]}-{first_ts[6:8]}" if first_ts and len(first_ts) >= 8 else None,
|
|
1236
|
+
"months_archived_24m": len(months) if available else None,
|
|
1237
|
+
# Dynamic pages change their digest on every crawl; at the row cap
|
|
1238
|
+
# the count is a floor and the page is simply "dynamic".
|
|
1239
|
+
"distinct_versions_24m": len(versions) if available else None,
|
|
1240
|
+
"versions_capped": available and len(rows) >= CDX_ROW_LIMIT,
|
|
1241
|
+
"window_months": HISTORY_MONTHS,
|
|
1242
|
+
"tier": "B" if available else "C",
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
|
|
1246
|
+
# ── traffic mix ─────────────────────────────────────────────────────────────
|
|
1247
|
+
|
|
1248
|
+
def _share_rows(domain: str, visits: Any, shares: Any, source: str) -> list[dict[str, Any]]:
|
|
1249
|
+
rows: list[dict[str, Any]] = []
|
|
1250
|
+
if not isinstance(shares, dict):
|
|
1251
|
+
return rows
|
|
1252
|
+
visits_value = None
|
|
1253
|
+
if isinstance(visits, (int, float)) and not isinstance(visits, bool):
|
|
1254
|
+
visits_value = int(visits)
|
|
1255
|
+
elif isinstance(visits, list) and visits:
|
|
1256
|
+
last = visits[-1]
|
|
1257
|
+
if isinstance(last, dict):
|
|
1258
|
+
last = last.get("visits") or last.get("value")
|
|
1259
|
+
if isinstance(last, (int, float)) and not isinstance(last, bool):
|
|
1260
|
+
visits_value = int(last)
|
|
1261
|
+
for key, value in shares.items():
|
|
1262
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
1263
|
+
continue
|
|
1264
|
+
share = float(value) * (100.0 if float(value) <= 1.0 else 1.0)
|
|
1265
|
+
channel_key = re.sub(r"[^a-z]", "_", str(key).lower()).strip("_")
|
|
1266
|
+
rows.append({
|
|
1267
|
+
"type": "traffic_mix",
|
|
1268
|
+
"domain": domain,
|
|
1269
|
+
"channel": channel_key,
|
|
1270
|
+
"paid_channel": TRAFFIC_CHANNEL_MAP.get(channel_key),
|
|
1271
|
+
"share_pct": round(share, 2),
|
|
1272
|
+
"share_pct_raw": share,
|
|
1273
|
+
"visits_monthly": visits_value,
|
|
1274
|
+
"source": source,
|
|
1275
|
+
"market_scope": "global",
|
|
1276
|
+
"tier": "B",
|
|
1277
|
+
})
|
|
1278
|
+
return rows
|
|
1279
|
+
|
|
1280
|
+
|
|
1281
|
+
def collect_traffic(client: OsClient, spec: dict[str, Any], domains: list[str], coverage: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
1282
|
+
traffic: list[dict[str, Any]] = []
|
|
1283
|
+
for chunk in _chunks(domains, int(spec.get("batch_domains") or 10)):
|
|
1284
|
+
outcome = _attempt(client, coverage, "traffic", chunk, lambda: _call_with_context(client, ", ".join(chunk), "Similarweb actor", lambda: client.run_actor(
|
|
1285
|
+
spec["actor"], {"startUrls": [{"url": f"https://{d}"} for d in chunk]}, limit=len(chunk) * 2,
|
|
1286
|
+
)))
|
|
1287
|
+
if outcome is None:
|
|
1288
|
+
continue
|
|
1289
|
+
rows, _ = outcome
|
|
1290
|
+
traffic.extend(_traffic_rows(rows, chunk, coverage))
|
|
1291
|
+
return traffic
|
|
1292
|
+
|
|
1293
|
+
|
|
1294
|
+
def _traffic_rows(rows: list[dict[str, Any]], domains: list[str], coverage: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
1295
|
+
traffic: list[dict[str, Any]] = []
|
|
1296
|
+
seen: set[str] = set()
|
|
1297
|
+
for row in rows:
|
|
1298
|
+
# Live shape (2026-10-02): {meta: {website, request_parameter},
|
|
1299
|
+
# traffic: {history: [{month, visits}], sources_share: {...}}};
|
|
1300
|
+
# flat {domain, visits, sources_share} is accepted as well.
|
|
1301
|
+
meta = row.get("meta") if isinstance(row.get("meta"), dict) else {}
|
|
1302
|
+
traffic_block = row.get("traffic") if isinstance(row.get("traffic"), dict) else {}
|
|
1303
|
+
domain = registrable_domain(str(meta.get("website") or meta.get("request_parameter") or row.get("domain") or row.get("url") or row.get("site") or ""))
|
|
1304
|
+
if domain not in domains or domain in seen:
|
|
1305
|
+
continue
|
|
1306
|
+
seen.add(domain)
|
|
1307
|
+
shares = traffic_block.get("sources_share") or row.get("sources_share") or row.get("traffic_sources") or row.get("sourcesShare") or {}
|
|
1308
|
+
visits = traffic_block.get("history") or row.get("visits")
|
|
1309
|
+
traffic.extend(_share_rows(domain, visits, shares, "similarweb_actor"))
|
|
1310
|
+
coverage.extend(
|
|
1311
|
+
{"type": "coverage", "domain": d, "source": "traffic", "status": "ok" if d in seen else "no_data", "detail": "" if d in seen else "no public Similarweb estimate (small or unlisted site)", "tier": "C"}
|
|
1312
|
+
for d in domains
|
|
1313
|
+
)
|
|
1314
|
+
return traffic
|
|
1315
|
+
|
|
1316
|
+
|
|
1317
|
+
# ── ad libraries ────────────────────────────────────────────────────────────
|
|
1318
|
+
|
|
1319
|
+
def _model() -> ModuleType:
|
|
1320
|
+
return _load_script("channel-footprint-model.py", "pmr_channel_footprint_model")
|
|
1321
|
+
|
|
1322
|
+
|
|
1323
|
+
def _presence(domain: str, platform: str, source: str, ads: list[dict[str, Any]], *, market_scope: str, verified_by: str, today: date, note: str = "", country: str | None = None, request_limit: int | None = None, raw_count: int | None = None, native_date_scope: str = "global", native_activity_scope: str = "global", retrieved_on: str | None = None) -> dict[str, Any]:
|
|
1324
|
+
"""Raw source observation; eligibility is decided after product relevance."""
|
|
1325
|
+
model = _model()
|
|
1326
|
+
policy, _ = _load_script("research-policy.py", "pmr_channel_policy").load_policy()
|
|
1327
|
+
observed = min(today.isoformat(), retrieved_on) if retrieved_on else today.isoformat()
|
|
1328
|
+
observations = [model.observation(a, source, country, market_scope, date.fromisoformat(observed), policy, native_date_scope, native_activity_scope) for a in ads]
|
|
1329
|
+
durations = [o["served_days"] for o in observations if o["served_days"] is not None]
|
|
1330
|
+
formats: dict[str, int] = {}
|
|
1331
|
+
formats_recent: dict[str, int] = {}
|
|
1332
|
+
for ad, obs in zip(ads, observations):
|
|
1333
|
+
fmt = str(ad.get("format") or "unknown").lower()
|
|
1334
|
+
formats[fmt] = formats.get(fmt, 0) + 1
|
|
1335
|
+
if obs["status"] in {"active", "recent"}:
|
|
1336
|
+
formats_recent[fmt] = formats_recent.get(fmt, 0) + 1
|
|
1337
|
+
received = len(ads) if raw_count is None else raw_count
|
|
1338
|
+
capped = bool(request_limit and received >= request_limit)
|
|
1339
|
+
return {"type": "library_presence", "domain": domain, "platform": platform, "source": source,
|
|
1340
|
+
"matched_ads": len(ads), "active_ads": sum(o["status"] in {"active", "recent"} for o in observations),
|
|
1341
|
+
"oldest_days": max(durations) if durations else None,
|
|
1342
|
+
"latest_seen": max((o["last_shown"] for o in observations if o["last_shown"]), default=None),
|
|
1343
|
+
"formats": formats, "formats_recent": formats_recent, "present": bool(ads), "market_scope": market_scope,
|
|
1344
|
+
"verified_by": verified_by, "note": note, "tier": "A" if ads else "B",
|
|
1345
|
+
"country": country, "countries": [country] if country else [], "native_date_scope": native_date_scope,
|
|
1346
|
+
"native_activity_scope": native_activity_scope, "observed_on": observed, "source_retrieved_on": retrieved_on or observed, "ads": ads,
|
|
1347
|
+
"status": "observed" if ads else ("not_verifiable" if source == "reddit_inspiration_actor" else "no_matches_in_sample"),
|
|
1348
|
+
"sampling": {"request_limit": request_limit, "raw_received": received, "identity_matched": len(ads),
|
|
1349
|
+
"limit_reached": capped, "complete": False if capped else None, "pagination_complete": None,
|
|
1350
|
+
"total_ads": None, "total_lower_bound": received if capped else None}}
|
|
1351
|
+
|
|
1352
|
+
|
|
1353
|
+
def collect_google(client: OsClient, spec: dict[str, Any], domains: list[str], today: date, country: str | None = None, coverage: list[dict[str, Any]] | None = None) -> list[dict[str, Any]]:
|
|
1354
|
+
per_domain = int(spec.get("ads_per_domain") or 40)
|
|
1355
|
+
results: list[dict[str, Any]] = []
|
|
1356
|
+
for chunk in _chunks(domains, int(spec.get("batch_domains") or 5)):
|
|
1357
|
+
outcome = _cumulative_answer(coverage, "google", chunk, country, lambda: _call_with_context(client, ", ".join(chunk), "Google Ads Transparency", lambda: client.run_actor(
|
|
1358
|
+
spec["actor"], {"domains": chunk, "resultType": "ads", "region": country or "ANYWHERE", "maxAdsPerSearch": per_domain}, limit=per_domain * len(chunk),
|
|
1359
|
+
)))
|
|
1360
|
+
if outcome is None:
|
|
1361
|
+
continue
|
|
1362
|
+
rows, retrieved_on = outcome
|
|
1363
|
+
by_domain: dict[str, list[dict[str, Any]]] = {d: [] for d in chunk}
|
|
1364
|
+
for row in rows:
|
|
1365
|
+
destination = registrable_domain(str(row.get("domain") or ""))
|
|
1366
|
+
if destination not in by_domain:
|
|
1367
|
+
continue
|
|
1368
|
+
first = str(row.get("firstShown") or "")[:10] or None
|
|
1369
|
+
last = str(row.get("lastShown") or "")[:10] or None
|
|
1370
|
+
by_domain[destination].append({"first_seen": first, "last_seen": last, "is_active": None, "format": row.get("format"), "raw": row})
|
|
1371
|
+
for domain in chunk:
|
|
1372
|
+
results.append(_presence(domain, "google_ads", "google_transparency_actor", by_domain[domain], market_scope="target" if country else "global", verified_by="destination", today=today, country=country, request_limit=per_domain, raw_count=sum(registrable_domain(str(r.get("domain") or "")) == domain for r in rows), retrieved_on=retrieved_on))
|
|
1373
|
+
return results
|
|
1374
|
+
|
|
1375
|
+
|
|
1376
|
+
def collect_meta(client: OsClient, spec: dict[str, Any], domains: list[str], today: date, country: str, eu_country: str, coverage: list[dict[str, Any]] | None = None) -> list[dict[str, Any]]:
|
|
1377
|
+
per_domain = int(spec.get("ads_per_domain") or 30)
|
|
1378
|
+
results: list[dict[str, Any]] = []
|
|
1379
|
+
for chunk in _chunks(domains, int(spec.get("batch_domains") or 5)):
|
|
1380
|
+
brands = {d: brand_token(d) for d in chunk}
|
|
1381
|
+
outcome = _cumulative_answer(coverage, "meta", chunk, country, lambda: _call_with_context(client, ", ".join(chunk), "Meta Ads Library (page search actor)", lambda: client.run_actor(
|
|
1382
|
+
spec["actor"], {"urls": [{"url": meta_search_url(brands[d], country)} for d in chunk], "count": per_domain}, limit=per_domain * len(chunk),
|
|
1383
|
+
)))
|
|
1384
|
+
if outcome is None:
|
|
1385
|
+
continue
|
|
1386
|
+
rows, retrieved_on = outcome
|
|
1387
|
+
by_domain: dict[str, list[dict[str, Any]]] = {d: [] for d in chunk}
|
|
1388
|
+
for row in rows:
|
|
1389
|
+
page = str(row.get("page_name") or row.get("pageName") or "")
|
|
1390
|
+
snapshot = row.get("snapshot") or {}
|
|
1391
|
+
# The display caption is the second chance when the click link is
|
|
1392
|
+
# a wrapper or short host that hides the destination.
|
|
1393
|
+
link_domain = (usable_destination(snapshot.get("link_url")) or usable_destination(snapshot.get("caption"))) if isinstance(snapshot, dict) else ""
|
|
1394
|
+
# Destination first: a foreign destination excludes the ad even
|
|
1395
|
+
# when the page name resembles the brand (namesake policy of the
|
|
1396
|
+
# Competitor Ads runner); without a usable destination the name
|
|
1397
|
+
# decides.
|
|
1398
|
+
if link_domain:
|
|
1399
|
+
target = next((d for d in chunk if link_domain == d), None)
|
|
1400
|
+
else:
|
|
1401
|
+
target = next((d for d in chunk if _name_matches(page, brands[d])), None)
|
|
1402
|
+
if target is None:
|
|
1403
|
+
continue
|
|
1404
|
+
first = _iso_from_epoch(row.get("start_date") or row.get("startDate"))
|
|
1405
|
+
last = _iso_from_epoch(row.get("end_date") or row.get("endDate"))
|
|
1406
|
+
is_active = row.get("is_active")
|
|
1407
|
+
by_domain[target].append({"first_seen": first, "last_seen": last, "is_active": bool(is_active) if is_active is not None else None, "format": None, "raw": row})
|
|
1408
|
+
for domain in chunk:
|
|
1409
|
+
results.append(_presence(domain, "meta_ads", "meta_library_actor", by_domain[domain], market_scope="target", verified_by="destination_or_page_name", today=today, note=f"page search country {country}; provider delivery dates are global unless native scope is explicit", country=country, request_limit=per_domain, raw_count=len(rows), native_activity_scope="global", retrieved_on=retrieved_on))
|
|
1410
|
+
if not eu_country:
|
|
1411
|
+
return results
|
|
1412
|
+
for domain in domains:
|
|
1413
|
+
brand = brand_token(domain)
|
|
1414
|
+
outcome = _cumulative_answer(coverage, "meta", [domain], eu_country, lambda: _call_with_context(client, domain, "Meta Ads Library (official EU API)", lambda: client.meta_library_search(
|
|
1415
|
+
{"query": brand, "countries": [eu_country], "activeStatus": "ALL", "limit": per_domain},
|
|
1416
|
+
)))
|
|
1417
|
+
if outcome is None:
|
|
1418
|
+
continue
|
|
1419
|
+
rows, retrieved_on = outcome
|
|
1420
|
+
ads: list[dict[str, Any]] = []
|
|
1421
|
+
for row in rows:
|
|
1422
|
+
page = str(row.get("page_name") or "")
|
|
1423
|
+
links = row.get("ad_creative_link_captions") or []
|
|
1424
|
+
if isinstance(links, str):
|
|
1425
|
+
links = [links]
|
|
1426
|
+
# Captions are display text ('Learn more', 'MURAL.CO | ...') as
|
|
1427
|
+
# often as addresses: only address-shaped captions count as a
|
|
1428
|
+
# destination. Destination first: a known foreign destination
|
|
1429
|
+
# excludes the ad whatever the page is called; the name decides
|
|
1430
|
+
# only when the row carries no usable destination.
|
|
1431
|
+
link_domains = {usable_destination(x) for x in links if x} - {""} if isinstance(links, list) else set()
|
|
1432
|
+
if link_domains:
|
|
1433
|
+
if domain not in link_domains:
|
|
1434
|
+
continue
|
|
1435
|
+
elif not _name_matches(page, brand):
|
|
1436
|
+
continue
|
|
1437
|
+
first = str(row.get("ad_delivery_start_time") or "")[:10] or None
|
|
1438
|
+
last = str(row.get("ad_delivery_stop_time") or "")[:10] or None
|
|
1439
|
+
# No stop time = delivering now; a stop time only dates the ad,
|
|
1440
|
+
# the recency window decides whether it still counts as recent.
|
|
1441
|
+
ads.append({"first_seen": first, "last_seen": last, "is_active": True if (first and last is None) else (False if last else None), "format": None, "raw": row})
|
|
1442
|
+
results.append(_presence(domain, "meta_ads", "meta_library_official_eu", ads, market_scope="target" if eu_country == country else "eu_proxy", verified_by="destination_or_page_name", today=today, note=f"EU delivery country {eu_country}; ad-level dates have global scope unless provider supplies country dates", country=eu_country, request_limit=per_domain, raw_count=len(rows), native_activity_scope="global", retrieved_on=retrieved_on))
|
|
1443
|
+
return results
|
|
1444
|
+
|
|
1445
|
+
|
|
1446
|
+
def collect_linkedin(client: OsClient, spec: dict[str, Any], domains: list[str], today: date, company_slugs: dict[str, str] | None, coverage: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
1447
|
+
"""Advertiser search by LinkedIn company page URL. A keyword query
|
|
1448
|
+
returns ads whose copy mentions the word (observed live: 'coddy' matched
|
|
1449
|
+
'Caddi'), so the query is the company page, resolved from the
|
|
1450
|
+
competitor's own site when the tag scan saw it and guessed from the
|
|
1451
|
+
brand token otherwise. Rows are attributed by the query they came from
|
|
1452
|
+
and checked against the paying advertiser's name."""
|
|
1453
|
+
per_domain = int(spec.get("ads_per_domain") or 30)
|
|
1454
|
+
slugs = company_slugs or {}
|
|
1455
|
+
results: list[dict[str, Any]] = []
|
|
1456
|
+
for chunk in _chunks(domains, int(spec.get("batch_domains") or 5)):
|
|
1457
|
+
brands = {d: brand_token(d) for d in chunk}
|
|
1458
|
+
queries = {d: f"https://www.linkedin.com/company/{slugs.get(d) or brands[d]}" for d in chunk}
|
|
1459
|
+
outcome = _attempt(client, coverage, "linkedin", chunk, lambda: _call_with_context(client, ", ".join(chunk), "LinkedIn Ad Library", lambda: client.run_actor(
|
|
1460
|
+
spec["actor"], {"queries": [queries[d] for d in chunk], "maxResults": per_domain * len(chunk), "enrich": True}, limit=per_domain * len(chunk),
|
|
1461
|
+
)))
|
|
1462
|
+
if outcome is None:
|
|
1463
|
+
continue
|
|
1464
|
+
rows, retrieved_on = outcome
|
|
1465
|
+
by_domain: dict[str, list[dict[str, Any]]] = {d: [] for d in chunk}
|
|
1466
|
+
for row in rows:
|
|
1467
|
+
advertiser = row.get("advertiser_name") or row.get("advertiserName") or ""
|
|
1468
|
+
query = str(row.get("query") or "").strip().lower()
|
|
1469
|
+
target = next((d for d in chunk if query and query == queries[d].lower()), None)
|
|
1470
|
+
if target is None:
|
|
1471
|
+
target = next((d for d in chunk if _name_matches(advertiser, brands[d])), None)
|
|
1472
|
+
if target is None:
|
|
1473
|
+
continue
|
|
1474
|
+
else:
|
|
1475
|
+
# Query attribution still needs the paying advertiser to BE
|
|
1476
|
+
# the brand (or the company page read from its own site):
|
|
1477
|
+
# a guessed slug can land on a namesake company page.
|
|
1478
|
+
slug_brand = re.sub(r"[^a-z0-9]", "", slugs.get(target, "").lower())
|
|
1479
|
+
if not (_name_matches(advertiser, brands[target]) or (slug_brand and _name_matches(advertiser, slug_brand))):
|
|
1480
|
+
continue
|
|
1481
|
+
first = str(row.get("_first_seen") or row.get("first_seen") or "")[:10] or None
|
|
1482
|
+
last = str(row.get("_last_seen") or row.get("last_seen") or "")[:10] or None
|
|
1483
|
+
by_domain[target].append({"first_seen": first, "last_seen": last, "is_active": None, "format": row.get("creative_format"), "raw": row})
|
|
1484
|
+
for domain in chunk:
|
|
1485
|
+
source_note = f"company page {queries[domain]}" + ("" if slugs.get(domain) else " (slug guessed from the brand token)")
|
|
1486
|
+
results.append(_presence(domain, "linkedin_ads", "linkedin_library_actor", by_domain[domain], market_scope="global", verified_by="company_page_query", today=today, note=f"{source_note}; LinkedIn publishes no delivery dates; sightings are listing history", request_limit=per_domain, raw_count=len(rows), native_date_scope="unknown", retrieved_on=retrieved_on))
|
|
1487
|
+
return results
|
|
1488
|
+
|
|
1489
|
+
|
|
1490
|
+
def collect_tiktok(client: OsClient, spec: dict[str, Any], domains: list[str], today: date, eu_country: str, coverage: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
1491
|
+
per_domain = int(spec.get("ads_per_domain") or 30)
|
|
1492
|
+
results: list[dict[str, Any]] = []
|
|
1493
|
+
for domain in domains:
|
|
1494
|
+
brand = brand_token(domain)
|
|
1495
|
+
outcome = _attempt(client, coverage, "tiktok", [domain], lambda: _call_with_context(client, domain, "TikTok Ad Library", lambda: client.run_actor(
|
|
1496
|
+
spec["actor"], {"scrapeMode": "ad_library", "searchQuery": brand, "country": eu_country, "quickSearch": True, "maxResults": per_domain}, limit=per_domain,
|
|
1497
|
+
)))
|
|
1498
|
+
if outcome is None:
|
|
1499
|
+
continue
|
|
1500
|
+
rows, retrieved_on = outcome
|
|
1501
|
+
ads = []
|
|
1502
|
+
for row in rows:
|
|
1503
|
+
if not _name_matches(row.get("advertiserName"), brand):
|
|
1504
|
+
continue
|
|
1505
|
+
first = str(row.get("firstShownDate") or "")[:10] or None
|
|
1506
|
+
last = str(row.get("lastShownDate") or "")[:10] or None
|
|
1507
|
+
status = str(row.get("status") or "").lower()
|
|
1508
|
+
ads.append({"first_seen": first, "last_seen": last, "is_active": True if status == "active" else (False if status == "inactive" else None), "format": row.get("adFormat"), "raw": row})
|
|
1509
|
+
results.append(_presence(domain, "tiktok_ads", "tiktok_library_actor", ads, market_scope="target", verified_by="advertiser_name", today=today, note=f"EU library country {eu_country}; native date scope remains global unless explicitly returned", country=eu_country, request_limit=per_domain, raw_count=len(rows), native_activity_scope="global", retrieved_on=retrieved_on))
|
|
1510
|
+
return results
|
|
1511
|
+
|
|
1512
|
+
|
|
1513
|
+
def collect_reddit(client: OsClient, spec: dict[str, Any], domains: list[str], today: date, coverage: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
1514
|
+
per_domain = int(spec.get("ads_per_domain") or 30)
|
|
1515
|
+
results: list[dict[str, Any]] = []
|
|
1516
|
+
for domain in domains:
|
|
1517
|
+
brand = brand_token(domain)
|
|
1518
|
+
outcome = _attempt(client, coverage, "reddit", [domain], lambda: _call_with_context(client, domain, "Reddit ads library", lambda: client.run_actor(
|
|
1519
|
+
spec["actor"], {"query": brand, "maxItems": per_domain}, limit=per_domain,
|
|
1520
|
+
)))
|
|
1521
|
+
if outcome is None:
|
|
1522
|
+
continue
|
|
1523
|
+
rows, retrieved_on = outcome
|
|
1524
|
+
ads = []
|
|
1525
|
+
for row in rows:
|
|
1526
|
+
advertiser = row.get("advertiserName") or row.get("profile_name") or ""
|
|
1527
|
+
# Destination first, name only when the row has no usable
|
|
1528
|
+
# destination (a promoted post links to reddit.com itself).
|
|
1529
|
+
destination = usable_destination(row.get("destinationUrl") or row.get("destination_url") or "")
|
|
1530
|
+
if destination:
|
|
1531
|
+
if destination != domain:
|
|
1532
|
+
continue
|
|
1533
|
+
elif not _name_matches(advertiser, brand):
|
|
1534
|
+
continue
|
|
1535
|
+
created = _iso_from_epoch(row.get("createdAt") or row.get("created_at"))
|
|
1536
|
+
ads.append({"first_seen": created, "last_seen": created, "is_active": None, "format": str(row.get("creativeType") or row.get("type") or "unknown").lower(), "budget": row.get("budgetCategory") or row.get("budget_category"), "placements": row.get("placements"), "raw": row})
|
|
1537
|
+
presence = _presence(domain, "reddit_ads", "reddit_inspiration_actor", ads, market_scope="global", verified_by="destination_or_advertiser_name", today=today, note="inspiration examples; current campaigns are not verifiable; dates are creation dates, not run spans", request_limit=per_domain, raw_count=len(rows), native_date_scope="unknown", retrieved_on=retrieved_on)
|
|
1538
|
+
budgets = sorted({str(a.get("budget")) for a in ads if a.get("budget")})
|
|
1539
|
+
if budgets:
|
|
1540
|
+
presence["note"] += f"; budget tiers {', '.join(budgets)}"
|
|
1541
|
+
results.append(presence)
|
|
1542
|
+
return results
|
|
1543
|
+
|
|
1544
|
+
|
|
1545
|
+
# ── partner signals ─────────────────────────────────────────────────────────
|
|
1546
|
+
|
|
1547
|
+
def _paid_search_row(domain: str, country: str, data: dict[str, Any], retrieved_on: str) -> dict[str, Any]:
|
|
1548
|
+
"""Per competitor: SpyFu's modeled Google Ads spend over the complete
|
|
1549
|
+
months checked. `found: false` means SpyFu has no data for the domain,
|
|
1550
|
+
which is not the same as zero spend."""
|
|
1551
|
+
months = [m for m in (data.get("rows") or []) if isinstance(m, dict)]
|
|
1552
|
+
complete = [m for m in months if not m.get("partial_month")]
|
|
1553
|
+
spending = [m for m in complete if float(m.get("monthly_budget_usd") or 0) > 0]
|
|
1554
|
+
latest = complete[0] if complete else None # the gateway returns the newest month first
|
|
1555
|
+
budgets = [float(m["monthly_budget_usd"]) for m in spending]
|
|
1556
|
+
|
|
1557
|
+
def latest_value(key: str) -> Any:
|
|
1558
|
+
# SpyFu models clicks as floats (64759.99999999999); evidence keeps whole numbers.
|
|
1559
|
+
value = latest.get(key) if latest else None
|
|
1560
|
+
return int(round(float(value))) if isinstance(value, (int, float)) and not isinstance(value, bool) else None
|
|
1561
|
+
|
|
1562
|
+
return {
|
|
1563
|
+
"type": "paid_search_stats",
|
|
1564
|
+
"domain": domain,
|
|
1565
|
+
"platform": "google_ads",
|
|
1566
|
+
"source": "spyfu",
|
|
1567
|
+
"country": country,
|
|
1568
|
+
"found": bool(data.get("found")),
|
|
1569
|
+
"months_checked": len(complete),
|
|
1570
|
+
"months_with_spend": len(spending),
|
|
1571
|
+
"spending_now": bool(latest and float(latest.get("monthly_budget_usd") or 0) > 0),
|
|
1572
|
+
"latest_month": f"{int(latest['year']):04d}-{int(latest['month']):02d}" if latest and latest.get("year") and latest.get("month") else None,
|
|
1573
|
+
"latest_monthly_budget_usd": latest_value("monthly_budget_usd"),
|
|
1574
|
+
"latest_monthly_paid_clicks": latest_value("monthly_paid_clicks"),
|
|
1575
|
+
"latest_paid_keywords": latest_value("paid_keywords"),
|
|
1576
|
+
"avg_monthly_budget_usd": round(sum(budgets) / len(budgets)) if budgets else None,
|
|
1577
|
+
"max_monthly_budget_usd": round(max(budgets)) if budgets else None,
|
|
1578
|
+
"retrieved_on": retrieved_on,
|
|
1579
|
+
"market_scope": "target",
|
|
1580
|
+
"tier": "B",
|
|
1581
|
+
}
|
|
1582
|
+
|
|
1583
|
+
|
|
1584
|
+
def collect_spyfu(client: OsClient, domains: list[str], country: str, coverage: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
1585
|
+
rows: list[dict[str, Any]] = []
|
|
1586
|
+
for domain in domains:
|
|
1587
|
+
answer = _attempt(client, coverage, "spyfu", [domain], lambda d=domain: client.spyfu_domain_stats(d, country, SPYFU_MONTHS))
|
|
1588
|
+
if answer is None:
|
|
1589
|
+
continue
|
|
1590
|
+
data, retrieved_on = answer
|
|
1591
|
+
row = _paid_search_row(domain, country, data, retrieved_on)
|
|
1592
|
+
rows.append(row)
|
|
1593
|
+
coverage.append({"type": "coverage", "domain": domain, "source": "spyfu", "status": "ok" if row["found"] else "no_data",
|
|
1594
|
+
"detail": "" if row["found"] else "SpyFu has no Google search data for this domain", "tier": "C"})
|
|
1595
|
+
return rows
|
|
1596
|
+
|
|
1597
|
+
|
|
1598
|
+
def _partner_hit(kind: str, domain: str, brand: str, url: str, title: str) -> bool:
|
|
1599
|
+
"""Precision over recall: a hit must be the competitor's own program
|
|
1600
|
+
page, a known affiliate network or coupon site, or a page whose own
|
|
1601
|
+
address names the brand together with the signal word. Generic
|
|
1602
|
+
listicles that merely mention the brand do not count."""
|
|
1603
|
+
host = registrable_domain(url)
|
|
1604
|
+
path = urllib.parse.urlparse(url).path.lower()
|
|
1605
|
+
own = host == domain
|
|
1606
|
+
if kind == "affiliate_program":
|
|
1607
|
+
if own:
|
|
1608
|
+
return bool(re.search(r"affiliate|partner|referral|ambassador", path))
|
|
1609
|
+
return host in AFFILIATE_NETWORK_DOMAINS and _mentions(f"{url} {title}", brand)
|
|
1610
|
+
if kind == "coupon_listing":
|
|
1611
|
+
if own:
|
|
1612
|
+
return False
|
|
1613
|
+
return (host in COUPON_DOMAINS and _mentions(f"{url} {title}", brand)) or (
|
|
1614
|
+
bool(re.search(r"coupon|promo|discount", path)) and _mentions(path, brand)
|
|
1615
|
+
)
|
|
1616
|
+
if kind == "sponsorship":
|
|
1617
|
+
if own:
|
|
1618
|
+
return False
|
|
1619
|
+
return bool(re.search(r"sponsor", f"{url} {title}", re.IGNORECASE)) and _mentions(f"{url} {title}", brand)
|
|
1620
|
+
return False
|
|
1621
|
+
|
|
1622
|
+
|
|
1623
|
+
def collect_partners(client: OsClient, domain: str, country: str, coverage: list[dict[str, Any]] | None = None) -> list[dict[str, Any]]:
|
|
1624
|
+
brand = brand_token(domain)
|
|
1625
|
+
rows: list[dict[str, Any]] = []
|
|
1626
|
+
for kind, template in PARTNER_KINDS:
|
|
1627
|
+
query = template.format(brand=brand)
|
|
1628
|
+
try:
|
|
1629
|
+
response, _ = client.brave(query, BRAVE_COUNT, country.lower())
|
|
1630
|
+
except BudgetExhausted as exc:
|
|
1631
|
+
if "cumulative" not in str(exc):
|
|
1632
|
+
raise
|
|
1633
|
+
if coverage is not None:
|
|
1634
|
+
coverage.append({"type": "coverage", "domain": domain, "source": "partners", "country": country, "status": "budget_deferred", "detail": f"{kind}: {exc}", "tier": "C"})
|
|
1635
|
+
rows.append({"type": "partner_signal", "domain": domain, "country": country, "kind": kind, "query": query, "hits": None, "urls": [], "source": "brave", "status": "budget_deferred", "tier": "C"})
|
|
1636
|
+
continue
|
|
1637
|
+
except OsUnreachable:
|
|
1638
|
+
raise
|
|
1639
|
+
except ChannelFootprintError as exc:
|
|
1640
|
+
rows.append({"type": "partner_signal", "domain": domain, "kind": kind, "query": query, "hits": None, "urls": [], "unavailable": _provider_detail(exc)[:200], "source": "brave", "tier": "C"})
|
|
1641
|
+
continue
|
|
1642
|
+
hits = 0
|
|
1643
|
+
urls: list[str] = []
|
|
1644
|
+
for item in response.get("results") or []:
|
|
1645
|
+
if not isinstance(item, dict):
|
|
1646
|
+
continue
|
|
1647
|
+
url = str(item.get("url") or "")
|
|
1648
|
+
title = str(item.get("title") or "")
|
|
1649
|
+
if _partner_hit(kind, domain, brand, url, title):
|
|
1650
|
+
hits += 1
|
|
1651
|
+
if len(urls) < PARTNER_URLS_PER_KIND:
|
|
1652
|
+
urls.append(url)
|
|
1653
|
+
rows.append({"type": "partner_signal", "domain": domain, "kind": kind, "query": query, "hits": hits, "urls": urls, "source": "brave", "tier": "B" if hits else "C"})
|
|
1654
|
+
return rows
|
|
1655
|
+
|
|
1656
|
+
|
|
1657
|
+
# ── aggregation ─────────────────────────────────────────────────────────────
|
|
1658
|
+
|
|
1659
|
+
def observed_channels(presences=(), scans=(), traffic=(), paid_search=()) -> tuple[str, ...]:
|
|
1660
|
+
names = set(CHANNELS)
|
|
1661
|
+
names.update(p.get("platform") for p in presences if p.get("platform"))
|
|
1662
|
+
names.update(channel for scan in scans for channel in scan.get("channels_detected", []))
|
|
1663
|
+
names.update(row.get("paid_channel") for row in traffic if row.get("paid_channel"))
|
|
1664
|
+
names.update(row.get("platform") for row in paid_search if row.get("platform"))
|
|
1665
|
+
return tuple(sorted(name for name in names if isinstance(name, str) and re.fullmatch(r"[a-z][a-z0-9_]+", name)))
|
|
1666
|
+
|
|
1667
|
+
|
|
1668
|
+
def channel_label(channel: str) -> str:
|
|
1669
|
+
return CHANNEL_LABELS.get(channel, channel)
|
|
1670
|
+
|
|
1671
|
+
|
|
1672
|
+
def rank_channels(domains: list[str], presences: list[dict[str, Any]], scans: list[dict[str, Any]], traffic: list[dict[str, Any]], partners: list[dict[str, Any]], paid_search: list[dict[str, Any]] = (), competitor_context: dict[str, dict[str, Any]] | None = None) -> list[dict[str, Any]]:
|
|
1673
|
+
"""Describe direct-competitor observations; do not rank channel merit.
|
|
1674
|
+
|
|
1675
|
+
Evidence kinds are preserved because tags, listings and modeled spend are
|
|
1676
|
+
different observations. Counts never imply acquisition success.
|
|
1677
|
+
"""
|
|
1678
|
+
context = competitor_context or {}
|
|
1679
|
+
direct = {d for d in domains if context.get(d, {}).get("competitor_tier") == "direct"}
|
|
1680
|
+
rows = []
|
|
1681
|
+
for channel in observed_channels(presences, scans, traffic, paid_search):
|
|
1682
|
+
evidence: dict[str, set[str]] = {}
|
|
1683
|
+
active_ids, proven_domains, durations = set(), set(), []
|
|
1684
|
+
for p in presences:
|
|
1685
|
+
if p.get("domain") not in direct or p.get("platform") != channel:
|
|
1686
|
+
continue
|
|
1687
|
+
if p.get("present") and p.get("relevance") == "relevant":
|
|
1688
|
+
evidence.setdefault(p["domain"], set()).add("library" if p.get("active_ads") else "listing")
|
|
1689
|
+
active_ids.update(p.get("current_ad_ids") or [])
|
|
1690
|
+
if p.get("current_presence"):
|
|
1691
|
+
proven_domains.add(p["domain"])
|
|
1692
|
+
if p.get("oldest_days") is not None:
|
|
1693
|
+
durations.append(p["oldest_days"])
|
|
1694
|
+
for scan in scans:
|
|
1695
|
+
if scan["domain"] in direct and channel in scan.get("channels_detected", []):
|
|
1696
|
+
evidence.setdefault(scan["domain"], set()).add("pixel")
|
|
1697
|
+
samples = [t for t in traffic if t["domain"] in direct and t.get("paid_channel") == channel and t.get("share_pct") is not None]
|
|
1698
|
+
for row in samples:
|
|
1699
|
+
if float(row.get("share_pct_raw", row["share_pct"])) > PRESENT_SHARE_PCT:
|
|
1700
|
+
evidence.setdefault(row["domain"], set()).add("modeled_traffic")
|
|
1701
|
+
if channel == "affiliate":
|
|
1702
|
+
for row in partners:
|
|
1703
|
+
if row["domain"] in direct and row.get("kind") in {"affiliate_program", "coupon_listing"} and row.get("hits", 0):
|
|
1704
|
+
evidence.setdefault(row["domain"], set()).add("partner_signal")
|
|
1705
|
+
budgets = [r for r in paid_search if r["domain"] in direct and r.get("platform") == channel and r.get("spending_now") and r.get("latest_monthly_budget_usd") is not None]
|
|
1706
|
+
for row in budgets:
|
|
1707
|
+
evidence.setdefault(row["domain"], set()).add("modeled_spend")
|
|
1708
|
+
count = len(evidence)
|
|
1709
|
+
rows.append({"type": "channel_observation", "channel": channel, "label": channel_label(channel),
|
|
1710
|
+
"competitor_tier": "direct", "competitors_present": count, "competitors_total": len(direct), "present_domains": sorted(evidence),
|
|
1711
|
+
"evidence_kinds": {d: sorted(kinds) for d, kinds in sorted(evidence.items())}, "active_ads_total": len(active_ids),
|
|
1712
|
+
"active_competitors": len(proven_domains), "max_oldest_days": max(durations) if durations else None,
|
|
1713
|
+
"avg_share_pct": round(sum(float(r.get("share_pct_raw", r["share_pct"])) for r in samples) / len(samples), 2) if samples else None,
|
|
1714
|
+
"share_sample_n": len(samples), "share_sample_domains": sorted({r["domain"] for r in samples}), "share_scope": "modeled_domain_global",
|
|
1715
|
+
"est_monthly_budget_usd": sum(float(r["latest_monthly_budget_usd"]) for r in budgets) if budgets else None,
|
|
1716
|
+
"why": f"{count} of {len(direct)} direct competitor(s) with a positive observation; evidence kinds remain separate", "tier": "B" if count else "C",
|
|
1717
|
+
"campaign_data_status": "activity_observed" if proven_domains else "activity_unknown"})
|
|
1718
|
+
rows.sort(key=lambda r: r["channel"])
|
|
1719
|
+
return rows
|
|
1720
|
+
|
|
1721
|
+
|
|
1722
|
+
# ── rendering ───────────────────────────────────────────────────────────────
|
|
1723
|
+
|
|
1724
|
+
def _cell(value: Any, cap: int = 120) -> str:
|
|
1725
|
+
text = " ".join(str(value if value is not None else "—").split()).replace("|", "\\|")
|
|
1726
|
+
return text if len(text) <= cap else text[: cap - 1] + "…"
|
|
1727
|
+
|
|
1728
|
+
|
|
1729
|
+
# ── orchestration ───────────────────────────────────────────────────────────
|
|
1730
|
+
|
|
1731
|
+
EU_COUNTRIES = frozenset("AT BE BG HR CY CZ DE DK EE ES FI FR GR HU IE IT LT LU LV MT NL PL PT RO SE SI SK".split())
|
|
1732
|
+
|
|
1733
|
+
|
|
1734
|
+
def _effective_policy(config: dict[str, Any]) -> tuple[dict[str, Any], str]:
|
|
1735
|
+
module = _load_script("research-policy.py", "pmr_channel_policy")
|
|
1736
|
+
taxonomy_path = Path(__file__).resolve().parent.parent / "references/config/creative-taxonomy.v1.json"
|
|
1737
|
+
return module.policy_from_taxonomy(json.loads(taxonomy_path.read_text()))
|
|
1738
|
+
|
|
1739
|
+
|
|
1740
|
+
def _coverage_grid(inputs: RunInputs, presences: list[dict[str, Any]], coverage: list[dict[str, Any]], raw: dict[str, Any] | None = None) -> None:
|
|
1741
|
+
for domain in inputs.domains:
|
|
1742
|
+
for source in inputs.sources:
|
|
1743
|
+
for country in inputs.target_countries:
|
|
1744
|
+
existing = [r for r in coverage if r.get("domain") == domain and r.get("source") == source and r.get("country") in {country, None}]
|
|
1745
|
+
if existing:
|
|
1746
|
+
continue
|
|
1747
|
+
matching = [p for p in presences if p["domain"] == domain and p["source"].startswith({"google": "google_", "meta": "meta_", "linkedin": "linkedin_", "reddit": "reddit_", "tiktok": "tiktok_"}.get(source, source)) and p.get("country") in {country, None}]
|
|
1748
|
+
global_only = source in {"pixels", "history", "traffic", "linkedin", "reddit"}
|
|
1749
|
+
unsupported = source == "tiktok" and country not in EU_COUNTRIES
|
|
1750
|
+
measurements = [r for r in (raw or {}).get({"history": "history", "partners": "partners", "spyfu": "paid_search", "traffic": "traffic", "pixels": "scans"}.get(source, ""), []) if r.get("domain") == domain and r.get("country") in {country, None}]
|
|
1751
|
+
if unsupported:
|
|
1752
|
+
status = "unsupported_market"
|
|
1753
|
+
elif source in {"linkedin", "reddit"} and matching:
|
|
1754
|
+
status = "not_verifiable"
|
|
1755
|
+
elif matching:
|
|
1756
|
+
status = "ok" if any(p.get("ads") for p in matching) else "no_matches_in_sample"
|
|
1757
|
+
elif measurements:
|
|
1758
|
+
if source == "partners":
|
|
1759
|
+
status = "ok" if any(r.get("hits", 0) for r in measurements) else "no_matches_in_sample"
|
|
1760
|
+
else:
|
|
1761
|
+
status = next((r.get("status") for r in measurements if r.get("status")), "ok")
|
|
1762
|
+
else:
|
|
1763
|
+
status = "not_checked"
|
|
1764
|
+
coverage.append({"type": "coverage", "domain": domain, "source": source, "country": country,
|
|
1765
|
+
"market_scope": "global" if global_only else "target", "status": status,
|
|
1766
|
+
"detail": "source cannot verify current campaigns or native delivery duration" if source in {"linkedin", "reddit"} else ("source does not support this target country" if unsupported else ""), "tier": "C"})
|
|
1767
|
+
|
|
1768
|
+
|
|
1769
|
+
def finalize_pipeline(raw: dict[str, Any], inputs: RunInputs, config: dict[str, Any], today: date, stale_decisions: str = "raise") -> dict[str, Any]:
|
|
1770
|
+
model = _model()
|
|
1771
|
+
policy, policy_hash = _effective_policy(config)
|
|
1772
|
+
raw_presences = raw["presences"]
|
|
1773
|
+
corpus = model.build_corpus(raw_presences, inputs.product_context_sha256, policy)
|
|
1774
|
+
for ad in corpus:
|
|
1775
|
+
ad["competitor_tier"] = inputs.competitor_context.get(ad["domain"], {}).get("competitor_tier", "unknown")
|
|
1776
|
+
ad["activity_status"] = "unknown" # use native observation status for decisions
|
|
1777
|
+
ad["countries"] = sorted({o["country"] for o in ad["observations"] if o.get("country")})
|
|
1778
|
+
try:
|
|
1779
|
+
pending = model.apply_decisions(corpus, inputs.relevance_decisions, inputs.product_context_sha256, stale=stale_decisions)
|
|
1780
|
+
except ValueError as exc:
|
|
1781
|
+
raise ChannelFootprintError(str(exc)) from exc
|
|
1782
|
+
presences = model.summarize_presences(raw_presences, corpus, inputs.competitor_context, policy, inputs.product_context_sha256)
|
|
1783
|
+
# A native placement creates a separately scoped observation. A video
|
|
1784
|
+
# format alone stays Google and never asserts YouTube delivery.
|
|
1785
|
+
for placement, channel in (("youtube", "youtube_ads"), ("display", "display_native")):
|
|
1786
|
+
sub_raw = []
|
|
1787
|
+
for presence in raw_presences:
|
|
1788
|
+
if presence["platform"] != "google_ads":
|
|
1789
|
+
continue
|
|
1790
|
+
ids = {a["ad_id"] for a in corpus if any(o.get("placement") == placement and o.get("country") == presence.get("country") for o in a["observations"])}
|
|
1791
|
+
selected_ids = [x for x in presence.get("ad_ids", []) if x in ids]
|
|
1792
|
+
if selected_ids:
|
|
1793
|
+
sub_raw.append({**presence, "platform": channel, "ad_ids": selected_ids})
|
|
1794
|
+
if sub_raw:
|
|
1795
|
+
presences.extend(model.summarize_presences(sub_raw, corpus, inputs.competitor_context, policy, inputs.product_context_sha256, placement_filter=placement))
|
|
1796
|
+
for row in raw["scans"] + raw["traffic"] + raw["paid_search"] + raw["partners"] + raw["history"]:
|
|
1797
|
+
row["competitor_tier"] = inputs.competitor_context.get(row["domain"], {}).get("competitor_tier", "unknown")
|
|
1798
|
+
row.setdefault("market_scope", "global")
|
|
1799
|
+
country_rows = model.country_rows(presences, raw["scans"], inputs.target_countries, inputs.competitor_context, policy, observed_channels(presences, raw["scans"], raw["traffic"], raw["paid_search"]))
|
|
1800
|
+
ranks = rank_channels(inputs.domains, presences, raw["scans"], raw["traffic"], raw["partners"], raw["paid_search"], inputs.competitor_context)
|
|
1801
|
+
coverage = [dict(row) for row in raw["coverage"]]
|
|
1802
|
+
_coverage_grid(inputs, raw_presences, coverage, raw)
|
|
1803
|
+
totals = raw["totals"]
|
|
1804
|
+
retrieved_on = totals["first_retrieved_on"] if totals["first_retrieved_on"] == totals["last_retrieved_on"] else f"{totals['first_retrieved_on']}..{totals['last_retrieved_on']}"
|
|
1805
|
+
evidence = {"schema_version": SCHEMA_VERSION, "section_id": SECTION_ID, "generated_by": Path(__file__).name, "retrieved_on": retrieved_on,
|
|
1806
|
+
"inputs": {"domains": inputs.domains, "domain_sources": inputs.domain_sources, "competitor_context": inputs.competitor_context,
|
|
1807
|
+
"product_domain": inputs.product_domain, "product_context_sha256": inputs.product_context_sha256,
|
|
1808
|
+
"sources": inputs.sources, "country": inputs.country, "eu_country": inputs.eu_country, "target_countries": inputs.target_countries,
|
|
1809
|
+
"present_share_pct": PRESENT_SHARE_PCT, "effective_policy": policy, "policy_sha256": policy_hash},
|
|
1810
|
+
"rows": ranks + country_rows + presences + corpus + raw["paid_search"] + raw["scans"] + raw["traffic"] + raw["partners"] + raw["history"] + coverage,
|
|
1811
|
+
"totals": {k: totals[k] for k in ("actor_runs", "meta_searches", "search_calls", "spyfu_calls", "fetches", "cdx_calls", "provider_calls", "rejected_actor_runs", "rejected_spyfu_calls", "observations")}}
|
|
1812
|
+
section = render_section_v2(inputs, presences, corpus, ranks, country_rows, {**raw, "coverage": coverage}, retrieved_on)
|
|
1813
|
+
return {"evidence": evidence, "section_markdown": section, "totals": totals, "attempt": raw["attempt"], "raw": raw,
|
|
1814
|
+
"pending_relevance": pending, "corpus": corpus, "collection_complete": raw.get("collection_complete", True)}
|
|
1815
|
+
|
|
1816
|
+
|
|
1817
|
+
def run_pipeline(client: OsClient, inputs: RunInputs, config: dict[str, Any], today: date) -> dict[str, Any]:
|
|
1818
|
+
actors = config["actors"]
|
|
1819
|
+
policy, _ = _effective_policy(config)
|
|
1820
|
+
inputs.effective_policy = policy
|
|
1821
|
+
inputs.target_countries = _model().countries(inputs.target_countries or [inputs.country])
|
|
1822
|
+
inputs.country = inputs.target_countries[0]
|
|
1823
|
+
raw = {"scans": [], "history": [], "traffic": [], "presences": [], "partners": [], "coverage": [], "paid_search": []}
|
|
1824
|
+
primary = list(inputs.domains) # All explicitly scoped competitors; roles describe comparability, not eligibility.
|
|
1825
|
+
|
|
1826
|
+
def collect(domains: list[str], deficits: set[tuple[str, str]] | None = None) -> None:
|
|
1827
|
+
if not domains:
|
|
1828
|
+
return
|
|
1829
|
+
def call(source: str, country: str | None, fn: Callable[[], Any], target: str):
|
|
1830
|
+
coverage_before = len(raw["coverage"])
|
|
1831
|
+
try:
|
|
1832
|
+
value = fn()
|
|
1833
|
+
except BudgetExhausted as exc:
|
|
1834
|
+
if "cumulative" not in str(exc):
|
|
1835
|
+
raise
|
|
1836
|
+
raw["coverage"].extend({"type": "coverage", "domain": d, "source": source, "country": country, "status": "budget_deferred", "detail": str(exc), "tier": "C"} for d in domains)
|
|
1837
|
+
return
|
|
1838
|
+
for row in raw["coverage"][coverage_before:]:
|
|
1839
|
+
row.setdefault("country", country)
|
|
1840
|
+
if value:
|
|
1841
|
+
if country and target in {"paid_search", "partners"}:
|
|
1842
|
+
for row in value:
|
|
1843
|
+
row.setdefault("country", country)
|
|
1844
|
+
raw[target].extend(value)
|
|
1845
|
+
if "pixels" in inputs.sources:
|
|
1846
|
+
call("pixels", None, lambda: [scan_site(client, d) for d in domains], "scans")
|
|
1847
|
+
opaque = [s["domain"] for s in raw["scans"] if s["domain"] in domains and (s["fetch_status"] != "ok" or not s.get("tags_detected"))]
|
|
1848
|
+
if opaque and inputs.pixel_actor_fallback:
|
|
1849
|
+
for chunk in _chunks(opaque, int(actors["pixel_detector"].get("batch_domains") or 5)):
|
|
1850
|
+
try:
|
|
1851
|
+
replacements = _attempt(client, raw["coverage"], "pixels_actor", chunk, lambda: scan_sites_with_actor(client, actors["pixel_detector"]["actor"], chunk))
|
|
1852
|
+
except BudgetExhausted as exc:
|
|
1853
|
+
if "cumulative" not in str(exc):
|
|
1854
|
+
raise
|
|
1855
|
+
replacements = None
|
|
1856
|
+
raw["coverage"].extend({"type": "coverage", "domain": d, "source": "pixels_actor", "status": "budget_deferred", "detail": str(exc), "tier": "C"} for d in chunk)
|
|
1857
|
+
if replacements:
|
|
1858
|
+
for scan in raw["scans"]:
|
|
1859
|
+
replacement = replacements.get(scan["domain"])
|
|
1860
|
+
if replacement is not None:
|
|
1861
|
+
replacement["linkedin_company"] = scan.get("linkedin_company")
|
|
1862
|
+
replacement["gtm_containers"] = scan.get("gtm_containers") or []
|
|
1863
|
+
raw["scans"] = [replacements.get(s["domain"], s) for s in raw["scans"]]
|
|
1864
|
+
for scan in raw["scans"]:
|
|
1865
|
+
if scan["domain"] in domains:
|
|
1866
|
+
raw["coverage"].append({"type": "coverage", "domain": scan["domain"], "source": "pixels", "market_scope": "global", "status": "ok" if scan["fetch_status"] == "ok" else scan["fetch_status"], "detail": "domain tracking signal; campaign details unavailable" if scan.get("channels_detected") else "", "tier": "C"})
|
|
1867
|
+
if "history" in inputs.sources:
|
|
1868
|
+
call("history", None, lambda: [collect_history(client, d, today) for d in domains], "history")
|
|
1869
|
+
if "traffic" in inputs.sources:
|
|
1870
|
+
call("traffic", None, lambda: collect_traffic(client, actors["traffic_mix"], domains, raw["coverage"]), "traffic")
|
|
1871
|
+
for country in inputs.target_countries:
|
|
1872
|
+
if "google" in inputs.sources and (deficits is None or ("google_ads", country) in deficits):
|
|
1873
|
+
call("google", country, lambda: collect_google(client, actors["google_ads"], domains, today, country, raw["coverage"]), "presences")
|
|
1874
|
+
if "spyfu" in inputs.sources and (deficits is None or ("google_ads", country) in deficits):
|
|
1875
|
+
call("spyfu", country, lambda: collect_spyfu(client, domains, country, raw["coverage"]), "paid_search")
|
|
1876
|
+
if "meta" in inputs.sources and (deficits is None or ("meta_ads", country) in deficits):
|
|
1877
|
+
call("meta", country, lambda: collect_meta(client, actors["meta_ads"], domains, today, country, country if country in EU_COUNTRIES else "", raw["coverage"]), "presences")
|
|
1878
|
+
if "partners" in inputs.sources and (deficits is None or ("affiliate", country) in deficits):
|
|
1879
|
+
call("partners", country, lambda: [r for d in domains for r in collect_partners(client, d, country, raw["coverage"])], "partners")
|
|
1880
|
+
if "tiktok" in inputs.sources and country in EU_COUNTRIES and (deficits is None or ("tiktok_ads", country) in deficits):
|
|
1881
|
+
call("tiktok", country, lambda: collect_tiktok(client, actors["tiktok_ads"], domains, today, country, raw["coverage"]), "presences")
|
|
1882
|
+
if "linkedin" in inputs.sources and (deficits is None or any(ch == "linkedin_ads" for ch, _ in deficits)):
|
|
1883
|
+
slugs = {s["domain"]: s["linkedin_company"] for s in raw["scans"] if s.get("linkedin_company")}
|
|
1884
|
+
call("linkedin", None, lambda: collect_linkedin(client, actors["linkedin_ads"], domains, today, slugs, raw["coverage"]), "presences")
|
|
1885
|
+
if "reddit" in inputs.sources and (deficits is None or any(ch == "reddit_ads" for ch, _ in deficits)):
|
|
1886
|
+
call("reddit", None, lambda: collect_reddit(client, actors["reddit_ads"], domains, today, raw["coverage"]), "presences")
|
|
1887
|
+
collect(primary)
|
|
1888
|
+
raw["totals"] = client.ledger_totals()
|
|
1889
|
+
raw["attempt"] = {"actor_runs": client.actor_runs, "meta_searches": client.meta_searches, "search_calls": client.search_calls, "spyfu_calls": client.spyfu_calls, "fetches": client.fetches, "cdx_calls": client.cdx_calls, "relevance_calls": client.relevance_calls, "cache_hits": client.cache_hits}
|
|
1890
|
+
# The base sample lacks fields that the expansion may fill in an ad
|
|
1891
|
+
# decided after it, so this gate takes decisions by ad_id; the passes on
|
|
1892
|
+
# the expanded corpus send such a changed ad back to review instead.
|
|
1893
|
+
preliminary = finalize_pipeline(raw, inputs, config, today, stale_decisions="accept")
|
|
1894
|
+
# A reviewed direct sample without comparable ads may be expanded once.
|
|
1895
|
+
# The enlarged payload is cached; repeated collect never adds another step.
|
|
1896
|
+
# A collected expansion is replayed whatever a later review says: an ad it
|
|
1897
|
+
# enriched and the founder then found relevant keeps its expanded sample.
|
|
1898
|
+
reviewed = not preliminary["pending_relevance"]
|
|
1899
|
+
expanded_before = {(e.get("domain"), e.get("source"), e.get("country")) for e in inputs.prior_expansions}
|
|
1900
|
+
raw["expansions"] = []
|
|
1901
|
+
for presence in list(raw["presences"]):
|
|
1902
|
+
if not presence.get("sampling", {}).get("limit_reached"):
|
|
1903
|
+
continue
|
|
1904
|
+
if (presence["domain"], presence["source"], presence.get("country")) not in expanded_before:
|
|
1905
|
+
if not reviewed:
|
|
1906
|
+
continue
|
|
1907
|
+
comparable = next((r for r in preliminary["evidence"]["rows"] if r["type"] == "library_presence" and r["domain"] == presence["domain"] and r["platform"] == presence["platform"] and r.get("country") == presence.get("country")), {})
|
|
1908
|
+
if comparable.get("relevant_ads"):
|
|
1909
|
+
continue
|
|
1910
|
+
source_key = {"google_transparency_actor": "google_ads", "meta_library_actor": "meta_ads", "linkedin_library_actor": "linkedin_ads", "tiktok_library_actor": "tiktok_ads"}.get(presence["source"])
|
|
1911
|
+
if not source_key:
|
|
1912
|
+
continue
|
|
1913
|
+
expanded_spec = dict(actors[source_key])
|
|
1914
|
+
base = int(expanded_spec.get("ads_per_domain") or 30)
|
|
1915
|
+
maximum = int(expanded_spec.get("max_ads_per_domain") or base * 2)
|
|
1916
|
+
if maximum <= base:
|
|
1917
|
+
continue
|
|
1918
|
+
expanded_spec["ads_per_domain"] = maximum
|
|
1919
|
+
domain, country = presence["domain"], presence.get("country")
|
|
1920
|
+
try:
|
|
1921
|
+
if source_key == "google_ads":
|
|
1922
|
+
added = collect_google(client, expanded_spec, [domain], today, country, raw["coverage"])
|
|
1923
|
+
elif source_key == "meta_ads":
|
|
1924
|
+
added = collect_meta(client, expanded_spec, [domain], today, country, country if country in EU_COUNTRIES else "", raw["coverage"])
|
|
1925
|
+
elif source_key == "linkedin_ads":
|
|
1926
|
+
slugs = {s["domain"]: s["linkedin_company"] for s in raw["scans"] if s.get("linkedin_company")}
|
|
1927
|
+
added = collect_linkedin(client, expanded_spec, [domain], today, slugs, raw["coverage"])
|
|
1928
|
+
else:
|
|
1929
|
+
added = collect_tiktok(client, expanded_spec, [domain], today, country, raw["coverage"])
|
|
1930
|
+
raw["presences"].extend(added)
|
|
1931
|
+
if added:
|
|
1932
|
+
raw["expansions"].append({"domain": domain, "source": presence["source"], "country": country})
|
|
1933
|
+
except BudgetExhausted as exc:
|
|
1934
|
+
if "cumulative" not in str(exc):
|
|
1935
|
+
raise
|
|
1936
|
+
raw["coverage"].append({"type": "coverage", "domain": domain, "source": source_key, "country": country, "status": "budget_deferred", "detail": str(exc), "tier": "C"})
|
|
1937
|
+
if reviewed or raw["expansions"]:
|
|
1938
|
+
preliminary = finalize_pipeline(raw, inputs, config, today, stale_decisions="pending")
|
|
1939
|
+
raw["collection_complete"] = True
|
|
1940
|
+
raw["totals"] = client.ledger_totals()
|
|
1941
|
+
raw["attempt"] = {"actor_runs": client.actor_runs, "meta_searches": client.meta_searches, "search_calls": client.search_calls, "spyfu_calls": client.spyfu_calls, "fetches": client.fetches, "cdx_calls": client.cdx_calls, "relevance_calls": client.relevance_calls, "cache_hits": client.cache_hits}
|
|
1942
|
+
return finalize_pipeline(raw, inputs, config, today, stale_decisions="pending")
|
|
1943
|
+
|
|
1944
|
+
|
|
1945
|
+
def render_section_v2(inputs, presences, corpus, ranks, country_rows, raw, retrieved_on):
|
|
1946
|
+
lines = [f"Observed: {retrieved_on}. Target countries: {', '.join(inputs.target_countries)}.", "",
|
|
1947
|
+
"Counts describe the bounded source sample, not channel priority or success. Tracking tags, modeled traffic and ad-library observations remain distinct; a tag does not describe campaigns or country delivery.", ""]
|
|
1948
|
+
for role, label in (("direct", "Direct competitors"), ("indirect", "Indirect competitors"), ("unknown", "Unclassified domains")):
|
|
1949
|
+
domains = [d for d in inputs.domains if inputs.competitor_context.get(d, {}).get("competitor_tier", "unknown") == role]
|
|
1950
|
+
if not domains:
|
|
1951
|
+
continue
|
|
1952
|
+
lines += [f"### {label}: channel footprint matrix", "", "| Competitor | Channel | Country / scope | Observations | Campaign data |", "| --- | --- | --- | --- | --- |"]
|
|
1953
|
+
for domain in domains:
|
|
1954
|
+
for channel in (r["channel"] for r in ranks):
|
|
1955
|
+
pixels = any(s["domain"] == domain and channel in s.get("channels_detected", []) for s in raw["scans"])
|
|
1956
|
+
rows = [r for r in presences if r["domain"] == domain and r["platform"] == channel]
|
|
1957
|
+
if not rows and not pixels:
|
|
1958
|
+
continue
|
|
1959
|
+
if not rows:
|
|
1960
|
+
lines.append(f"| {domain} | {channel_label(channel)} | global / unknown | pixel found; campaign data unavailable | needs_campaign_data |")
|
|
1961
|
+
for row in rows:
|
|
1962
|
+
parts = ["pixel found (global / unknown)"] if pixels else []
|
|
1963
|
+
parts += [f"comparable ads {row['relevant_ads']}; active {row['active_ads']} ({', '.join(row['activity_scopes']) or 'unknown'} scope); country current {len(row['current_ad_ids'])}; stopped {row['stopped_ads']}; undated {row['unknown_activity_ads']}"]
|
|
1964
|
+
if row["unknown_relevance_ads"]:
|
|
1965
|
+
parts.append(f"relevance unknown {row['unknown_relevance_ads']}")
|
|
1966
|
+
parts.append(row["status"])
|
|
1967
|
+
lines.append(f"| {domain} | {channel_label(channel)} | {_cell(row.get('country') or row['market_scope'])} | {_cell('; '.join(parts), 300)} | {'activity_observed' if row['current_presence'] else 'activity_unknown'} |")
|
|
1968
|
+
lines += ["", "### Channel observations (direct competitors)", "", "| Channel | Direct positive observations | Active competitors | Activity coverage |", "| --- | --- | --- | --- |"]
|
|
1969
|
+
for row in ranks:
|
|
1970
|
+
lines.append(f"| {row['label']} | {row['competitors_present']}/{row['competitors_total']} | {row['active_competitors']} | {row['campaign_data_status']} |")
|
|
1971
|
+
lines += ["", "### Country observations", "", "| Channel | Country | Direct observed | Indirect observed | Active domains | Coverage |", "| --- | --- | --- | --- | --- | --- |"]
|
|
1972
|
+
for row in country_rows:
|
|
1973
|
+
lines.append(f"| {channel_label(row['channel'])} | {row['country']} | {len(row['observed_domains_by_role']['direct'])} | {len(row['observed_domains_by_role']['indirect'])} | {len(row['active_domains'])} | {row['status']} |")
|
|
1974
|
+
lines += ["", "### Source samples", "", "| Competitor | Source | Country / scope | Received before relevance | Comparable | Limit / completeness | Max observed delivery duration |", "| --- | --- | --- | --- | --- | --- | --- |"]
|
|
1975
|
+
for row in presences:
|
|
1976
|
+
samples = row.get("sampling", [])
|
|
1977
|
+
cap = any(x.get("limit_reached") for x in samples)
|
|
1978
|
+
received = max((x.get("raw_received", 0) for x in samples), default=0)
|
|
1979
|
+
lines.append(f"| {row['domain']} | {_cell(', '.join(row['sources']))} | {_cell(row.get('country') or row['market_scope'])} | {'≥' if cap else ''}{received} | {row['relevant_ads']} found | {'limit reached; total unknown' if cap else 'completeness unknown'} | {_cell(row['oldest_days'])} days ({row['duration_scope']}) |")
|
|
1980
|
+
lines += ["", "### Coverage", "", "| Competitor | Source | Country / scope | Status | Detail |", "| --- | --- | --- | --- | --- |"]
|
|
1981
|
+
for row in raw["coverage"]:
|
|
1982
|
+
lines.append(f"| {row['domain']} | {row['source']} | {_cell(row.get('country') or row.get('market_scope') or 'global / unknown')} | {row['status']} | {_cell(row.get('detail', ''), 300)} |")
|
|
1983
|
+
lines += ["", "### Source measurements", "", "| Competitor | Source / type | Market | Measurement |", "| --- | --- | --- | --- |"]
|
|
1984
|
+
for row in raw["traffic"] + raw["paid_search"] + raw["partners"] + raw["history"]:
|
|
1985
|
+
measurement = {k: v for k, v in row.items() if k not in {"type", "domain", "competitor_tier", "tier", "market_scope"}}
|
|
1986
|
+
lines.append(f"| {row['domain']} | {_cell(row.get('source') or row['type'])} | {_cell(row.get('country') or row.get('market_scope') or 'unknown')} | {_cell(json.dumps(measurement, ensure_ascii=False), 500)} |")
|
|
1987
|
+
lines += ["", "Legend: pixel found; active native status; stopped; undated listing; no matches in sample; not_checked; not_verifiable; unsupported_market; budget_deferred. Repeated listing observations are presence only. Empty samples never assert channel absence.", ""]
|
|
1988
|
+
return "\n".join(lines)
|
|
1989
|
+
|
|
1990
|
+
|
|
1991
|
+
def _collection_matches(stored: dict[str, Any], context_hash: str, policy_hash: str, sources: list[str], config: dict[str, Any]) -> bool:
|
|
1992
|
+
model = _model()
|
|
1993
|
+
return (stored.get("input_context_sha256") == context_hash and stored.get("policy_sha256") == policy_hash and stored.get("sources") == sources
|
|
1994
|
+
and stored.get("sources_config_sha256") == model.digest(config) and stored.get("raw_sha256") == model.digest(stored.get("raw")))
|
|
1995
|
+
|
|
1996
|
+
|
|
1997
|
+
def _write_atomic(path: Path, content: str) -> None:
|
|
1998
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
1999
|
+
tmp = path.with_suffix(path.suffix + ".tmp")
|
|
2000
|
+
tmp.write_text(content, encoding="utf-8")
|
|
2001
|
+
tmp.replace(path)
|
|
2002
|
+
|
|
2003
|
+
|
|
2004
|
+
def _arg(args: argparse.Namespace, name: str, default: Any) -> Any:
|
|
2005
|
+
value = getattr(args, name, None)
|
|
2006
|
+
return default if value is None else value
|
|
2007
|
+
|
|
2008
|
+
|
|
2009
|
+
def run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
|
|
2010
|
+
workspace = workspace.resolve()
|
|
2011
|
+
io = _load_script("research-artifact-io.py", "pmr_artifact_io")
|
|
2012
|
+
with io.section_attempt(workspace, SECTION_ID, context_file=getattr(args, "context_file", None)):
|
|
2013
|
+
return _run(workspace, args)
|
|
2014
|
+
|
|
2015
|
+
|
|
2016
|
+
def _run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
|
|
2017
|
+
workspace = workspace.resolve()
|
|
2018
|
+
runtime = workspace / RUNTIME_REL
|
|
2019
|
+
cache_dir = runtime / "cache"
|
|
2020
|
+
runtime.mkdir(parents=True, exist_ok=True)
|
|
2021
|
+
|
|
2022
|
+
config = load_sources_config(Path(_arg(args, "sources_config", "")) if _arg(args, "sources_config", "") else None)
|
|
2023
|
+
sources = parse_sources(str(_arg(args, "sources", "") or ""))
|
|
2024
|
+
stage = str(_arg(args, "stage", "all"))
|
|
2025
|
+
if stage not in {"all", "collect", "finalize"}:
|
|
2026
|
+
raise ChannelFootprintError("stage must be collect, finalize or all")
|
|
2027
|
+
policy, policy_hash = _effective_policy(config)
|
|
2028
|
+
max_competitors = max(1, int(_arg(args, "max_competitors", MAX_COMPETITORS)))
|
|
2029
|
+
product_domain = registrable_domain(str(_arg(args, "product_domain", "") or ""))
|
|
2030
|
+
explicit = [d for d in parse_domains(_arg(args, "domains", ""), MAX_EXPLICIT_DOMAINS) if d != product_domain]
|
|
2031
|
+
tiered = domains_from_competitor_set(workspace, MAX_COMPETITORS)
|
|
2032
|
+
rejected = domains_excluded_by_competitor_set(workspace) - set(explicit)
|
|
2033
|
+
discovered = domains_from_keyword_evidence(workspace, MAX_COMPETITORS, exclude=rejected)
|
|
2034
|
+
domains, domain_sources = [], {}
|
|
2035
|
+
for source, candidates in (("explicit", explicit), ("competitor_set", tiered), ("keyword_economics", discovered)):
|
|
2036
|
+
for domain in candidates:
|
|
2037
|
+
if len(domains) >= max_competitors:
|
|
2038
|
+
break
|
|
2039
|
+
if domain and domain != product_domain and domain not in domains:
|
|
2040
|
+
domains.append(domain)
|
|
2041
|
+
domain_sources[domain] = source
|
|
2042
|
+
if not domains:
|
|
2043
|
+
raise ChannelFootprintError("no competitor domains available: pass --domains, or complete the Competitor Set (or Keyword Economics) section first")
|
|
2044
|
+
try:
|
|
2045
|
+
provenance = json.loads((runtime / "competitor-set-inputs.json").read_text())
|
|
2046
|
+
except (OSError, json.JSONDecodeError):
|
|
2047
|
+
provenance = {}
|
|
2048
|
+
competitor_context = {r["domain"]: {**r, "source_sha256": provenance.get("source_sha256"), "source_path": _artifact_path("egbe-docs/research/market/evidence/competitors.json")} for r in provenance.get("inputs", []) if r.get("domain") in domains}
|
|
2049
|
+
for domain in domains:
|
|
2050
|
+
competitor_context.setdefault(domain, {"domain": domain, "competitor_tier": "unknown", "reason": "no approved Competitor Set role", "source_path": None, "source_sha256": None})
|
|
2051
|
+
intake_path = workspace / _artifact_path("egbe-docs/research/market/intake.json")
|
|
2052
|
+
intake = typed_intake(workspace)
|
|
2053
|
+
intake_hash = IO.sha256(intake_path) if intake_path.is_file() else None
|
|
2054
|
+
research_context = IO.context_binding(workspace, SECTION_ID)
|
|
2055
|
+
context = IO.context_text(workspace, SECTION_ID)
|
|
2056
|
+
geography = (_arg(args, "countries", "") or _arg(args, "country", "")
|
|
2057
|
+
or intake.get("markets") or [DEFAULT_COUNTRY])
|
|
2058
|
+
target_countries = _model().countries(geography)
|
|
2059
|
+
context_hash = _model().digest({"context_sha256": research_context["sha256"], "intake_sha256": intake_hash, "competitor_context": competitor_context, "product_domain": product_domain, "target_countries": target_countries})
|
|
2060
|
+
relevance_inputs = relevance_context(workspace, args) if _arg(args, "relevance_by_model", False) else None
|
|
2061
|
+
decision_path = Path(_arg(args, "relevance_decisions", "") or runtime / "relevance-decisions.json")
|
|
2062
|
+
try:
|
|
2063
|
+
decisions = json.loads(decision_path.read_text()) if decision_path.is_file() else {}
|
|
2064
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
2065
|
+
raise ChannelFootprintError(f"invalid relevance decisions: {exc}") from exc
|
|
2066
|
+
if decisions and decisions.get("product_context_sha256") != context_hash:
|
|
2067
|
+
if stage == "finalize":
|
|
2068
|
+
raise ChannelFootprintError("relevance decisions belong to a different product/intake context; collect the new candidates")
|
|
2069
|
+
decisions = {} # retain the old input on disk; expose new context-bound candidates
|
|
2070
|
+
inputs = RunInputs(domains=domains, domain_sources=domain_sources, product_domain=product_domain, sources=sources,
|
|
2071
|
+
country=target_countries[0], eu_country=str(_arg(args, "eu_country", "") or ""), pixel_actor_fallback=bool(_arg(args, "pixel_actor_fallback", False)),
|
|
2072
|
+
target_countries=target_countries, competitor_context=competitor_context, product_context_sha256=context_hash,
|
|
2073
|
+
relevance_decisions=decisions, effective_policy=policy, stage=stage)
|
|
2074
|
+
plan = collection_plan([(domain, competitor_context[domain]["competitor_tier"]) for domain in domains], sources, target_countries,
|
|
2075
|
+
pixel_actor_fallback=inputs.pixel_actor_fallback)
|
|
2076
|
+
budget = paid_start_budget(plan, config)
|
|
2077
|
+
max_total = int(_arg(args, "max_total_paid_starts", budget["cumulative_limit"]))
|
|
2078
|
+
if max_total < 0:
|
|
2079
|
+
raise ChannelFootprintError("total paid-start budget must be nonnegative")
|
|
2080
|
+
plan_doc = {"schema_version": 1, "input_context_sha256": context_hash, "policy_sha256": policy_hash,
|
|
2081
|
+
"base_paid_starts_upper_bound": budget["base"], "bounded_paid_starts_upper_bound": budget["bounded"], "retry_reserve_factor": budget["retry_reserve_factor"], "cumulative_limit": max_total, "money_estimate": None, "checks": plan}
|
|
2082
|
+
_write_atomic(runtime / "collection-plan.json", json.dumps(plan_doc, indent=1))
|
|
2083
|
+
collected_path = runtime / "collected.json"
|
|
2084
|
+
if stage == "finalize":
|
|
2085
|
+
try:
|
|
2086
|
+
stored = json.loads(collected_path.read_text())
|
|
2087
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
2088
|
+
raise ChannelFootprintError(f"collect the section before finalize: {exc}") from exc
|
|
2089
|
+
if not _collection_matches(stored, context_hash, policy_hash, sources, config):
|
|
2090
|
+
raise ChannelFootprintError("collected data context or policy changed; rerun collect using compatible provider cache")
|
|
2091
|
+
outcome = finalize_pipeline(stored["raw"], inputs, config, date.fromisoformat(stored["captured_on"]))
|
|
2092
|
+
else:
|
|
2093
|
+
api_key = (os.environ.get("OS_API_KEY") or "").strip()
|
|
2094
|
+
if not api_key:
|
|
2095
|
+
raise ChannelFootprintError("OS_API_KEY is required for collection")
|
|
2096
|
+
os_url = (os.environ.get("OS_URL") or DEFAULT_OS_URL).strip() or DEFAULT_OS_URL
|
|
2097
|
+
client = OsClient(
|
|
2098
|
+
os_url=os_url,
|
|
2099
|
+
api_key=api_key,
|
|
2100
|
+
cache_dir=cache_dir,
|
|
2101
|
+
max_actor_runs=int(_arg(args, "max_actor_runs", MAX_ACTOR_RUNS_PER_ATTEMPT)),
|
|
2102
|
+
max_meta_searches=int(_arg(args, "max_meta_searches", MAX_META_SEARCHES_PER_ATTEMPT)),
|
|
2103
|
+
max_search_calls=int(_arg(args, "max_search_calls", MAX_SEARCH_CALLS_PER_ATTEMPT)),
|
|
2104
|
+
max_fetches=int(_arg(args, "max_fetches", MAX_FETCHES_PER_ATTEMPT)),
|
|
2105
|
+
max_cdx_calls=int(_arg(args, "max_cdx_calls", MAX_CDX_CALLS_PER_ATTEMPT)),
|
|
2106
|
+
max_spyfu_calls=int(_arg(args, "max_spyfu_calls", MAX_SPYFU_CALLS_PER_ATTEMPT)),
|
|
2107
|
+
max_total_paid_starts=max_total, cumulative_budget_path=runtime / "budget.json",
|
|
2108
|
+
)
|
|
2109
|
+
|
|
2110
|
+
try:
|
|
2111
|
+
previous = json.loads(collected_path.read_text())
|
|
2112
|
+
except (OSError, json.JSONDecodeError):
|
|
2113
|
+
previous = {}
|
|
2114
|
+
if _collection_matches(previous, context_hash, policy_hash, sources, config):
|
|
2115
|
+
inputs.prior_expansions = (previous.get("raw") or {}).get("expansions") or []
|
|
2116
|
+
outcome = run_pipeline(client, inputs, config, date.today())
|
|
2117
|
+
if _arg(args, "relevance_by_model", False):
|
|
2118
|
+
product_context, languages = relevance_inputs
|
|
2119
|
+
for _ in range(MODEL_RELEVANCE_ROUNDS):
|
|
2120
|
+
decided = model_relevance(client, outcome["corpus"], outcome["pending_relevance"], decisions, context_hash, product_context, languages)
|
|
2121
|
+
if decided is None:
|
|
2122
|
+
break
|
|
2123
|
+
decisions = inputs.relevance_decisions = decided
|
|
2124
|
+
_write_atomic(decision_path, json.dumps(decided, indent=1, ensure_ascii=False))
|
|
2125
|
+
# A collected expansion is replayed in the next pass, as on a rerun.
|
|
2126
|
+
inputs.prior_expansions = outcome["raw"]["expansions"] + [e for e in inputs.prior_expansions if e not in outcome["raw"]["expansions"]]
|
|
2127
|
+
outcome = run_pipeline(client, inputs, config, date.today())
|
|
2128
|
+
_write_atomic(collected_path, json.dumps({"schema_version": 2, "input_context_sha256": context_hash, "policy_sha256": policy_hash, "sources": sources, "sources_config_sha256": _model().digest(config), "captured_on": date.today().isoformat(), "raw_sha256": _model().digest(outcome["raw"]), "raw": outcome["raw"]}, indent=1))
|
|
2129
|
+
no_observations_budget_deferred = outcome["totals"]["observations"] == 0 and any(r.get("status") == "budget_deferred" for r in outcome["raw"]["coverage"])
|
|
2130
|
+
if outcome["totals"]["observations"] == 0 and not no_observations_budget_deferred:
|
|
2131
|
+
raise ChannelFootprintError(
|
|
2132
|
+
"no source answered for any competitor (every provider call was rejected or failed); the section "
|
|
2133
|
+
"cannot complete without at least one successful observation. Rerun later or change --sources."
|
|
2134
|
+
)
|
|
2135
|
+
|
|
2136
|
+
candidates = {"schema_version": 1, "product_context_sha256": context_hash, "policy_sha256": policy_hash,
|
|
2137
|
+
"product_context": {"context": context, "intake": intake, "product_domain": product_domain, "competitor_context": competitor_context},
|
|
2138
|
+
"candidates": outcome["corpus"], "pending_ad_ids": outcome["pending_relevance"]}
|
|
2139
|
+
_write_atomic(runtime / "relevance-candidates.json", json.dumps(candidates, indent=1, ensure_ascii=False))
|
|
2140
|
+
if outcome["pending_relevance"] or not outcome["collection_complete"] or stage == "collect" or no_observations_budget_deferred:
|
|
2141
|
+
status = "budget_deferred" if no_observations_budget_deferred else "awaiting_relevance" if outcome["pending_relevance"] else ("awaiting_collection" if not outcome["collection_complete"] else "awaiting_finalize")
|
|
2142
|
+
state = {"schema_version": 1, "section_id": SECTION_ID, "status": status, "domains": domains, "target_countries": target_countries,
|
|
2143
|
+
"research_context": research_context,
|
|
2144
|
+
"input_context_sha256": context_hash, "policy_sha256": policy_hash, "pending_ad_ids": outcome["pending_relevance"], "coverage": outcome["raw"]["coverage"],
|
|
2145
|
+
"candidates_path": (RUNTIME_REL / "relevance-candidates.json").as_posix(), "collected_path": (RUNTIME_REL / "collected.json").as_posix(), "error": None}
|
|
2146
|
+
_write_atomic(runtime / "request.json", json.dumps(state, indent=1))
|
|
2147
|
+
return state
|
|
2148
|
+
|
|
2149
|
+
evidence_path = workspace / EVIDENCE_REL
|
|
2150
|
+
evidence_json = json.dumps(outcome["evidence"], indent=1, ensure_ascii=False)
|
|
2151
|
+
_write_atomic(evidence_path, evidence_json)
|
|
2152
|
+
section_file = runtime / "result.md"
|
|
2153
|
+
_write_atomic(section_file, outcome["section_markdown"])
|
|
2154
|
+
|
|
2155
|
+
upsert = _load_script("upsert-markdown-section.py", "pmr_upsert_markdown_section")
|
|
2156
|
+
report_path = workspace / REPORT_REL
|
|
2157
|
+
report_path.parent.mkdir(parents=True, exist_ok=True)
|
|
2158
|
+
upsert.upsert_markdown_section(report_path, SECTION_HEADING, outcome["section_markdown"])
|
|
2159
|
+
|
|
2160
|
+
persist = _load_script("persist-research-report.py", "pmr_persist_research_report")
|
|
2161
|
+
persistence = persist.persist_research_report(
|
|
2162
|
+
workspace,
|
|
2163
|
+
commit_message=f"research: update {SECTION_HEADING.lower()} section",
|
|
2164
|
+
extra_paths=(Path(_artifact_path("egbe-docs/research/market/evidence/channel-footprint.json")),),
|
|
2165
|
+
)
|
|
2166
|
+
|
|
2167
|
+
state = {
|
|
2168
|
+
"schema_version": 1,
|
|
2169
|
+
"section_id": SECTION_ID,
|
|
2170
|
+
"status": "completed",
|
|
2171
|
+
"domains": inputs.domains,
|
|
2172
|
+
"domain_sources": inputs.domain_sources,
|
|
2173
|
+
"sources": inputs.sources,
|
|
2174
|
+
"country": inputs.country,
|
|
2175
|
+
"eu_country": inputs.eu_country,
|
|
2176
|
+
"target_countries": inputs.target_countries,
|
|
2177
|
+
"input_context_sha256": context_hash,
|
|
2178
|
+
"intake_sha256": intake_hash,
|
|
2179
|
+
"competitor_set_sha256": provenance.get("source_sha256"),
|
|
2180
|
+
"policy_sha256": policy_hash,
|
|
2181
|
+
"effective_policy": policy,
|
|
2182
|
+
"relevance_decisions_sha256": _model().digest(decisions),
|
|
2183
|
+
"sources_config_sha256": _model().digest(config),
|
|
2184
|
+
"evidence_path": EVIDENCE_REL.as_posix(),
|
|
2185
|
+
"evidence_sha256": hashlib.sha256(evidence_json.encode("utf-8")).hexdigest(),
|
|
2186
|
+
"result_path": (RUNTIME_REL / "result.md").as_posix(),
|
|
2187
|
+
"result_sha256": hashlib.sha256(outcome["section_markdown"].encode("utf-8")).hexdigest(),
|
|
2188
|
+
"provider_calls": outcome["totals"]["provider_calls"],
|
|
2189
|
+
"actor_runs": outcome["totals"]["actor_runs"],
|
|
2190
|
+
"meta_searches": outcome["totals"]["meta_searches"],
|
|
2191
|
+
"search_calls": outcome["totals"]["search_calls"],
|
|
2192
|
+
"fetches": outcome["totals"]["fetches"],
|
|
2193
|
+
"cdx_calls": outcome["totals"]["cdx_calls"],
|
|
2194
|
+
"spyfu_calls": outcome["totals"]["spyfu_calls"],
|
|
2195
|
+
"relevance_calls": outcome["totals"].get("relevance_calls", 0),
|
|
2196
|
+
"rejected_actor_runs": outcome["totals"]["rejected_actor_runs"],
|
|
2197
|
+
"rejected_spyfu_calls": outcome["totals"]["rejected_spyfu_calls"],
|
|
2198
|
+
"observations": outcome["totals"]["observations"],
|
|
2199
|
+
"actor_starts_recorded": outcome["totals"]["actor_starts_recorded"],
|
|
2200
|
+
"attempt_actor_runs": outcome["attempt"]["actor_runs"],
|
|
2201
|
+
"attempt_meta_searches": outcome["attempt"]["meta_searches"],
|
|
2202
|
+
"attempt_search_calls": outcome["attempt"]["search_calls"],
|
|
2203
|
+
"attempt_fetches": outcome["attempt"]["fetches"],
|
|
2204
|
+
"attempt_cdx_calls": outcome["attempt"]["cdx_calls"],
|
|
2205
|
+
"attempt_cache_hits": outcome["attempt"]["cache_hits"],
|
|
2206
|
+
"report_commit_sha": persistence.commit_sha,
|
|
2207
|
+
"publication_mode": persistence.publication_mode,
|
|
2208
|
+
"research_context": _load_script("research-artifact-io.py", "pmr_input_io").context_binding(workspace, SECTION_ID),
|
|
2209
|
+
"error": None,
|
|
2210
|
+
}
|
|
2211
|
+
_write_atomic(runtime / "request.json", json.dumps(state, indent=1))
|
|
2212
|
+
return state
|
|
2213
|
+
|
|
2214
|
+
|
|
2215
|
+
def main() -> int:
|
|
2216
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
2217
|
+
parser.add_argument("--context-file", type=Path, help="Explicit context for this study; no project files are discovered implicitly.")
|
|
2218
|
+
parser.add_argument("--workspace", type=Path, required=True)
|
|
2219
|
+
parser.add_argument("--domains", default="", help="Comma-separated competitor domains; merged ahead of the Competitor Set and Keyword Economics domains.")
|
|
2220
|
+
parser.add_argument("--product-domain", default="", help="The product's own domain; never scanned as a competitor.")
|
|
2221
|
+
parser.add_argument("--sources", default="", help=f"Comma-separated subset of {', '.join(ALL_SOURCES)} (default: all).")
|
|
2222
|
+
parser.add_argument("--sources-config", default="", help="Path to an alternative sources config JSON (actor ids and batch sizes).")
|
|
2223
|
+
parser.add_argument("--max-competitors", type=int, default=MAX_COMPETITORS)
|
|
2224
|
+
parser.add_argument("--country", default="", help="Explicit single target country; otherwise typed intake markets, then US.")
|
|
2225
|
+
parser.add_argument("--eu-country", default="", help="Deprecated compatibility argument; target markets come from intake/--countries.")
|
|
2226
|
+
parser.add_argument("--countries", "--target-countries", default="", help="Explicit ISO target countries; otherwise --country or typed intake markets.")
|
|
2227
|
+
parser.add_argument("--languages", default="", help="Explicit accepted language codes for relevance classification; otherwise typed intake languages.")
|
|
2228
|
+
parser.add_argument("--stage", choices=("all", "collect", "finalize"), default="all")
|
|
2229
|
+
parser.add_argument("--relevance-decisions", default="", help="Context-bound decisions JSON (default runtime/relevance-decisions.json).")
|
|
2230
|
+
parser.add_argument("--relevance-by-model", action="store_true", help="Decide ads with text by the creative classifier's relevance verdict; the rest stay for review.")
|
|
2231
|
+
parser.add_argument("--max-total-paid-starts", type=int, default=None, help="Durable section cap across every retry and rerun.")
|
|
2232
|
+
parser.add_argument("--pixel-actor-fallback", action="store_true", help="Use the headless tag-detector actor for sites whose HTML cannot be fetched directly (one actor run per batch).")
|
|
2233
|
+
parser.add_argument("--max-actor-runs", type=int, default=MAX_ACTOR_RUNS_PER_ATTEMPT)
|
|
2234
|
+
parser.add_argument("--max-meta-searches", type=int, default=MAX_META_SEARCHES_PER_ATTEMPT)
|
|
2235
|
+
parser.add_argument("--max-search-calls", type=int, default=MAX_SEARCH_CALLS_PER_ATTEMPT)
|
|
2236
|
+
parser.add_argument("--max-fetches", type=int, default=MAX_FETCHES_PER_ATTEMPT)
|
|
2237
|
+
parser.add_argument("--max-cdx-calls", type=int, default=MAX_CDX_CALLS_PER_ATTEMPT)
|
|
2238
|
+
parser.add_argument("--max-spyfu-calls", type=int, default=MAX_SPYFU_CALLS_PER_ATTEMPT, help="SpyFu domain-stats lookups per attempt (one per competitor).")
|
|
2239
|
+
args = parser.parse_args()
|
|
2240
|
+
|
|
2241
|
+
try:
|
|
2242
|
+
state = run(args.workspace, args)
|
|
2243
|
+
except (ChannelFootprintError, ValueError) as exc:
|
|
2244
|
+
print(f"{DISPLAY_NAME}: {exc}", file=sys.stderr)
|
|
2245
|
+
return 1
|
|
2246
|
+
if state["status"] != "completed":
|
|
2247
|
+
print(f"{DISPLAY_NAME}: {state['status']} — {state.get('candidates_path', '')}; review decisions, then collect/finalize")
|
|
2248
|
+
return 2
|
|
2249
|
+
print(
|
|
2250
|
+
f"{DISPLAY_NAME}: completed — {len(state['domains'])} competitor(s), {state['actor_runs']} actor run(s), "
|
|
2251
|
+
f"{state['meta_searches']} Meta library search(es), {state['search_calls']} web search(es), evidence at "
|
|
2252
|
+
f"{state['evidence_path']}, report commit {(state['report_commit_sha'] or '')[:12] or 'n/a'}"
|
|
2253
|
+
)
|
|
2254
|
+
return 0
|
|
2255
|
+
|
|
2256
|
+
|
|
2257
|
+
if __name__ == "__main__":
|
|
2258
|
+
raise SystemExit(main())
|