@egbe-ai/opencode 0.0.0-stage → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +124 -2
- package/index.js +5 -0
- package/package.json +55 -4
- package/plugin.js +230 -0
- package/session-log.js +315 -0
- package/skills/.egbe-catalog/manifest.json +1 -0
- package/skills/egbe-os-analytics/SKILL.md +499 -0
- package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
- package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
- package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
- package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
- package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
- package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
- package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
- package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
- package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
- package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
- package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
- package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
- package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
- package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
- package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
- package/skills/egbe-os-db/SKILL.md +174 -0
- package/skills/egbe-os-deploy/SKILL.md +412 -0
- package/skills/egbe-os-design/SKILL.md +182 -0
- package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
- package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
- package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
- package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
- package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
- package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
- package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
- package/skills/egbe-os-destination-readiness/references/report.md +116 -0
- package/skills/egbe-os-domains/SKILL.md +189 -0
- package/skills/egbe-os-errors/SKILL.md +259 -0
- package/skills/egbe-os-git/SKILL.md +275 -0
- package/skills/egbe-os-git/references/issues.md +33 -0
- package/skills/egbe-os-google-ads/SKILL.md +163 -0
- package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
- package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
- package/skills/egbe-os-google-ads/references/conversions.md +153 -0
- package/skills/egbe-os-google-ads/references/delivery.md +128 -0
- package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
- package/skills/egbe-os-google-ads/references/reporting.md +105 -0
- package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
- package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
- package/skills/egbe-os-google-auth/SKILL.md +285 -0
- package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
- package/skills/egbe-os-images/SKILL.md +163 -0
- package/skills/egbe-os-images/references/stock-images.md +21 -0
- package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
- package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
- package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
- package/skills/egbe-os-llm/SKILL.md +160 -0
- package/skills/egbe-os-mail/SKILL.md +180 -0
- package/skills/egbe-os-me/SKILL.md +119 -0
- package/skills/egbe-os-meta-ads/SKILL.md +114 -0
- package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
- package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
- package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
- package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
- package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
- package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
- package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
- package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
- package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
- package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
- package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
- package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
- package/skills/egbe-os-notifications/SKILL.md +78 -0
- package/skills/egbe-os-openai-ads/SKILL.md +149 -0
- package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
- package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
- package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
- package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
- package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
- package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
- package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
- package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
- package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
- package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
- package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
- package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
- package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
- package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
- package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
- package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
- package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
- package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
- package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
- package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
- package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
- package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
- package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
- package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
- package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
- package/skills/egbe-os-payments/SKILL.md +215 -0
- package/skills/egbe-os-payments/agents/openai.yaml +4 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
- package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
- package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
- package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
- package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
- package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
- package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
- package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
- package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
- package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
- package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
- package/skills/egbe-os-playbook/SKILL.md +79 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
- package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
- package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
- package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
- package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
- package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
- package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
- package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
- package/skills/egbe-os-playbook/playbooks/product.md +67 -0
- package/skills/egbe-os-playbook/references/catalog.md +127 -0
- package/skills/egbe-os-playbook/references/changes.md +167 -0
- package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
- package/skills/egbe-os-product-auth/SKILL.md +151 -0
- package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
- package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
- package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
- package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
- package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
- package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
- package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
- package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
- package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
- package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
- package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
- package/skills/egbe-os-product-build/SKILL.md +111 -0
- package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-build/references/capabilities.md +25 -0
- package/skills/egbe-os-product-build/references/engineering.md +72 -0
- package/skills/egbe-os-product-build/references/execution.md +55 -0
- package/skills/egbe-os-product-build/references/foundations.md +93 -0
- package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
- package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
- package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
- package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
- package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
- package/skills/egbe-os-product-market-research/SKILL.md +159 -0
- package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
- package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
- package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
- package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
- package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
- package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
- package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
- package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
- package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
- package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
- package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
- package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
- package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
- package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
- package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
- package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
- package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
- package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
- package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
- package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
- package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
- package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
- package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
- package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
- package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
- package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
- package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
- package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
- package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
- package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
- package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
- package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
- package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
- package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
- package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
- package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
- package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
- package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
- package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
- package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
- package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
- package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
- package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
- package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
- package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
- package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
- package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
- package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
- package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
- package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
- package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
- package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
- package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
- package/skills/egbe-os-product-report/SKILL.md +150 -0
- package/skills/egbe-os-product-report/references/artifacts.md +84 -0
- package/skills/egbe-os-product-report/references/brief.md +34 -0
- package/skills/egbe-os-product-report/references/daily.md +76 -0
- package/skills/egbe-os-product-report/references/sources.md +68 -0
- package/skills/egbe-os-product-routines/SKILL.md +166 -0
- package/skills/egbe-os-product-routines/references/catalog.md +22 -0
- package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
- package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
- package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
- package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
- package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
- package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
- package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
- package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
- package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
- package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
- package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
- package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
- package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
- package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
- package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
- package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
- package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
- package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
- package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
- package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
- package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
- package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
- package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
- package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
- package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
- package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
- package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
- package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
- package/skills/egbe-os-report-ui/SKILL.md +133 -0
- package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
- package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
- package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
- package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
- package/skills/egbe-os-report-ui/references/components.md +209 -0
- package/skills/egbe-os-research/SKILL.md +125 -0
- package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
- package/skills/egbe-os-scrape/SKILL.md +160 -0
- package/skills/egbe-os-search/SKILL.md +182 -0
- package/skills/egbe-os-search/references/google-places.md +81 -0
- package/skills/egbe-os-support/SKILL.md +149 -0
- package/skills/egbe-os-telemetry/SKILL.md +344 -0
- package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
- package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
- package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
- package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
- package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
- package/skills/egbe-os-tools/SKILL.md +180 -0
- package/skills/egbe-os-tools/references/theona.md +26 -0
- package/skills/egbe-os-wallet/SKILL.md +181 -0
- package/skills/egbe-setup/EGBE.md +484 -0
- package/skills/egbe-setup/SKILL.md +116 -0
- package/skills/egbe-setup/references/project.md +80 -0
- package/state.js +37 -0
- package/update.js +361 -0
|
@@ -0,0 +1,1156 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Run the deterministic Competitor Ads section against ad-transparency data.
|
|
3
|
+
|
|
4
|
+
Collect scoped ad-library observations, identity, native dates and coverage.
|
|
5
|
+
Advertising duration does not establish spend, conversion or profitability.
|
|
6
|
+
|
|
7
|
+
Providers (all reached through the OS gateway with the claw's own key):
|
|
8
|
+
- Google Ads Transparency via the Apify actor
|
|
9
|
+
`scrapesage/google-ads-transparency-scraper` (domain-keyed: the
|
|
10
|
+
actor's `domains` input returns the creatives whose destination is the
|
|
11
|
+
competitor's site, with firstShown/lastShown/numServedDays);
|
|
12
|
+
- Meta Ads Library via the Apify actor
|
|
13
|
+
`curious_coder/facebook-ads-library-scraper` (US coverage, ad start/end
|
|
14
|
+
timestamps, creative texts);
|
|
15
|
+
- the official Meta Ad Library endpoint (`/tools/meta-ad-library/search`)
|
|
16
|
+
for EU delivery data and reach.
|
|
17
|
+
|
|
18
|
+
Competitor domains come from explicit `--domains`, then the completed
|
|
19
|
+
Competitor Set, then the Keyword Economics evidence file (the
|
|
20
|
+
`competitor_domain` rows discovered on the demand surface). Identity is
|
|
21
|
+
destination-first: Google rows are collected by destination domain (the
|
|
22
|
+
actor's `domains` input); Meta rows are kept when their landing destination
|
|
23
|
+
resolves to the competitor's domain or, when the ad carries no destination,
|
|
24
|
+
when the page name is an exact brand match (`verified_by: exact_name`);
|
|
25
|
+
contains-only namesakes and foreign destinations are excluded and reported
|
|
26
|
+
separately.
|
|
27
|
+
|
|
28
|
+
Caching and spend provenance mirror the Keyword Economics runner: every
|
|
29
|
+
provider response is cached under the runtime directory keyed by request
|
|
30
|
+
hash, re-runs replay for free, and durable totals come from the cache ledger
|
|
31
|
+
so retries stay byte-stable.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import sys as _artifact_sys
|
|
37
|
+
from pathlib import Path as _ArtifactPath
|
|
38
|
+
_artifact_sys.path.insert(0, str(_ArtifactPath(__file__).resolve().parent))
|
|
39
|
+
from egbe_artifacts import artifact_path as _artifact_path
|
|
40
|
+
|
|
41
|
+
import runpy as _egbe_runpy
|
|
42
|
+
from pathlib import Path as _EgbePath
|
|
43
|
+
_egbe_adapter = _EgbePath(__file__).with_name("connection_identity.py")
|
|
44
|
+
_connection_headers = _egbe_runpy.run_path(str(_egbe_adapter))["connection_headers"] if _egbe_adapter.is_file() else lambda _url: {}
|
|
45
|
+
|
|
46
|
+
import argparse
|
|
47
|
+
import hashlib
|
|
48
|
+
import importlib.util
|
|
49
|
+
import json
|
|
50
|
+
import os
|
|
51
|
+
import re
|
|
52
|
+
import sys
|
|
53
|
+
import time
|
|
54
|
+
import urllib.error
|
|
55
|
+
import urllib.request
|
|
56
|
+
from dataclasses import dataclass
|
|
57
|
+
from datetime import date, datetime, timezone
|
|
58
|
+
from pathlib import Path
|
|
59
|
+
from types import ModuleType
|
|
60
|
+
from typing import Any, Callable
|
|
61
|
+
from urllib.parse import quote
|
|
62
|
+
|
|
63
|
+
SCHEMA_VERSION = 1
|
|
64
|
+
SECTION_ID = "competitor_ads"
|
|
65
|
+
SECTION_HEADING = "Competitor ads"
|
|
66
|
+
DISPLAY_NAME = "Competitor Ads"
|
|
67
|
+
|
|
68
|
+
DEFAULT_OS_URL = "https://os.hz.egbe.dev"
|
|
69
|
+
RUNTIME_REL = Path(_artifact_path("egbe-docs/research/market/work")) / SECTION_ID
|
|
70
|
+
EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/competitor-ads.json"))
|
|
71
|
+
REPORT_REL = Path(_artifact_path("egbe-docs/research/market/full-report.md"))
|
|
72
|
+
|
|
73
|
+
# Google collection is DOMAIN-keyed: the actor's `domains` input returns every
|
|
74
|
+
# advertiser account whose ads point at the target domain (validated live:
|
|
75
|
+
# clickup.com -> "Mango Technologies, Inc.", ClickUp's legal entity, with zero
|
|
76
|
+
# same-name lookalikes — name search returned only "CLICKUP LEGALTECH
|
|
77
|
+
# SERVICES SL"). Identity comes from ad destination, never advertiser names.
|
|
78
|
+
GOOGLE_ACTOR = "scrapesage/google-ads-transparency-scraper"
|
|
79
|
+
META_ACTOR = "curious_coder/facebook-ads-library-scraper"
|
|
80
|
+
|
|
81
|
+
# Every Apify actor run places a wallet hold on the claw, so the per-attempt
|
|
82
|
+
# budget is deliberately tight; cache hits are free.
|
|
83
|
+
MAX_ACTOR_RUNS_PER_ATTEMPT = 6
|
|
84
|
+
MAX_META_SEARCHES_PER_ATTEMPT = 8
|
|
85
|
+
# Each competitor costs two actor runs (Google + Meta), so the default
|
|
86
|
+
# competitor count is sized to the actor budget; run() trims further if the
|
|
87
|
+
# caller lowers --max-actor-runs.
|
|
88
|
+
MAX_COMPETITORS = 3
|
|
89
|
+
GOOGLE_ADS_PER_COMPETITOR = 60
|
|
90
|
+
META_ADS_PER_COMPETITOR = 25
|
|
91
|
+
STORED_ADS_PER_DOMAIN = 15
|
|
92
|
+
|
|
93
|
+
# Synchronous actor runs occasionally die with a transient 5xx at the edge
|
|
94
|
+
# while the upstream run itself is fine. One bounded retry, carrying a
|
|
95
|
+
# deterministic Idempotency-Key derived from the request hash, lets the
|
|
96
|
+
# gateway reuse the same upstream run instead of paying for a duplicate.
|
|
97
|
+
ACTOR_RETRY_DELAY_SECONDS = 20.0
|
|
98
|
+
|
|
99
|
+
_PRICE_RE = re.compile(
|
|
100
|
+
r"(\$\s?\d+(?:[.,]\d{1,2})?|\d+\s?%\s?off|per\s+month|/mo\b|a\s+month)",
|
|
101
|
+
re.IGNORECASE,
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class CompetitorAdsError(RuntimeError):
|
|
106
|
+
"""Terminal runner failure; runtime state is preserved for resume."""
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _load_script(filename: str, module_name: str) -> ModuleType:
|
|
110
|
+
path = Path(__file__).resolve().parent / filename
|
|
111
|
+
spec = importlib.util.spec_from_file_location(module_name, path)
|
|
112
|
+
if spec is None or spec.loader is None:
|
|
113
|
+
raise CompetitorAdsError(f"cannot load bundled script {filename}")
|
|
114
|
+
module = importlib.util.module_from_spec(spec)
|
|
115
|
+
sys.modules[spec.name] = module
|
|
116
|
+
previous = sys.dont_write_bytecode
|
|
117
|
+
sys.dont_write_bytecode = True
|
|
118
|
+
try:
|
|
119
|
+
spec.loader.exec_module(module)
|
|
120
|
+
finally:
|
|
121
|
+
sys.dont_write_bytecode = previous
|
|
122
|
+
return module
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
POLICY = _load_script("research-policy.py", "pmr_policy_for_competitor_ads")
|
|
126
|
+
LONG_RUNNING_DAYS = None # Retained call parameter; no duration-to-success classification.
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# ── OS client: generic POST with cache ledger and per-kind budgets ─────────
|
|
130
|
+
|
|
131
|
+
@dataclass
|
|
132
|
+
class OsClient:
|
|
133
|
+
os_url: str
|
|
134
|
+
api_key: str
|
|
135
|
+
cache_dir: Path
|
|
136
|
+
max_actor_runs: int = MAX_ACTOR_RUNS_PER_ATTEMPT
|
|
137
|
+
max_meta_searches: int = MAX_META_SEARCHES_PER_ATTEMPT
|
|
138
|
+
actor_runs: int = 0
|
|
139
|
+
meta_searches: int = 0
|
|
140
|
+
cache_hits: int = 0
|
|
141
|
+
touched: set[str] = None # type: ignore[assignment] # set in __post_init__
|
|
142
|
+
request: Callable[..., dict[str, Any]] | None = None
|
|
143
|
+
sleep: Callable[[float], None] | None = None
|
|
144
|
+
|
|
145
|
+
def __post_init__(self) -> None:
|
|
146
|
+
if self.touched is None:
|
|
147
|
+
self.touched = set()
|
|
148
|
+
|
|
149
|
+
def _http(self, path: str, payload: dict[str, Any], timeout: float, *, idempotency_key: str | None = None) -> dict[str, Any]:
|
|
150
|
+
url = f"{self.os_url.rstrip('/')}{path}"
|
|
151
|
+
headers = {
|
|
152
|
+
"Authorization": f"Bearer {self.api_key}",
|
|
153
|
+
"Content-Type": "application/json",
|
|
154
|
+
"Accept": "application/json",
|
|
155
|
+
}
|
|
156
|
+
if idempotency_key:
|
|
157
|
+
headers["Idempotency-Key"] = idempotency_key
|
|
158
|
+
req = urllib.request.Request(
|
|
159
|
+
url,
|
|
160
|
+
data=json.dumps(payload).encode("utf-8"),
|
|
161
|
+
headers=headers,
|
|
162
|
+
method="POST",
|
|
163
|
+
)
|
|
164
|
+
try:
|
|
165
|
+
for _name, _value in _connection_headers(req.full_url).items():
|
|
166
|
+
req.add_unredirected_header(_name, _value)
|
|
167
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
168
|
+
envelope = json.loads(resp.read().decode("utf-8"))
|
|
169
|
+
except urllib.error.HTTPError as exc:
|
|
170
|
+
detail = exc.read().decode("utf-8", errors="replace")[:500]
|
|
171
|
+
raise CompetitorAdsError(f"OS {path} failed: HTTP {exc.code} {detail}") from exc
|
|
172
|
+
except urllib.error.URLError as exc:
|
|
173
|
+
raise CompetitorAdsError(f"OS {path} unreachable: {exc.reason}") from exc
|
|
174
|
+
data = envelope.get("data")
|
|
175
|
+
if envelope.get("code") not in (None, 200) or not isinstance(data, dict):
|
|
176
|
+
raise CompetitorAdsError(
|
|
177
|
+
f"OS {path} returned no data: {str(envelope)[:300]}"
|
|
178
|
+
)
|
|
179
|
+
return data
|
|
180
|
+
|
|
181
|
+
def _cached(self, kind: str, path: str, payload: dict[str, Any], timeout: float, budget_check: Callable[[], None], count: Callable[[], None], retry_transient: bool = False) -> tuple[dict[str, Any], str]:
|
|
182
|
+
"""Return (response, retrieved_on). Paid starts are counted BEFORE the
|
|
183
|
+
transport call (fail closed): an ambiguous sync failure may still have
|
|
184
|
+
started a billable upstream run, so a retry consumes budget exactly
|
|
185
|
+
like a first attempt."""
|
|
186
|
+
key_raw = json.dumps({"kind": kind, "path": path, "payload": payload}, sort_keys=True)
|
|
187
|
+
digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
|
|
188
|
+
cache_file = self.cache_dir / f"{digest}.json"
|
|
189
|
+
self.touched.add(digest)
|
|
190
|
+
if cache_file.is_file():
|
|
191
|
+
try:
|
|
192
|
+
entry = json.loads(cache_file.read_text(encoding="utf-8"))
|
|
193
|
+
except (OSError, json.JSONDecodeError):
|
|
194
|
+
# A torn write must not poison replay: discard and refetch.
|
|
195
|
+
cache_file.unlink(missing_ok=True)
|
|
196
|
+
entry = None
|
|
197
|
+
if entry is not None:
|
|
198
|
+
self.cache_hits += 1
|
|
199
|
+
return entry["response"], entry.get("retrieved_on") or date.today().isoformat()
|
|
200
|
+
transport = self.request or self._http
|
|
201
|
+
attempts = 0
|
|
202
|
+
# Salted ONCE per call, before the retry loop: a within-call
|
|
203
|
+
# transient-5xx retry must reuse the SAME wire key (so the gateway
|
|
204
|
+
# can dedupe an ambiguous sync failure into one upstream run), but a
|
|
205
|
+
# fresh call — a prior attempt having already failed and journaled a
|
|
206
|
+
# start for this digest — must get a NEW key. The cache key (digest)
|
|
207
|
+
# stays the pure payload hash so replay lookups are unaffected.
|
|
208
|
+
wire_key = f"{digest}-{self._prior_start_count(digest)}"
|
|
209
|
+
while True:
|
|
210
|
+
budget_check()
|
|
211
|
+
count()
|
|
212
|
+
self._journal_start(kind, digest)
|
|
213
|
+
try:
|
|
214
|
+
response = transport(path, payload, timeout, idempotency_key=wire_key)
|
|
215
|
+
break
|
|
216
|
+
except CompetitorAdsError as exc:
|
|
217
|
+
if (
|
|
218
|
+
not retry_transient
|
|
219
|
+
or attempts >= 1
|
|
220
|
+
or not re.search(r"HTTP 5\d\d", str(exc))
|
|
221
|
+
):
|
|
222
|
+
raise
|
|
223
|
+
attempts += 1
|
|
224
|
+
(self.sleep or time.sleep)(ACTOR_RETRY_DELAY_SECONDS)
|
|
225
|
+
retrieved_on = date.today().isoformat()
|
|
226
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
227
|
+
tmp_file = cache_file.with_suffix(".json.tmp")
|
|
228
|
+
tmp_file.write_text(
|
|
229
|
+
json.dumps(
|
|
230
|
+
{
|
|
231
|
+
"kind": kind,
|
|
232
|
+
"path": path,
|
|
233
|
+
"payload": payload,
|
|
234
|
+
"response": response,
|
|
235
|
+
"retrieved_on": retrieved_on,
|
|
236
|
+
}
|
|
237
|
+
),
|
|
238
|
+
encoding="utf-8",
|
|
239
|
+
)
|
|
240
|
+
tmp_file.replace(cache_file)
|
|
241
|
+
return response, retrieved_on
|
|
242
|
+
|
|
243
|
+
def _journal_start(self, kind: str, digest: str) -> None:
|
|
244
|
+
"""Append-only durable record of every PAID start, successful or not
|
|
245
|
+
— the cache holds only successes, but a failed sync start may still
|
|
246
|
+
have billed upstream."""
|
|
247
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
248
|
+
with (self.cache_dir / "starts.log").open("a", encoding="utf-8") as handle:
|
|
249
|
+
handle.write(f"{date.today().isoformat()} {kind} {digest}\n")
|
|
250
|
+
|
|
251
|
+
def _prior_start_count(self, digest: str) -> int:
|
|
252
|
+
"""Paid starts already journaled for this digest by EARLIER
|
|
253
|
+
invocations — the salt for the wire Idempotency-Key. A failed
|
|
254
|
+
attempt's key must never be replayed verbatim: on the gateway (see
|
|
255
|
+
#3715), a client key can poison a wallet hold, and a byte-identical
|
|
256
|
+
retry of a poisoned key fails identically forever."""
|
|
257
|
+
starts_log = self.cache_dir / "starts.log"
|
|
258
|
+
try:
|
|
259
|
+
lines = starts_log.read_text(encoding="utf-8").splitlines()
|
|
260
|
+
except OSError:
|
|
261
|
+
return 0
|
|
262
|
+
return sum(1 for line in lines if line.endswith(f" {digest}"))
|
|
263
|
+
|
|
264
|
+
def run_actor(self, actor: str, actor_input: dict[str, Any], limit: int) -> list[dict[str, Any]]:
|
|
265
|
+
def check() -> None:
|
|
266
|
+
if self.actor_runs >= self.max_actor_runs:
|
|
267
|
+
raise CompetitorAdsError(
|
|
268
|
+
f"actor-run budget exhausted ({self.max_actor_runs} per attempt); "
|
|
269
|
+
"narrow the competitor list or re-run to continue from cache"
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
def count() -> None:
|
|
273
|
+
self.actor_runs += 1
|
|
274
|
+
|
|
275
|
+
response, retrieved_on = self._cached(
|
|
276
|
+
"actor",
|
|
277
|
+
"/scrape/apify/run",
|
|
278
|
+
{"actor": actor, "input": actor_input, "sync": True, "limit": limit},
|
|
279
|
+
timeout=330.0,
|
|
280
|
+
budget_check=check,
|
|
281
|
+
count=count,
|
|
282
|
+
retry_transient=True,
|
|
283
|
+
)
|
|
284
|
+
rows = response.get("rows")
|
|
285
|
+
return (rows if isinstance(rows, list) else []), retrieved_on
|
|
286
|
+
|
|
287
|
+
def meta_library_search(self, payload: dict[str, Any]) -> list[dict[str, Any]]:
|
|
288
|
+
def check() -> None:
|
|
289
|
+
if self.meta_searches >= self.max_meta_searches:
|
|
290
|
+
raise CompetitorAdsError(
|
|
291
|
+
f"meta-ad-library budget exhausted ({self.max_meta_searches} per attempt)"
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
def count() -> None:
|
|
295
|
+
self.meta_searches += 1
|
|
296
|
+
|
|
297
|
+
response, retrieved_on = self._cached(
|
|
298
|
+
"meta-ad-library",
|
|
299
|
+
"/tools/meta-ad-library/search",
|
|
300
|
+
payload,
|
|
301
|
+
timeout=90.0,
|
|
302
|
+
budget_check=check,
|
|
303
|
+
count=count,
|
|
304
|
+
)
|
|
305
|
+
rows = response.get("data")
|
|
306
|
+
return (rows if isinstance(rows, list) else []), retrieved_on
|
|
307
|
+
|
|
308
|
+
def ledger_totals(self) -> dict[str, Any]:
|
|
309
|
+
"""Durable provenance over the entries THIS execution touched (fresh
|
|
310
|
+
or replayed): one entry = one successful provider call, whichever
|
|
311
|
+
attempt made it. Entries from unrelated earlier runs against other
|
|
312
|
+
inputs in the same workspace are excluded."""
|
|
313
|
+
actor_runs = 0
|
|
314
|
+
meta_searches = 0
|
|
315
|
+
dates: list[str] = []
|
|
316
|
+
if self.cache_dir.is_dir():
|
|
317
|
+
for digest in sorted(self.touched):
|
|
318
|
+
entry_path = self.cache_dir / f"{digest}.json"
|
|
319
|
+
try:
|
|
320
|
+
entry = json.loads(entry_path.read_text(encoding="utf-8"))
|
|
321
|
+
except (OSError, json.JSONDecodeError):
|
|
322
|
+
continue
|
|
323
|
+
if entry.get("kind") == "actor":
|
|
324
|
+
actor_runs += 1
|
|
325
|
+
elif entry.get("kind") == "meta-ad-library":
|
|
326
|
+
meta_searches += 1
|
|
327
|
+
if isinstance(entry.get("retrieved_on"), str):
|
|
328
|
+
dates.append(entry["retrieved_on"])
|
|
329
|
+
today = date.today().isoformat()
|
|
330
|
+
starts_log = self.cache_dir / "starts.log"
|
|
331
|
+
try:
|
|
332
|
+
starts_recorded = sum(
|
|
333
|
+
1 for line in starts_log.read_text(encoding="utf-8").splitlines() if line
|
|
334
|
+
)
|
|
335
|
+
except OSError:
|
|
336
|
+
starts_recorded = 0
|
|
337
|
+
return {
|
|
338
|
+
"actor_runs": actor_runs,
|
|
339
|
+
"meta_searches": meta_searches,
|
|
340
|
+
"provider_calls": actor_runs + meta_searches,
|
|
341
|
+
"provider_starts_recorded": starts_recorded,
|
|
342
|
+
"first_retrieved_on": min(dates) if dates else today,
|
|
343
|
+
"last_retrieved_on": max(dates) if dates else today,
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
# ── deterministic helpers ──────────────────────────────────────────────────
|
|
348
|
+
|
|
349
|
+
_KEYWORD_ECONOMICS: ModuleType | None = None
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def registrable_domain(value: str) -> str:
|
|
353
|
+
"""The shared normalization of the Keyword Economics runner: two-level
|
|
354
|
+
public suffixes, sellers' subdomains on course and creator hosts."""
|
|
355
|
+
global _KEYWORD_ECONOMICS
|
|
356
|
+
if _KEYWORD_ECONOMICS is None:
|
|
357
|
+
_KEYWORD_ECONOMICS = _load_script("run-keyword-economics.py", "pmr_keyword_economics_for_competitor_ads")
|
|
358
|
+
return _KEYWORD_ECONOMICS.registrable_domain(value)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def brand_token(domain: str) -> str:
|
|
362
|
+
"""clickup.com -> clickup; the search key for name-matched ad libraries."""
|
|
363
|
+
stem = registrable_domain(domain).split(".")[0]
|
|
364
|
+
return re.sub(r"[^a-z0-9]", "", stem)
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def _normalize_name(value: str) -> str:
|
|
368
|
+
return re.sub(r"[^a-z0-9]", "", (value or "").lower())
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def name_matches_brand(name: str, brand: str) -> bool:
|
|
372
|
+
"""Ad-library keyword search matches advertiser NAMES; keep only entities
|
|
373
|
+
whose normalized name contains the brand token (validated live: a search
|
|
374
|
+
for 'clickup' also returned 'CLICKUP LEGALTECH SERVICES SL')."""
|
|
375
|
+
if not brand:
|
|
376
|
+
return False
|
|
377
|
+
return brand in _normalize_name(name)
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def verify_ad_identity(
|
|
381
|
+
ads: list[dict[str, Any]], target_domain: str, brand: str
|
|
382
|
+
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
383
|
+
"""Split candidate ads into (verified, excluded_namesakes).
|
|
384
|
+
|
|
385
|
+
Identity policy, strongest evidence first:
|
|
386
|
+
1. destination — the ad's landing destination resolves to the target
|
|
387
|
+
domain (Meta snapshot link_url/caption, official
|
|
388
|
+
ad_creative_link_captions);
|
|
389
|
+
2. exact_name — no destination data, but the advertiser/page name
|
|
390
|
+
normalizes exactly to the brand token;
|
|
391
|
+
3. anything else (contains-only namesakes, foreign destinations) is
|
|
392
|
+
EXCLUDED from all verdicts and reported separately — a lookalike's ads
|
|
393
|
+
must never prove a competitor's channel."""
|
|
394
|
+
target = registrable_domain(target_domain)
|
|
395
|
+
verified: list[dict[str, Any]] = []
|
|
396
|
+
excluded: list[dict[str, Any]] = []
|
|
397
|
+
for ad in ads:
|
|
398
|
+
destination = ad.pop("_destination", None)
|
|
399
|
+
if destination:
|
|
400
|
+
if registrable_domain(destination) == target:
|
|
401
|
+
verified.append({**ad, "verified_by": "destination"})
|
|
402
|
+
else:
|
|
403
|
+
excluded.append(
|
|
404
|
+
{
|
|
405
|
+
"type": "excluded_namesake",
|
|
406
|
+
"domain": target_domain,
|
|
407
|
+
"platform": ad.get("platform"),
|
|
408
|
+
"advertiser": ad.get("advertiser"),
|
|
409
|
+
"ad_id": ad.get("ad_id"),
|
|
410
|
+
"destination": destination,
|
|
411
|
+
"reason": "destination_mismatch",
|
|
412
|
+
"tier": "B",
|
|
413
|
+
}
|
|
414
|
+
)
|
|
415
|
+
continue
|
|
416
|
+
if _normalize_name(ad.get("advertiser") or "") == brand:
|
|
417
|
+
verified.append({**ad, "verified_by": "exact_name"})
|
|
418
|
+
continue
|
|
419
|
+
excluded.append(
|
|
420
|
+
{
|
|
421
|
+
"type": "excluded_namesake",
|
|
422
|
+
"domain": target_domain,
|
|
423
|
+
"platform": ad.get("platform"),
|
|
424
|
+
"advertiser": ad.get("advertiser"),
|
|
425
|
+
"ad_id": ad.get("ad_id"),
|
|
426
|
+
"destination": None,
|
|
427
|
+
"reason": "name_contains_only",
|
|
428
|
+
"tier": "B",
|
|
429
|
+
}
|
|
430
|
+
)
|
|
431
|
+
return verified, excluded
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def dedupe_meta_ads(
|
|
435
|
+
actor_ads: list[dict[str, Any]], official_ads: list[dict[str, Any]]
|
|
436
|
+
) -> list[dict[str, Any]]:
|
|
437
|
+
"""The US actor and the EU official API can return the SAME ad archive
|
|
438
|
+
id; counting it twice would double every longevity statistic. Merge by ad
|
|
439
|
+
identity, preferring the official row's EU fields and the larger measured
|
|
440
|
+
age."""
|
|
441
|
+
merged: dict[str, dict[str, Any]] = {}
|
|
442
|
+
ordered: list[dict[str, Any]] = []
|
|
443
|
+
for ad in actor_ads + official_ads:
|
|
444
|
+
key = str(ad.get("ad_id") or "") or f"__anon_{len(ordered)}"
|
|
445
|
+
existing = merged.get(key)
|
|
446
|
+
if existing is None:
|
|
447
|
+
row = dict(ad)
|
|
448
|
+
merged[key] = row
|
|
449
|
+
ordered.append(row)
|
|
450
|
+
continue
|
|
451
|
+
for field_name in ("eu_total_reach", "publisher_platforms", "link", "text"):
|
|
452
|
+
if existing.get(field_name) in (None, "") and ad.get(field_name) not in (None, ""):
|
|
453
|
+
existing[field_name] = ad[field_name]
|
|
454
|
+
for day_field in ("served_days",):
|
|
455
|
+
a, b = existing.get(day_field), ad.get(day_field)
|
|
456
|
+
if isinstance(b, int) and (not isinstance(a, int) or b > a):
|
|
457
|
+
existing[day_field] = b
|
|
458
|
+
existing["platform"] = "meta"
|
|
459
|
+
return ordered
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def parse_terms(raw: str, cap: int) -> list[str]:
|
|
463
|
+
seen: dict[str, None] = {}
|
|
464
|
+
for part in re.split(r"[,;\n]", raw or ""):
|
|
465
|
+
term = part.strip().lower()
|
|
466
|
+
if term and term not in seen:
|
|
467
|
+
seen[term] = None
|
|
468
|
+
return list(seen)[:cap]
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def domains_from_keyword_evidence(workspace: Path, cap: int, exclude=frozenset()) -> list[str]:
|
|
473
|
+
evidence = _load_script("research-evidence.py", "pmr_evidence")
|
|
474
|
+
return evidence.keyword_inputs(workspace, consumer=SECTION_ID, row_type="competitor_domain", field="domain", cap=cap, exclude=exclude)
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def domains_from_competitor_set(workspace: Path, cap: int) -> list[str]:
|
|
478
|
+
"""Tiered competitors from a completed Competitor Set section, in rank order."""
|
|
479
|
+
evidence = _load_script("research-evidence.py", "pmr_evidence")
|
|
480
|
+
return evidence.competitor_set_inputs(workspace, consumer=SECTION_ID, cap=cap)
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def domains_excluded_by_competitor_set(workspace: Path) -> set[str]:
|
|
484
|
+
"""Domains the completed Competitor Set rejected; never refilled automatically."""
|
|
485
|
+
evidence = _load_script("research-evidence.py", "pmr_evidence")
|
|
486
|
+
return evidence.competitor_set_excluded_domains(workspace)
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _days_between(start_iso: str, end_iso: str) -> int | None:
|
|
490
|
+
try:
|
|
491
|
+
start = date.fromisoformat(start_iso[:10])
|
|
492
|
+
end = date.fromisoformat(end_iso[:10])
|
|
493
|
+
except (ValueError, TypeError):
|
|
494
|
+
return None
|
|
495
|
+
return max(0, (end - start).days)
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def _epoch_to_iso(value: Any) -> str | None:
|
|
499
|
+
if not isinstance(value, (int, float)) or value <= 0:
|
|
500
|
+
return None
|
|
501
|
+
return datetime.fromtimestamp(float(value), tz=timezone.utc).date().isoformat()
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def normalize_google_ad(row: dict[str, Any], domain: str) -> dict[str, Any] | None:
|
|
505
|
+
"""Domain-mode rows are destination-verified by construction; still check
|
|
506
|
+
the echoed destination domain and drop anything foreign."""
|
|
507
|
+
destination = registrable_domain(str(row.get("domain") or ""))
|
|
508
|
+
if destination != registrable_domain(domain):
|
|
509
|
+
return None
|
|
510
|
+
first = str(row.get("firstShown") or "")[:10] or None
|
|
511
|
+
last = str(row.get("lastShown") or "")[:10] or None
|
|
512
|
+
served = row.get("shownForDays")
|
|
513
|
+
if not isinstance(served, int) and first and last:
|
|
514
|
+
served = _days_between(first, last)
|
|
515
|
+
return {
|
|
516
|
+
"type": "ad_observation",
|
|
517
|
+
"domain": domain,
|
|
518
|
+
"platform": "google",
|
|
519
|
+
"advertiser": str(row.get("advertiserName") or ""),
|
|
520
|
+
"advertiser_id": row.get("advertiserId"),
|
|
521
|
+
"ad_id": row.get("creativeId"),
|
|
522
|
+
"format": row.get("format"),
|
|
523
|
+
"first_seen": first,
|
|
524
|
+
"last_seen": last,
|
|
525
|
+
"served_days": served,
|
|
526
|
+
"verified_by": "destination",
|
|
527
|
+
"text": None,
|
|
528
|
+
"link": row.get("adUrl") or row.get("previewUrl"),
|
|
529
|
+
"tier": "A",
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def normalize_meta_actor_ad(row: dict[str, Any], domain: str, today_iso: str) -> dict[str, Any] | None:
|
|
534
|
+
page = str(row.get("page_name") or row.get("pageName") or "")
|
|
535
|
+
snapshot_peek = row.get("snapshot") or {}
|
|
536
|
+
peek_destination = str(
|
|
537
|
+
snapshot_peek.get("link_url") or snapshot_peek.get("caption") or ""
|
|
538
|
+
)
|
|
539
|
+
# Admit a candidate when the page name resembles the brand OR the ad's
|
|
540
|
+
# destination points at the target domain (a page named after the product
|
|
541
|
+
# line, not the brand, still counts); verify_ad_identity decides.
|
|
542
|
+
if not name_matches_brand(page, brand_token(domain)) and registrable_domain(
|
|
543
|
+
peek_destination
|
|
544
|
+
) != registrable_domain(domain):
|
|
545
|
+
return None
|
|
546
|
+
snapshot = row.get("snapshot") or {}
|
|
547
|
+
body = snapshot.get("body") or {}
|
|
548
|
+
text = body.get("text") if isinstance(body, dict) else None
|
|
549
|
+
destination = (
|
|
550
|
+
str(snapshot.get("link_url") or "").strip()
|
|
551
|
+
or str(snapshot.get("caption") or "").strip()
|
|
552
|
+
or None
|
|
553
|
+
)
|
|
554
|
+
first = _epoch_to_iso(row.get("start_date") or row.get("startDate"))
|
|
555
|
+
last = _epoch_to_iso(row.get("end_date") or row.get("endDate"))
|
|
556
|
+
is_active = row.get("is_active")
|
|
557
|
+
effective_last = today_iso if is_active or not last else last
|
|
558
|
+
served = _days_between(first, effective_last) if first else None
|
|
559
|
+
ad_id = row.get("ad_archive_id") or row.get("adArchiveID") or row.get("ad_id")
|
|
560
|
+
return {
|
|
561
|
+
"type": "ad_observation",
|
|
562
|
+
"domain": domain,
|
|
563
|
+
"platform": "meta",
|
|
564
|
+
"advertiser": page,
|
|
565
|
+
"page_id": row.get("page_id"),
|
|
566
|
+
"ad_id": ad_id,
|
|
567
|
+
"format": None,
|
|
568
|
+
"first_seen": first,
|
|
569
|
+
"last_seen": last,
|
|
570
|
+
"is_active": bool(is_active) if is_active is not None else None,
|
|
571
|
+
"served_days": served,
|
|
572
|
+
"_destination": destination,
|
|
573
|
+
"text": (text or "")[:280] or None,
|
|
574
|
+
"link": f"https://www.facebook.com/ads/library/?id={ad_id}" if ad_id else None,
|
|
575
|
+
"tier": "A",
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def normalize_meta_official_ad(row: dict[str, Any], domain: str, today_iso: str) -> dict[str, Any] | None:
|
|
580
|
+
page = str(row.get("page_name") or "")
|
|
581
|
+
captions_peek = row.get("ad_creative_link_captions") or []
|
|
582
|
+
peek_destination = str(captions_peek[0]) if captions_peek else ""
|
|
583
|
+
if not name_matches_brand(page, brand_token(domain)) and registrable_domain(
|
|
584
|
+
peek_destination
|
|
585
|
+
) != registrable_domain(domain):
|
|
586
|
+
return None
|
|
587
|
+
first = str(row.get("ad_delivery_start_time") or "")[:10] or None
|
|
588
|
+
last = str(row.get("ad_delivery_stop_time") or "")[:10] or None
|
|
589
|
+
served = _days_between(first, last or today_iso) if first else None
|
|
590
|
+
bodies = row.get("ad_creative_bodies") or []
|
|
591
|
+
text = bodies[0] if bodies and isinstance(bodies[0], str) else None
|
|
592
|
+
captions = row.get("ad_creative_link_captions") or []
|
|
593
|
+
destination = (
|
|
594
|
+
str(captions[0]).strip() if captions and str(captions[0]).strip() else None
|
|
595
|
+
)
|
|
596
|
+
return {
|
|
597
|
+
"type": "ad_observation",
|
|
598
|
+
"domain": domain,
|
|
599
|
+
"platform": "meta_eu",
|
|
600
|
+
"_destination": destination,
|
|
601
|
+
"advertiser": page,
|
|
602
|
+
"page_id": row.get("page_id"),
|
|
603
|
+
"ad_id": row.get("id"),
|
|
604
|
+
"format": None,
|
|
605
|
+
"first_seen": first,
|
|
606
|
+
"last_seen": last,
|
|
607
|
+
"served_days": served,
|
|
608
|
+
"eu_total_reach": row.get("eu_total_reach"),
|
|
609
|
+
"publisher_platforms": row.get("publisher_platforms"),
|
|
610
|
+
"text": (text or "")[:280] or None,
|
|
611
|
+
"link": row.get("ad_snapshot_url"),
|
|
612
|
+
"tier": "A",
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def summarize_platform(
|
|
617
|
+
domain: str,
|
|
618
|
+
platform: str,
|
|
619
|
+
ads: list[dict[str, Any]],
|
|
620
|
+
threshold_days: int = LONG_RUNNING_DAYS,
|
|
621
|
+
) -> dict[str, Any]:
|
|
622
|
+
served = [a.get("served_days") for a in ads if isinstance(a.get("served_days"), int)]
|
|
623
|
+
# Advertiser presence SPAN survives creative rotation: a business swapping
|
|
624
|
+
# creatives weekly still shows a long earliest-first-seen -> latest-last-
|
|
625
|
+
# seen window, while any single creative's age resets on every edit.
|
|
626
|
+
firsts = [a.get("first_seen") for a in ads if a.get("first_seen")]
|
|
627
|
+
lasts = [a.get("last_seen") for a in ads if a.get("last_seen")]
|
|
628
|
+
span_days = _days_between(min(firsts), max(lasts)) if firsts and lasts else None
|
|
629
|
+
return {
|
|
630
|
+
"type": "platform_presence",
|
|
631
|
+
"domain": domain,
|
|
632
|
+
"platform": platform,
|
|
633
|
+
"ads_observed": len(ads),
|
|
634
|
+
"max_served_days": max(served) if served else None,
|
|
635
|
+
"median_served_days": sorted(served)[len(served) // 2] if served else None,
|
|
636
|
+
"presence_span_days": span_days,
|
|
637
|
+
"presence_first_seen": min(firsts) if firsts else None,
|
|
638
|
+
"presence_last_seen": max(lasts) if lasts else None,
|
|
639
|
+
"verified_by_destination": sum(1 for a in ads if a.get("verified_by") == "destination"),
|
|
640
|
+
"verified_by_exact_name": sum(1 for a in ads if a.get("verified_by") == "exact_name"),
|
|
641
|
+
"status": "observed" if ads else "no_matches_in_sample",
|
|
642
|
+
"tier": "A",
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
def extract_price_mentions(ads: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
647
|
+
mentions: list[dict[str, Any]] = []
|
|
648
|
+
seen: set[str] = set()
|
|
649
|
+
for ad in ads:
|
|
650
|
+
text = ad.get("text") or ""
|
|
651
|
+
for match in _PRICE_RE.finditer(text):
|
|
652
|
+
start = max(0, match.start() - 40)
|
|
653
|
+
excerpt = text[start : match.end() + 40].strip()
|
|
654
|
+
key = f"{ad['domain']}|{excerpt.lower()}"
|
|
655
|
+
if key in seen:
|
|
656
|
+
continue
|
|
657
|
+
seen.add(key)
|
|
658
|
+
mentions.append(
|
|
659
|
+
{
|
|
660
|
+
"type": "price_mention",
|
|
661
|
+
"domain": ad["domain"],
|
|
662
|
+
"platform": ad["platform"],
|
|
663
|
+
"excerpt": excerpt,
|
|
664
|
+
"ad_id": ad.get("ad_id"),
|
|
665
|
+
"tier": "B",
|
|
666
|
+
}
|
|
667
|
+
)
|
|
668
|
+
return mentions
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def meta_search_url(brand: str, country: str) -> str:
|
|
672
|
+
return (
|
|
673
|
+
"https://www.facebook.com/ads/library/?active_status=all&ad_type=all"
|
|
674
|
+
f"&country={quote(country)}&q=%22{quote(brand)}%22&search_type=keyword_unordered&media_type=all"
|
|
675
|
+
)
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
# ── rendering ──────────────────────────────────────────────────────────────
|
|
679
|
+
|
|
680
|
+
def _fmt(value: Any) -> str:
|
|
681
|
+
if value is None:
|
|
682
|
+
return "—"
|
|
683
|
+
if isinstance(value, int):
|
|
684
|
+
return f"{value:,}"
|
|
685
|
+
return str(value)
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def render_section(
|
|
689
|
+
*,
|
|
690
|
+
domains: list[str],
|
|
691
|
+
domain_sources: dict[str, str],
|
|
692
|
+
presences: list[dict[str, Any]],
|
|
693
|
+
ads: list[dict[str, Any]],
|
|
694
|
+
price_mentions: list[dict[str, Any]],
|
|
695
|
+
excluded_namesakes: list[dict[str, Any]],
|
|
696
|
+
totals: dict[str, Any],
|
|
697
|
+
retrieved_on: str,
|
|
698
|
+
eu_country: str,
|
|
699
|
+
threshold_days: int = LONG_RUNNING_DAYS,
|
|
700
|
+
) -> str:
|
|
701
|
+
lines: list[str] = []
|
|
702
|
+
add = lines.append
|
|
703
|
+
|
|
704
|
+
add(
|
|
705
|
+
f"Ad-transparency data retrieved {retrieved_on} from Google Ads "
|
|
706
|
+
f"Transparency and the Meta Ads Library (US coverage via scraper, EU "
|
|
707
|
+
f"delivery data via the official API, country `{eu_country}`). "
|
|
708
|
+
"Observed ad duration and advertiser span do not establish spend, "
|
|
709
|
+
"continuous delivery, conversion or profitability. "
|
|
710
|
+
"Identity is destination-verified: Google ads are collected by the "
|
|
711
|
+
"domain their ads point to (advertiser legal names like ClickUp's "
|
|
712
|
+
"'Mango Technologies, Inc.' are found automatically), and Meta ads "
|
|
713
|
+
"count only when their landing destination resolves to the target "
|
|
714
|
+
"domain or the page name is an exact brand match; same-name "
|
|
715
|
+
"lookalikes are excluded and listed below."
|
|
716
|
+
)
|
|
717
|
+
add("")
|
|
718
|
+
add(
|
|
719
|
+
"Domains examined: "
|
|
720
|
+
+ ", ".join(f"`{d}` ({domain_sources.get(d, 'explicit')})" for d in domains)
|
|
721
|
+
+ "."
|
|
722
|
+
)
|
|
723
|
+
add("")
|
|
724
|
+
|
|
725
|
+
add("### Platform presence and ad longevity")
|
|
726
|
+
add("")
|
|
727
|
+
add("| Domain | Platform | Ads observed | Oldest creative (days) | Advertiser span (days) | Coverage |")
|
|
728
|
+
add("| --- | --- | ---: | ---: | ---: | --- |")
|
|
729
|
+
for row in presences:
|
|
730
|
+
add(
|
|
731
|
+
f"| {row['domain']} | {row['platform']} | {_fmt(row['ads_observed'])} | "
|
|
732
|
+
f"{_fmt(row['max_served_days'])} | {_fmt(row.get('presence_span_days'))} | "
|
|
733
|
+
f"{row['status']} |"
|
|
734
|
+
)
|
|
735
|
+
add("")
|
|
736
|
+
add("No matches means no matching ads in this bounded sample, not absent advertising or absent demand.")
|
|
737
|
+
add("")
|
|
738
|
+
if excluded_namesakes:
|
|
739
|
+
by_name: dict[str, int] = {}
|
|
740
|
+
for row in excluded_namesakes:
|
|
741
|
+
key = f"{row.get('advertiser')} ({row.get('platform')}, {row.get('reason')})"
|
|
742
|
+
by_name[key] = by_name.get(key, 0) + 1
|
|
743
|
+
add(
|
|
744
|
+
"Excluded same-name lookalikes (not attributed to the scoped competitors): "
|
|
745
|
+
+ "; ".join(f"{name} × {count}" for name, count in sorted(by_name.items()))
|
|
746
|
+
+ "."
|
|
747
|
+
)
|
|
748
|
+
add("")
|
|
749
|
+
|
|
750
|
+
long_ads = sorted(
|
|
751
|
+
(a for a in ads if isinstance(a.get("served_days"), int)),
|
|
752
|
+
key=lambda a: a["served_days"],
|
|
753
|
+
reverse=True,
|
|
754
|
+
)
|
|
755
|
+
if long_ads:
|
|
756
|
+
add("### Longest-running creatives")
|
|
757
|
+
add("")
|
|
758
|
+
add("| Domain | Platform | Days served | First seen | Last seen | Creative |")
|
|
759
|
+
add("| --- | --- | ---: | --- | --- | --- |")
|
|
760
|
+
per_domain: dict[str, int] = {}
|
|
761
|
+
for ad in long_ads:
|
|
762
|
+
if per_domain.get(ad["domain"], 0) >= 3:
|
|
763
|
+
continue
|
|
764
|
+
per_domain[ad["domain"]] = per_domain.get(ad["domain"], 0) + 1
|
|
765
|
+
creative = (ad.get("text") or ad.get("format") or "—").replace("|", "/")
|
|
766
|
+
creative = " ".join(str(creative).split())[:110]
|
|
767
|
+
link = ad.get("link")
|
|
768
|
+
creative_cell = f"[{creative}]({link})" if link else creative
|
|
769
|
+
add(
|
|
770
|
+
f"| {ad['domain']} | {ad['platform']} | {ad['served_days']} | "
|
|
771
|
+
f"{_fmt(ad.get('first_seen'))} | {_fmt(ad.get('last_seen'))} | {creative_cell} |"
|
|
772
|
+
)
|
|
773
|
+
add("")
|
|
774
|
+
|
|
775
|
+
if price_mentions:
|
|
776
|
+
add("### Price anchors found in ad copy")
|
|
777
|
+
add("")
|
|
778
|
+
for mention in price_mentions[:10]:
|
|
779
|
+
add(f"- `{mention['domain']}` ({mention['platform']}): “{mention['excerpt']}”")
|
|
780
|
+
add("")
|
|
781
|
+
|
|
782
|
+
add("### Method and coverage")
|
|
783
|
+
add("")
|
|
784
|
+
add(
|
|
785
|
+
f"- Provider calls recorded: {totals['actor_runs']} Apify actor run(s) "
|
|
786
|
+
f"+ {totals['meta_searches']} official Meta Ad Library search(es), "
|
|
787
|
+
"accumulated across attempts; cached replays add no new spend."
|
|
788
|
+
)
|
|
789
|
+
add(
|
|
790
|
+
f"- Caps: {MAX_COMPETITORS} competitors, {GOOGLE_ADS_PER_COMPETITOR} "
|
|
791
|
+
f"Google creatives and {META_ADS_PER_COMPETITOR} Meta ads per "
|
|
792
|
+
f"competitor, {MAX_ACTOR_RUNS_PER_ATTEMPT} actor runs per attempt."
|
|
793
|
+
)
|
|
794
|
+
add(
|
|
795
|
+
"- Longevity from Google's own served-days and Meta delivery windows; "
|
|
796
|
+
"the advertiser span may include gaps and creative rotation. Not covered: LinkedIn/TikTok libraries, targeting details, "
|
|
797
|
+
"spend amounts (Meta discloses neither for commercial ads outside "
|
|
798
|
+
"the EU), and any judgement of which combo to pursue."
|
|
799
|
+
)
|
|
800
|
+
add("")
|
|
801
|
+
add("### Sources")
|
|
802
|
+
add("")
|
|
803
|
+
add(
|
|
804
|
+
f"- Google Ads Transparency via Apify `{GOOGLE_ACTOR}` "
|
|
805
|
+
f"(domain-keyed collection); Meta Ads Library via Apify "
|
|
806
|
+
f"`{META_ACTOR}` (US) and the official `ads_archive` API (EU, "
|
|
807
|
+
f"`{eu_country}`) — retrieved {retrieved_on}; evidence rows in "
|
|
808
|
+
f"`{EVIDENCE_REL.as_posix()}`."
|
|
809
|
+
)
|
|
810
|
+
return "\n".join(lines).strip() + "\n"
|
|
811
|
+
|
|
812
|
+
|
|
813
|
+
# ── orchestration ──────────────────────────────────────────────────────────
|
|
814
|
+
|
|
815
|
+
@dataclass
|
|
816
|
+
class RunInputs:
|
|
817
|
+
domains: list[str]
|
|
818
|
+
domain_sources: dict[str, str]
|
|
819
|
+
eu_country: str
|
|
820
|
+
long_running_days: int = LONG_RUNNING_DAYS
|
|
821
|
+
|
|
822
|
+
|
|
823
|
+
def _call_with_context(client: OsClient, domain: str, label: str, fn: Callable[[], Any]) -> Any:
|
|
824
|
+
"""Re-raise a provider-call failure with WHICH competitor/call failed and
|
|
825
|
+
how much of the attempt's paid budget is already spent — the raw gateway
|
|
826
|
+
envelope alone doesn't say either, so a human can't tell whether a retry
|
|
827
|
+
is even possible without this."""
|
|
828
|
+
try:
|
|
829
|
+
return fn()
|
|
830
|
+
except CompetitorAdsError as exc:
|
|
831
|
+
raise CompetitorAdsError(
|
|
832
|
+
f"{label} for `{domain}` failed: {exc} — this attempt had "
|
|
833
|
+
f"already started {client.actor_runs}/{client.max_actor_runs} "
|
|
834
|
+
f"actor run(s) and {client.meta_searches}/{client.max_meta_searches} "
|
|
835
|
+
"meta search(es); see GET /me/limits?resource=apify:runs for "
|
|
836
|
+
"remaining daily quota."
|
|
837
|
+
) from exc
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
def run_pipeline(client: OsClient, inputs: RunInputs) -> dict[str, Any]:
|
|
841
|
+
if not inputs.domains:
|
|
842
|
+
raise CompetitorAdsError(
|
|
843
|
+
"no competitor domains available: run the Keyword Economics "
|
|
844
|
+
"section first (its evidence feeds this section) or pass "
|
|
845
|
+
"--domains explicitly"
|
|
846
|
+
)
|
|
847
|
+
ads: list[dict[str, Any]] = []
|
|
848
|
+
presences: list[dict[str, Any]] = []
|
|
849
|
+
excluded_namesakes: list[dict[str, Any]] = []
|
|
850
|
+
for domain in inputs.domains:
|
|
851
|
+
brand = brand_token(domain)
|
|
852
|
+
|
|
853
|
+
google_rows, _ = _call_with_context(
|
|
854
|
+
client,
|
|
855
|
+
domain,
|
|
856
|
+
"Google Ads Transparency",
|
|
857
|
+
lambda: client.run_actor(
|
|
858
|
+
GOOGLE_ACTOR,
|
|
859
|
+
{
|
|
860
|
+
"domains": [registrable_domain(domain)],
|
|
861
|
+
"resultType": "ads",
|
|
862
|
+
"region": "ANYWHERE",
|
|
863
|
+
"maxAdsPerSearch": GOOGLE_ADS_PER_COMPETITOR,
|
|
864
|
+
},
|
|
865
|
+
limit=GOOGLE_ADS_PER_COMPETITOR,
|
|
866
|
+
),
|
|
867
|
+
)
|
|
868
|
+
google_ads = [
|
|
869
|
+
ad for ad in (normalize_google_ad(r, domain) for r in google_rows) if ad
|
|
870
|
+
]
|
|
871
|
+
|
|
872
|
+
# Active ads age up to the date their snapshot was RETRIEVED, never
|
|
873
|
+
# the current wall clock: a cache replay must not fabricate extra
|
|
874
|
+
# longevity for an ad that may have stopped the day after capture.
|
|
875
|
+
meta_rows, meta_as_of = _call_with_context(
|
|
876
|
+
client,
|
|
877
|
+
domain,
|
|
878
|
+
"Meta Ads Library (US actor)",
|
|
879
|
+
lambda: client.run_actor(
|
|
880
|
+
META_ACTOR,
|
|
881
|
+
{
|
|
882
|
+
"urls": [{"url": meta_search_url(brand, "US")}],
|
|
883
|
+
"count": META_ADS_PER_COMPETITOR,
|
|
884
|
+
},
|
|
885
|
+
limit=META_ADS_PER_COMPETITOR,
|
|
886
|
+
),
|
|
887
|
+
)
|
|
888
|
+
meta_candidates = [
|
|
889
|
+
ad
|
|
890
|
+
for ad in (normalize_meta_actor_ad(r, domain, meta_as_of) for r in meta_rows)
|
|
891
|
+
if ad
|
|
892
|
+
]
|
|
893
|
+
meta_ads, meta_excluded = verify_ad_identity(meta_candidates, domain, brand)
|
|
894
|
+
excluded_namesakes.extend(meta_excluded)
|
|
895
|
+
|
|
896
|
+
official_rows, official_as_of = _call_with_context(
|
|
897
|
+
client,
|
|
898
|
+
domain,
|
|
899
|
+
"Meta Ads Library (official EU API)",
|
|
900
|
+
lambda: client.meta_library_search(
|
|
901
|
+
{
|
|
902
|
+
"query": brand,
|
|
903
|
+
"countries": [inputs.eu_country],
|
|
904
|
+
"activeStatus": "ALL",
|
|
905
|
+
"limit": META_ADS_PER_COMPETITOR,
|
|
906
|
+
}
|
|
907
|
+
),
|
|
908
|
+
)
|
|
909
|
+
official_candidates = [
|
|
910
|
+
ad
|
|
911
|
+
for ad in (
|
|
912
|
+
normalize_meta_official_ad(r, domain, official_as_of)
|
|
913
|
+
for r in official_rows
|
|
914
|
+
)
|
|
915
|
+
if ad
|
|
916
|
+
]
|
|
917
|
+
meta_eu_ads, official_excluded = verify_ad_identity(
|
|
918
|
+
official_candidates, domain, brand
|
|
919
|
+
)
|
|
920
|
+
excluded_namesakes.extend(official_excluded)
|
|
921
|
+
|
|
922
|
+
merged_meta = dedupe_meta_ads(meta_ads, meta_eu_ads)
|
|
923
|
+
ads.extend(google_ads + merged_meta)
|
|
924
|
+
presences.append(
|
|
925
|
+
summarize_platform(domain, "google", google_ads, inputs.long_running_days)
|
|
926
|
+
)
|
|
927
|
+
presences.append(
|
|
928
|
+
summarize_platform(domain, "meta", merged_meta, inputs.long_running_days)
|
|
929
|
+
)
|
|
930
|
+
|
|
931
|
+
price_mentions = extract_price_mentions(ads)
|
|
932
|
+
|
|
933
|
+
# Keep the evidence lean: per (domain, platform), store the longest-served
|
|
934
|
+
# ads — per-platform so a long-lived Google history cannot crowd Meta's
|
|
935
|
+
# creatives (and their price-bearing texts) out of the evidence file.
|
|
936
|
+
stored_ads: list[dict[str, Any]] = []
|
|
937
|
+
for domain in inputs.domains:
|
|
938
|
+
for platform in ("google", "meta"):
|
|
939
|
+
platform_ads = sorted(
|
|
940
|
+
(
|
|
941
|
+
a
|
|
942
|
+
for a in ads
|
|
943
|
+
if a["domain"] == domain and a["platform"] == platform
|
|
944
|
+
),
|
|
945
|
+
key=lambda a: a.get("served_days") or -1,
|
|
946
|
+
reverse=True,
|
|
947
|
+
)
|
|
948
|
+
stored_ads.extend(platform_ads[:STORED_ADS_PER_DOMAIN])
|
|
949
|
+
|
|
950
|
+
totals = client.ledger_totals()
|
|
951
|
+
retrieved_on = (
|
|
952
|
+
totals["first_retrieved_on"]
|
|
953
|
+
if totals["first_retrieved_on"] == totals["last_retrieved_on"]
|
|
954
|
+
else f"{totals['first_retrieved_on']}..{totals['last_retrieved_on']}"
|
|
955
|
+
)
|
|
956
|
+
|
|
957
|
+
domain_rows = [
|
|
958
|
+
{
|
|
959
|
+
"type": "competitor_domain",
|
|
960
|
+
"domain": domain,
|
|
961
|
+
"source": inputs.domain_sources.get(domain, "explicit"),
|
|
962
|
+
"tier": "A",
|
|
963
|
+
}
|
|
964
|
+
for domain in inputs.domains
|
|
965
|
+
]
|
|
966
|
+
|
|
967
|
+
evidence = {
|
|
968
|
+
"schema_version": SCHEMA_VERSION,
|
|
969
|
+
"section_id": SECTION_ID,
|
|
970
|
+
"generated_by": Path(__file__).name,
|
|
971
|
+
"retrieved_on": retrieved_on,
|
|
972
|
+
"inputs": {
|
|
973
|
+
"domains": inputs.domains,
|
|
974
|
+
"domain_sources": inputs.domain_sources,
|
|
975
|
+
"eu_country": inputs.eu_country,
|
|
976
|
+
},
|
|
977
|
+
"rows": domain_rows + presences + stored_ads + price_mentions + excluded_namesakes,
|
|
978
|
+
"totals": {
|
|
979
|
+
"actor_runs": totals["actor_runs"],
|
|
980
|
+
"meta_searches": totals["meta_searches"],
|
|
981
|
+
"provider_calls": totals["provider_calls"],
|
|
982
|
+
},
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
section_markdown = render_section(
|
|
986
|
+
domains=inputs.domains,
|
|
987
|
+
domain_sources=inputs.domain_sources,
|
|
988
|
+
presences=presences,
|
|
989
|
+
ads=stored_ads,
|
|
990
|
+
price_mentions=price_mentions,
|
|
991
|
+
excluded_namesakes=excluded_namesakes,
|
|
992
|
+
totals=totals,
|
|
993
|
+
retrieved_on=retrieved_on,
|
|
994
|
+
eu_country=inputs.eu_country,
|
|
995
|
+
threshold_days=inputs.long_running_days,
|
|
996
|
+
)
|
|
997
|
+
return {
|
|
998
|
+
"evidence": evidence,
|
|
999
|
+
"section_markdown": section_markdown,
|
|
1000
|
+
"totals": totals,
|
|
1001
|
+
"attempt": {
|
|
1002
|
+
"actor_runs": client.actor_runs,
|
|
1003
|
+
"meta_searches": client.meta_searches,
|
|
1004
|
+
"cache_hits": client.cache_hits,
|
|
1005
|
+
},
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
|
|
1009
|
+
def _write_atomic(path: Path, content: str) -> None:
|
|
1010
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
1011
|
+
tmp = path.with_suffix(path.suffix + ".tmp")
|
|
1012
|
+
tmp.write_text(content, encoding="utf-8")
|
|
1013
|
+
tmp.replace(path)
|
|
1014
|
+
|
|
1015
|
+
|
|
1016
|
+
def run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
|
|
1017
|
+
workspace = workspace.resolve()
|
|
1018
|
+
io = _load_script("research-artifact-io.py", "pmr_artifact_io")
|
|
1019
|
+
with io.section_attempt(workspace, SECTION_ID, context_file=getattr(args, "context_file", None)):
|
|
1020
|
+
return _run(workspace, args)
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
def _run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
|
|
1024
|
+
workspace = workspace.resolve()
|
|
1025
|
+
runtime = workspace / RUNTIME_REL
|
|
1026
|
+
cache_dir = runtime / "cache"
|
|
1027
|
+
runtime.mkdir(parents=True, exist_ok=True)
|
|
1028
|
+
|
|
1029
|
+
api_key = (os.environ.get("OS_API_KEY") or "").strip()
|
|
1030
|
+
if not api_key:
|
|
1031
|
+
raise CompetitorAdsError("OS_API_KEY is required")
|
|
1032
|
+
os_url = (os.environ.get("OS_URL") or DEFAULT_OS_URL).strip() or DEFAULT_OS_URL
|
|
1033
|
+
effective_policy, policy_sha256 = POLICY.load_policy()
|
|
1034
|
+
explicit = parse_terms(args.domains or "", MAX_COMPETITORS)
|
|
1035
|
+
explicit = [registrable_domain(d) for d in explicit if registrable_domain(d)]
|
|
1036
|
+
# Explicit domains first, then the tiered Competitor Set, then the
|
|
1037
|
+
# Keyword Economics demand surface; each source is optional.
|
|
1038
|
+
tiered = domains_from_competitor_set(workspace, MAX_COMPETITORS)
|
|
1039
|
+
# The tiering decisions constrain automatic fallbacks: a domain the
|
|
1040
|
+
# Competitor Set rejected is not pulled back from the demand surface.
|
|
1041
|
+
# Explicit --domains remain the founder's override.
|
|
1042
|
+
rejected = domains_excluded_by_competitor_set(workspace) - set(explicit)
|
|
1043
|
+
discovered = domains_from_keyword_evidence(workspace, MAX_COMPETITORS, exclude=rejected)
|
|
1044
|
+
|
|
1045
|
+
domains: list[str] = []
|
|
1046
|
+
domain_sources: dict[str, str] = {}
|
|
1047
|
+
for domain in explicit:
|
|
1048
|
+
if domain not in domains:
|
|
1049
|
+
domains.append(domain)
|
|
1050
|
+
domain_sources[domain] = "explicit"
|
|
1051
|
+
for source, candidates in (("competitor_set", tiered), ("keyword_economics", discovered)):
|
|
1052
|
+
for domain in candidates:
|
|
1053
|
+
if len(domains) >= args.max_competitors:
|
|
1054
|
+
break
|
|
1055
|
+
if domain not in domains:
|
|
1056
|
+
domains.append(domain)
|
|
1057
|
+
domain_sources[domain] = source
|
|
1058
|
+
domains = domains[: args.max_competitors]
|
|
1059
|
+
# Two actor runs per competitor: never admit more domains than the
|
|
1060
|
+
# actor budget can finish in one attempt (cache makes retries free, but a
|
|
1061
|
+
# first attempt must be able to complete).
|
|
1062
|
+
domains = domains[: max(1, args.max_actor_runs // 2)]
|
|
1063
|
+
|
|
1064
|
+
inputs = RunInputs(
|
|
1065
|
+
domains=domains,
|
|
1066
|
+
domain_sources=domain_sources,
|
|
1067
|
+
eu_country=args.eu_country.strip().upper() or "DE",
|
|
1068
|
+
long_running_days=None,
|
|
1069
|
+
)
|
|
1070
|
+
client = OsClient(
|
|
1071
|
+
os_url=os_url,
|
|
1072
|
+
api_key=api_key,
|
|
1073
|
+
cache_dir=cache_dir,
|
|
1074
|
+
max_actor_runs=args.max_actor_runs,
|
|
1075
|
+
)
|
|
1076
|
+
|
|
1077
|
+
outcome = run_pipeline(client, inputs)
|
|
1078
|
+
outcome["evidence"]["inputs"].update(effective_policy=effective_policy, policy_sha256=policy_sha256)
|
|
1079
|
+
|
|
1080
|
+
evidence_path = workspace / EVIDENCE_REL
|
|
1081
|
+
evidence_json = json.dumps(outcome["evidence"], indent=1, ensure_ascii=False)
|
|
1082
|
+
_write_atomic(evidence_path, evidence_json)
|
|
1083
|
+
|
|
1084
|
+
section_file = runtime / "result.md"
|
|
1085
|
+
_write_atomic(section_file, outcome["section_markdown"])
|
|
1086
|
+
|
|
1087
|
+
upsert = _load_script("upsert-markdown-section.py", "pmr_upsert_markdown_section")
|
|
1088
|
+
report_path = workspace / REPORT_REL
|
|
1089
|
+
report_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1090
|
+
upsert.upsert_markdown_section(report_path, SECTION_HEADING, outcome["section_markdown"])
|
|
1091
|
+
|
|
1092
|
+
persist = _load_script("persist-research-report.py", "pmr_persist_research_report")
|
|
1093
|
+
persistence = persist.persist_research_report(
|
|
1094
|
+
workspace,
|
|
1095
|
+
commit_message=f"research: update {SECTION_HEADING.lower()} section",
|
|
1096
|
+
extra_paths=(Path(_artifact_path("egbe-docs/research/market/evidence/competitor-ads.json")),),
|
|
1097
|
+
)
|
|
1098
|
+
|
|
1099
|
+
state = {
|
|
1100
|
+
"schema_version": SCHEMA_VERSION,
|
|
1101
|
+
"section_id": SECTION_ID,
|
|
1102
|
+
"status": "completed",
|
|
1103
|
+
"domains": inputs.domains,
|
|
1104
|
+
"domain_sources": inputs.domain_sources,
|
|
1105
|
+
"eu_country": inputs.eu_country,
|
|
1106
|
+
"evidence_path": EVIDENCE_REL.as_posix(),
|
|
1107
|
+
"evidence_sha256": hashlib.sha256(evidence_json.encode("utf-8")).hexdigest(),
|
|
1108
|
+
"result_path": (RUNTIME_REL / "result.md").as_posix(),
|
|
1109
|
+
"result_sha256": hashlib.sha256(outcome["section_markdown"].encode("utf-8")).hexdigest(),
|
|
1110
|
+
"provider_calls": outcome["totals"]["provider_calls"],
|
|
1111
|
+
"actor_runs": outcome["totals"]["actor_runs"],
|
|
1112
|
+
"meta_searches": outcome["totals"]["meta_searches"],
|
|
1113
|
+
"attempt_actor_runs": outcome["attempt"]["actor_runs"],
|
|
1114
|
+
"attempt_meta_searches": outcome["attempt"]["meta_searches"],
|
|
1115
|
+
"attempt_cache_hits": outcome["attempt"]["cache_hits"],
|
|
1116
|
+
"report_commit_sha": persistence.commit_sha,
|
|
1117
|
+
"publication_mode": persistence.publication_mode,
|
|
1118
|
+
"research_context": _load_script("research-artifact-io.py", "pmr_input_io").context_binding(workspace, SECTION_ID),
|
|
1119
|
+
"effective_policy": effective_policy,
|
|
1120
|
+
"policy_sha256": policy_sha256,
|
|
1121
|
+
"error": None,
|
|
1122
|
+
}
|
|
1123
|
+
_write_atomic(runtime / "request.json", json.dumps(state, indent=1))
|
|
1124
|
+
return state
|
|
1125
|
+
|
|
1126
|
+
|
|
1127
|
+
def main() -> int:
|
|
1128
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
1129
|
+
parser.add_argument("--context-file", type=Path, help="Explicit context for this study; no project files are discovered implicitly.")
|
|
1130
|
+
parser.add_argument("--workspace", type=Path, required=True)
|
|
1131
|
+
parser.add_argument(
|
|
1132
|
+
"--domains",
|
|
1133
|
+
default="",
|
|
1134
|
+
help="Comma-separated competitor domains; merged ahead of the "
|
|
1135
|
+
"domains discovered by Keyword Economics.",
|
|
1136
|
+
)
|
|
1137
|
+
parser.add_argument("--max-competitors", type=int, default=MAX_COMPETITORS)
|
|
1138
|
+
parser.add_argument("--eu-country", default="DE")
|
|
1139
|
+
parser.add_argument("--max-actor-runs", type=int, default=MAX_ACTOR_RUNS_PER_ATTEMPT)
|
|
1140
|
+
args = parser.parse_args()
|
|
1141
|
+
|
|
1142
|
+
try:
|
|
1143
|
+
state = run(args.workspace, args)
|
|
1144
|
+
except (CompetitorAdsError, ValueError) as exc:
|
|
1145
|
+
print(f"{DISPLAY_NAME}: {exc}", file=sys.stderr)
|
|
1146
|
+
return 1
|
|
1147
|
+
print(
|
|
1148
|
+
f"{DISPLAY_NAME}: completed — {state['actor_runs']} actor run(s) + "
|
|
1149
|
+
f"{state['meta_searches']} Meta library search(es), evidence at "
|
|
1150
|
+
f"{state['evidence_path']}, report commit {state['report_commit_sha'][:12]}"
|
|
1151
|
+
)
|
|
1152
|
+
return 0
|
|
1153
|
+
|
|
1154
|
+
|
|
1155
|
+
if __name__ == "__main__":
|
|
1156
|
+
raise SystemExit(main())
|