@egbe-ai/opencode 0.0.0-stage → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +124 -2
- package/index.js +5 -0
- package/package.json +55 -4
- package/plugin.js +230 -0
- package/session-log.js +315 -0
- package/skills/.egbe-catalog/manifest.json +1 -0
- package/skills/egbe-os-analytics/SKILL.md +499 -0
- package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
- package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
- package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
- package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
- package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
- package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
- package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
- package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
- package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
- package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
- package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
- package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
- package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
- package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
- package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
- package/skills/egbe-os-db/SKILL.md +174 -0
- package/skills/egbe-os-deploy/SKILL.md +412 -0
- package/skills/egbe-os-design/SKILL.md +182 -0
- package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
- package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
- package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
- package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
- package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
- package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
- package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
- package/skills/egbe-os-destination-readiness/references/report.md +116 -0
- package/skills/egbe-os-domains/SKILL.md +189 -0
- package/skills/egbe-os-errors/SKILL.md +259 -0
- package/skills/egbe-os-git/SKILL.md +275 -0
- package/skills/egbe-os-git/references/issues.md +33 -0
- package/skills/egbe-os-google-ads/SKILL.md +163 -0
- package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
- package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
- package/skills/egbe-os-google-ads/references/conversions.md +153 -0
- package/skills/egbe-os-google-ads/references/delivery.md +128 -0
- package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
- package/skills/egbe-os-google-ads/references/reporting.md +105 -0
- package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
- package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
- package/skills/egbe-os-google-auth/SKILL.md +285 -0
- package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
- package/skills/egbe-os-images/SKILL.md +163 -0
- package/skills/egbe-os-images/references/stock-images.md +21 -0
- package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
- package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
- package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
- package/skills/egbe-os-llm/SKILL.md +160 -0
- package/skills/egbe-os-mail/SKILL.md +180 -0
- package/skills/egbe-os-me/SKILL.md +119 -0
- package/skills/egbe-os-meta-ads/SKILL.md +114 -0
- package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
- package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
- package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
- package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
- package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
- package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
- package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
- package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
- package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
- package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
- package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
- package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
- package/skills/egbe-os-notifications/SKILL.md +78 -0
- package/skills/egbe-os-openai-ads/SKILL.md +149 -0
- package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
- package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
- package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
- package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
- package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
- package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
- package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
- package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
- package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
- package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
- package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
- package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
- package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
- package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
- package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
- package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
- package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
- package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
- package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
- package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
- package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
- package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
- package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
- package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
- package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
- package/skills/egbe-os-payments/SKILL.md +215 -0
- package/skills/egbe-os-payments/agents/openai.yaml +4 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
- package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
- package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
- package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
- package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
- package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
- package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
- package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
- package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
- package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
- package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
- package/skills/egbe-os-playbook/SKILL.md +79 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
- package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
- package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
- package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
- package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
- package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
- package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
- package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
- package/skills/egbe-os-playbook/playbooks/product.md +67 -0
- package/skills/egbe-os-playbook/references/catalog.md +127 -0
- package/skills/egbe-os-playbook/references/changes.md +167 -0
- package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
- package/skills/egbe-os-product-auth/SKILL.md +151 -0
- package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
- package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
- package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
- package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
- package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
- package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
- package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
- package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
- package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
- package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
- package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
- package/skills/egbe-os-product-build/SKILL.md +111 -0
- package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-build/references/capabilities.md +25 -0
- package/skills/egbe-os-product-build/references/engineering.md +72 -0
- package/skills/egbe-os-product-build/references/execution.md +55 -0
- package/skills/egbe-os-product-build/references/foundations.md +93 -0
- package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
- package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
- package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
- package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
- package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
- package/skills/egbe-os-product-market-research/SKILL.md +159 -0
- package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
- package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
- package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
- package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
- package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
- package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
- package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
- package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
- package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
- package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
- package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
- package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
- package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
- package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
- package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
- package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
- package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
- package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
- package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
- package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
- package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
- package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
- package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
- package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
- package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
- package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
- package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
- package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
- package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
- package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
- package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
- package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
- package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
- package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
- package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
- package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
- package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
- package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
- package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
- package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
- package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
- package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
- package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
- package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
- package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
- package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
- package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
- package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
- package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
- package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
- package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
- package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
- package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
- package/skills/egbe-os-product-report/SKILL.md +150 -0
- package/skills/egbe-os-product-report/references/artifacts.md +84 -0
- package/skills/egbe-os-product-report/references/brief.md +34 -0
- package/skills/egbe-os-product-report/references/daily.md +76 -0
- package/skills/egbe-os-product-report/references/sources.md +68 -0
- package/skills/egbe-os-product-routines/SKILL.md +166 -0
- package/skills/egbe-os-product-routines/references/catalog.md +22 -0
- package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
- package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
- package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
- package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
- package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
- package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
- package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
- package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
- package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
- package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
- package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
- package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
- package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
- package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
- package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
- package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
- package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
- package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
- package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
- package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
- package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
- package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
- package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
- package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
- package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
- package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
- package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
- package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
- package/skills/egbe-os-report-ui/SKILL.md +133 -0
- package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
- package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
- package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
- package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
- package/skills/egbe-os-report-ui/references/components.md +209 -0
- package/skills/egbe-os-research/SKILL.md +125 -0
- package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
- package/skills/egbe-os-scrape/SKILL.md +160 -0
- package/skills/egbe-os-search/SKILL.md +182 -0
- package/skills/egbe-os-search/references/google-places.md +81 -0
- package/skills/egbe-os-support/SKILL.md +149 -0
- package/skills/egbe-os-telemetry/SKILL.md +344 -0
- package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
- package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
- package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
- package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
- package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
- package/skills/egbe-os-tools/SKILL.md +180 -0
- package/skills/egbe-os-tools/references/theona.md +26 -0
- package/skills/egbe-os-wallet/SKILL.md +181 -0
- package/skills/egbe-setup/EGBE.md +484 -0
- package/skills/egbe-setup/SKILL.md +116 -0
- package/skills/egbe-setup/references/project.md +80 -0
- package/state.js +37 -0
- package/update.js +361 -0
|
@@ -0,0 +1,2514 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Collect a bounded competitor comparison sample for caller-scoped research.
|
|
3
|
+
|
|
4
|
+
Discovery records offer-query results, related-domain reports, supplied profiles,
|
|
5
|
+
comparison roles and ad observations. Collection caps and sample order are not
|
|
6
|
+
business-priority rules. The caller owns comparison criteria and interpretation;
|
|
7
|
+
source identity, profile URLs, aliases, cache and receipts remain validated.
|
|
8
|
+
|
|
9
|
+
Caching, spend provenance, budgets and retries mirror the other measured
|
|
10
|
+
runners: every provider response is cached, starts are journaled before the
|
|
11
|
+
call, replays add no spend.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import sys as _artifact_sys
|
|
17
|
+
from pathlib import Path as _ArtifactPath
|
|
18
|
+
_artifact_sys.path.insert(0, str(_ArtifactPath(__file__).resolve().parent))
|
|
19
|
+
from egbe_artifacts import artifact_path as _artifact_path
|
|
20
|
+
|
|
21
|
+
import runpy as _egbe_runpy
|
|
22
|
+
from pathlib import Path as _EgbePath
|
|
23
|
+
_egbe_adapter = _EgbePath(__file__).with_name("connection_identity.py")
|
|
24
|
+
_connection_headers = _egbe_runpy.run_path(str(_egbe_adapter))["connection_headers"] if _egbe_adapter.is_file() else lambda _url: {}
|
|
25
|
+
|
|
26
|
+
import argparse
|
|
27
|
+
import hashlib
|
|
28
|
+
import importlib.util
|
|
29
|
+
import json
|
|
30
|
+
import math
|
|
31
|
+
import os
|
|
32
|
+
import re
|
|
33
|
+
import sys
|
|
34
|
+
import time
|
|
35
|
+
import urllib.error
|
|
36
|
+
import urllib.parse
|
|
37
|
+
import urllib.request
|
|
38
|
+
from dataclasses import asdict, dataclass, field
|
|
39
|
+
from datetime import date, timedelta
|
|
40
|
+
from pathlib import Path
|
|
41
|
+
from types import ModuleType
|
|
42
|
+
from typing import Any, Callable
|
|
43
|
+
|
|
44
|
+
SCHEMA_VERSION = 1
|
|
45
|
+
SECTION_ID = "competitor_set"
|
|
46
|
+
SECTION_HEADING = "Competitor set"
|
|
47
|
+
DISPLAY_NAME = "Competitor Set"
|
|
48
|
+
|
|
49
|
+
DEFAULT_OS_URL = "https://os.hz.egbe.dev"
|
|
50
|
+
RUNTIME_REL = Path(_artifact_path("egbe-docs/research/market/work")) / SECTION_ID
|
|
51
|
+
EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/competitors.json"))
|
|
52
|
+
REPORT_REL = Path(_artifact_path("egbe-docs/research/market/full-report.md"))
|
|
53
|
+
INTAKE_REL = Path(_artifact_path("egbe-docs/research/market/intake.json"))
|
|
54
|
+
CANDIDATES_FILENAME = "candidates.json"
|
|
55
|
+
LISTICLES_FILENAME = "listicles.json"
|
|
56
|
+
PROFILES_FILENAME = "profiles.json"
|
|
57
|
+
DECISIONS_FILENAME = "decisions.json"
|
|
58
|
+
REVIEW_FILENAME = "review.json"
|
|
59
|
+
DECISIONS_SCHEMA_VERSION = 2
|
|
60
|
+
|
|
61
|
+
MAX_COMPETITORS = 10
|
|
62
|
+
MAX_NAMED_DOMAINS = 10
|
|
63
|
+
MAX_PAGE_DOMAINS = 10
|
|
64
|
+
MAX_KEYWORD_DOMAINS = 20
|
|
65
|
+
MAX_CATEGORY_PHRASES = 3
|
|
66
|
+
MAX_BRAND_QUERIES = 3
|
|
67
|
+
MAX_CANDIDATES = 150
|
|
68
|
+
# Bounds of the agent's enrichment work (profiles per round, domains it may
|
|
69
|
+
# add from its own reading, listicle pages it reads).
|
|
70
|
+
# At most 40 profiles in all: the agent's reading, not the pool, is the cost.
|
|
71
|
+
MAX_PROFILES_ROUND1 = 25
|
|
72
|
+
MAX_PROFILES_ROUND2 = 10
|
|
73
|
+
MAX_AGENT_ADDITIONS = 5
|
|
74
|
+
MAX_LISTICLES_TO_READ = 12
|
|
75
|
+
LISTICLES_PER_QUERY = 5
|
|
76
|
+
MAX_LISTICLE_ENTRIES = 60
|
|
77
|
+
ROUND2_SEEDS = 3
|
|
78
|
+
MAX_AD_CHECKS = 25
|
|
79
|
+
AD_CHECK_MONTHS = 6
|
|
80
|
+
AD_CHECK_EXTRA_MARKETS = 2
|
|
81
|
+
KEYWORD_ADS_ROWS = 50
|
|
82
|
+
# SpyFu's keyword ad history reaches back years; only an advertiser seen on a
|
|
83
|
+
# phrase within this window counts as buying traffic on it now.
|
|
84
|
+
KEYWORD_ADS_RECENT_DAYS = 365
|
|
85
|
+
LISTICLES_PER_SITE = 2
|
|
86
|
+
COMPETITORS_DOMAIN_LIMIT = 20
|
|
87
|
+
SPYFU_COMPETITORS = 20
|
|
88
|
+
# One result page: deeper SERP tasks are assembled from several pages and
|
|
89
|
+
# DataForSEO rejects the whole task when one page cannot be retrieved
|
|
90
|
+
# (status 40106, "partial results"), observed live on 2026-10-01.
|
|
91
|
+
SERP_DEPTH = 10
|
|
92
|
+
BRAVE_COUNT = 20
|
|
93
|
+
META_ADS_LIMIT = 25
|
|
94
|
+
GTM_CONTAINERS_PER_SITE = 2
|
|
95
|
+
FETCH_MAX_BYTES = 1_000_000
|
|
96
|
+
FETCH_TIMEOUT_SECONDS = 25.0
|
|
97
|
+
RETRY_DELAY_SECONDS = 20.0
|
|
98
|
+
EXCLUDED_ROWS_RENDERED = 25
|
|
99
|
+
PROFILED_ROWS_RENDERED = 40
|
|
100
|
+
_TASK_ERROR_RE = re.compile(r"HTTP 400 .*dataforseo_task_error", re.DOTALL)
|
|
101
|
+
|
|
102
|
+
MAX_DATAFORSEO_TASKS_PER_ATTEMPT = 15
|
|
103
|
+
MAX_SEARCH_CALLS_PER_ATTEMPT = 12
|
|
104
|
+
MAX_SPYFU_CALLS_PER_ATTEMPT = 40
|
|
105
|
+
MAX_META_SEARCHES_PER_ATTEMPT = 25
|
|
106
|
+
MAX_FETCHES_PER_ATTEMPT = 60
|
|
107
|
+
|
|
108
|
+
DEFAULT_LOCATION_CODE = 2840
|
|
109
|
+
DEFAULT_LANGUAGE_CODE = "en"
|
|
110
|
+
DEFAULT_MARKET = "US"
|
|
111
|
+
DEFAULT_EU_COUNTRY = "DE"
|
|
112
|
+
# DataForSEO location codes for the markets an offer card usually names.
|
|
113
|
+
LOCATION_CODES = {
|
|
114
|
+
"US": 2840, "UK": 2826, "CA": 2124, "AU": 2036, "NZ": 2554, "IE": 2372, "DE": 2276, "FR": 2250,
|
|
115
|
+
"ES": 2724, "IT": 2380, "NL": 2528, "SE": 2752, "CZ": 2203, "PL": 2616, "PT": 2620, "AT": 2040,
|
|
116
|
+
"CH": 2756, "BE": 2056, "DK": 2208, "NO": 2578, "FI": 2246, "IN": 2356, "SG": 2702, "ZA": 2710,
|
|
117
|
+
"BR": 2076, "MX": 2484,
|
|
118
|
+
}
|
|
119
|
+
# SpyFu research markets (SpyFu says UK, not GB).
|
|
120
|
+
SPYFU_MARKETS = frozenset({
|
|
121
|
+
"AR", "AT", "AU", "BE", "BR", "CA", "CH", "DE", "DK", "ES", "FR", "IE", "IN", "IT", "JP", "MX",
|
|
122
|
+
"NL", "NO", "NZ", "PL", "PT", "SE", "SG", "TR", "UA", "UK", "US", "ZA",
|
|
123
|
+
})
|
|
124
|
+
EU_COUNTRIES = frozenset({
|
|
125
|
+
"AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE", "IT", "LV", "LT",
|
|
126
|
+
"LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE",
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
DECISION_TIERS = ("direct", "indirect", "substitute", "excluded")
|
|
130
|
+
OFFER_MATCHES = ("same", "comparable", "different", "none")
|
|
131
|
+
RESOLUTIONS = ("accepted", "rejected")
|
|
132
|
+
MISSING_RESOLUTIONS = ("added", "rejected")
|
|
133
|
+
PROVENANCE_ORDER = ("operator_named", "product_page", "discovered", "keyword_economics", "agent_research")
|
|
134
|
+
AD_ORDER = {"ads_seen": 0, "pixels_only": 1, "unknown": 2, "none_detected": 3}
|
|
135
|
+
AD_LABELS = {"ads_seen": "ads seen", "pixels_only": "ad pixels only", "unknown": "ad check inconclusive", "none_detected": "no ads detected"}
|
|
136
|
+
GROUP_LABELS = {0: "direct", 1: "indirect", 2: "other scoped"}
|
|
137
|
+
# Tag channels from the Channel Footprint signature list that mean paid media.
|
|
138
|
+
AD_TAG_CHANNELS = frozenset({
|
|
139
|
+
"meta_ads", "google_ads", "reddit_ads", "linkedin_ads", "tiktok_ads", "microsoft_ads", "x_ads",
|
|
140
|
+
"pinterest_ads", "snap_ads", "quora_ads", "display_native",
|
|
141
|
+
})
|
|
142
|
+
|
|
143
|
+
# Signal strength: offer signals (ads and rankings on the offer's own search
|
|
144
|
+
# phrases, listicle mentions) outweigh relationship hits. It orders the
|
|
145
|
+
# profiling list and breaks ties; it never decides a tier, and naming a
|
|
146
|
+
# competitor (founder or product page) adds nothing.
|
|
147
|
+
SCORE_CATEGORY_PAID = 3.0
|
|
148
|
+
SCORE_KEYWORD_ADS = 2.0
|
|
149
|
+
SCORE_CATEGORY_ORGANIC = 2.0
|
|
150
|
+
SCORE_SECOND_ENGINE = 1.0
|
|
151
|
+
SCORE_BRAND_QUERY = 1.0
|
|
152
|
+
SCORE_LISTICLE = 1.0
|
|
153
|
+
SCORE_RELATIONSHIP = 1.0
|
|
154
|
+
SCORE_KEYWORD_ECONOMICS = 1.0
|
|
155
|
+
CAP_CATEGORY_PAID = 2
|
|
156
|
+
CAP_KEYWORD_ADS = 3
|
|
157
|
+
CAP_CATEGORY_ORGANIC = 3
|
|
158
|
+
CAP_SECOND_ENGINE = 3
|
|
159
|
+
CAP_BRAND_QUERY = 2
|
|
160
|
+
CAP_LISTICLE = 3
|
|
161
|
+
CAP_RELATIONSHIP = 4
|
|
162
|
+
|
|
163
|
+
_LISTICLE_RE = re.compile(r"\b(alternatives?|best|top \d+|vs\.?|comparison|compare)\b", re.IGNORECASE)
|
|
164
|
+
_DOMAIN_RE = re.compile(r"^[a-z0-9][a-z0-9.-]{0,251}\.[a-z]{2,24}$")
|
|
165
|
+
_URL_RE = re.compile(r"^https?://[^\s]+$", re.IGNORECASE)
|
|
166
|
+
_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
|
167
|
+
_SEARCH_PAGE_KEYS = {"q", "query", "search", "keyword", "keywords", "term"}
|
|
168
|
+
PAGES_PER_CANDIDATE = 3
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class CompetitorSetError(RuntimeError):
|
|
172
|
+
"""Terminal runner failure; runtime state is preserved for resume."""
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class BudgetExhausted(CompetitorSetError):
|
|
176
|
+
"""A per-attempt call budget is spent; always propagates so a rerun
|
|
177
|
+
resumes from the cache instead of publishing a partial search."""
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _load_script(filename: str, module_name: str) -> ModuleType:
|
|
181
|
+
path = Path(__file__).resolve().parent / filename
|
|
182
|
+
spec = importlib.util.spec_from_file_location(module_name, path)
|
|
183
|
+
if spec is None or spec.loader is None:
|
|
184
|
+
raise CompetitorSetError(f"cannot load bundled script {filename}")
|
|
185
|
+
module = importlib.util.module_from_spec(spec)
|
|
186
|
+
sys.modules[spec.name] = module
|
|
187
|
+
previous = sys.dont_write_bytecode
|
|
188
|
+
sys.dont_write_bytecode = True
|
|
189
|
+
try:
|
|
190
|
+
spec.loader.exec_module(module)
|
|
191
|
+
finally:
|
|
192
|
+
sys.dont_write_bytecode = previous
|
|
193
|
+
return module
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
# Cache entries are stored gzip-compressed; plain entries of older runs replay too.
|
|
197
|
+
IO = _load_script("research-artifact-io.py", "pmr_competitor_set_io")
|
|
198
|
+
_KEYWORD_ECONOMICS: ModuleType | None = None
|
|
199
|
+
_CHANNEL_FOOTPRINT: ModuleType | None = None
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _keyword_economics() -> ModuleType:
|
|
203
|
+
"""Domain normalization and the noise/directory lists live in the Keyword
|
|
204
|
+
Economics runner; load them from there instead of copying the lists."""
|
|
205
|
+
global _KEYWORD_ECONOMICS
|
|
206
|
+
if _KEYWORD_ECONOMICS is None:
|
|
207
|
+
_KEYWORD_ECONOMICS = _load_script("run-keyword-economics.py", "pmr_keyword_economics_for_competitor_set")
|
|
208
|
+
return _KEYWORD_ECONOMICS
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _channel_footprint() -> ModuleType:
|
|
212
|
+
"""The ad-tag signatures, the SpyFu spend row and the Meta identity rules
|
|
213
|
+
live in the Channel Footprint runner; the ad check reuses them."""
|
|
214
|
+
global _CHANNEL_FOOTPRINT
|
|
215
|
+
if _CHANNEL_FOOTPRINT is None:
|
|
216
|
+
_CHANNEL_FOOTPRINT = _load_script("run-channel-footprint.py", "pmr_channel_footprint_for_competitor_set")
|
|
217
|
+
return _CHANNEL_FOOTPRINT
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def registrable_domain(value: str) -> str:
|
|
221
|
+
return _keyword_economics().registrable_domain(value)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def classify_domain(domain: str) -> str:
|
|
225
|
+
return _keyword_economics().classify_domain(domain)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def brand_token(domain: str) -> str:
|
|
229
|
+
return _keyword_economics().brand_token(domain)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def hosted_seller(domain: str) -> bool:
|
|
233
|
+
"""A seller's own subdomain on a course or creator host (kept whole by
|
|
234
|
+
the shared domain normalization)."""
|
|
235
|
+
return _keyword_economics().hosted_seller(domain)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _domain(value: Any) -> str:
|
|
239
|
+
domain = registrable_domain(str(value or ""))
|
|
240
|
+
return domain if domain and _DOMAIN_RE.match(domain) else ""
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _text(value: Any, cap: int = 240) -> str:
|
|
244
|
+
text = " ".join(str(value if value is not None else "").split())
|
|
245
|
+
return text if len(text) <= cap else text[: cap - 1] + "…"
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _url(value: Any) -> str:
|
|
249
|
+
text = str(value or "").strip()
|
|
250
|
+
return text if _URL_RE.match(text) and len(text) <= 800 else ""
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _number(value: Any) -> float | None:
|
|
254
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0:
|
|
255
|
+
raise ValueError("not a non-negative number")
|
|
256
|
+
return float(value)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _money(value: float | None) -> str:
|
|
260
|
+
if value is None:
|
|
261
|
+
return "—"
|
|
262
|
+
return f"${value:,.0f}"
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
# ── OS client: metered provider calls + free page fetches ───────────────────
|
|
266
|
+
|
|
267
|
+
@dataclass
|
|
268
|
+
class OsClient:
|
|
269
|
+
os_url: str
|
|
270
|
+
api_key: str
|
|
271
|
+
cache_dir: Path
|
|
272
|
+
max_dataforseo_tasks: int = MAX_DATAFORSEO_TASKS_PER_ATTEMPT
|
|
273
|
+
max_search_calls: int = MAX_SEARCH_CALLS_PER_ATTEMPT
|
|
274
|
+
max_spyfu_calls: int = MAX_SPYFU_CALLS_PER_ATTEMPT
|
|
275
|
+
max_meta_searches: int = MAX_META_SEARCHES_PER_ATTEMPT
|
|
276
|
+
max_fetches: int = MAX_FETCHES_PER_ATTEMPT
|
|
277
|
+
dataforseo_tasks: int = 0
|
|
278
|
+
search_calls: int = 0
|
|
279
|
+
spyfu_calls: int = 0
|
|
280
|
+
meta_searches: int = 0
|
|
281
|
+
fetches: int = 0
|
|
282
|
+
cache_hits: int = 0
|
|
283
|
+
touched: set[str] = field(default_factory=set)
|
|
284
|
+
request: Callable[..., dict[str, Any]] | None = None
|
|
285
|
+
fetch: Callable[[str], tuple[int, str]] | None = None
|
|
286
|
+
sleep: Callable[[float], None] | None = None
|
|
287
|
+
|
|
288
|
+
def _http(self, path: str, payload: dict[str, Any], timeout: float, *, idempotency_key: str | None = None) -> dict[str, Any]:
|
|
289
|
+
url = f"{self.os_url.rstrip('/')}{path}"
|
|
290
|
+
headers = {
|
|
291
|
+
"Authorization": f"Bearer {self.api_key}",
|
|
292
|
+
"Content-Type": "application/json",
|
|
293
|
+
"Accept": "application/json",
|
|
294
|
+
}
|
|
295
|
+
if idempotency_key:
|
|
296
|
+
headers["Idempotency-Key"] = idempotency_key
|
|
297
|
+
req = urllib.request.Request(url, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST")
|
|
298
|
+
try:
|
|
299
|
+
for _name, _value in _connection_headers(req.full_url).items():
|
|
300
|
+
req.add_unredirected_header(_name, _value)
|
|
301
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
302
|
+
envelope = json.loads(resp.read().decode("utf-8"))
|
|
303
|
+
except urllib.error.HTTPError as exc:
|
|
304
|
+
detail = exc.read().decode("utf-8", errors="replace")[:500]
|
|
305
|
+
raise CompetitorSetError(f"OS {path} failed: HTTP {exc.code} {detail}") from exc
|
|
306
|
+
except urllib.error.URLError as exc:
|
|
307
|
+
raise CompetitorSetError(f"OS {path} unreachable: {exc.reason}") from exc
|
|
308
|
+
except (OSError, ValueError) as exc:
|
|
309
|
+
raise CompetitorSetError(f"OS {path} failed in transit: {type(exc).__name__}: {str(exc)[:200]}") from exc
|
|
310
|
+
data = envelope.get("data")
|
|
311
|
+
if envelope.get("code") not in (None, 200) or not isinstance(data, dict):
|
|
312
|
+
raise CompetitorSetError(f"OS {path} returned no data: {str(envelope)[:300]}")
|
|
313
|
+
return data
|
|
314
|
+
|
|
315
|
+
def _fetch_url(self, url: str) -> tuple[int, str]:
|
|
316
|
+
"""Plain GET of a public page; the status and a bounded body."""
|
|
317
|
+
req = urllib.request.Request(url, headers={
|
|
318
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
|
|
319
|
+
"Accept": "text/html,application/xhtml+xml,application/javascript;q=0.9,*/*;q=0.8",
|
|
320
|
+
"Accept-Language": "en-US,en;q=0.9",
|
|
321
|
+
})
|
|
322
|
+
try:
|
|
323
|
+
for _name, _value in _connection_headers(req.full_url).items():
|
|
324
|
+
req.add_unredirected_header(_name, _value)
|
|
325
|
+
with urllib.request.urlopen(req, timeout=FETCH_TIMEOUT_SECONDS) as resp:
|
|
326
|
+
body = resp.read(FETCH_MAX_BYTES + 1)
|
|
327
|
+
status = int(getattr(resp, "status", 200) or 200)
|
|
328
|
+
except urllib.error.HTTPError as exc:
|
|
329
|
+
return exc.code, ""
|
|
330
|
+
except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
|
|
331
|
+
raise CompetitorSetError(f"fetch of {url} failed: {getattr(exc, 'reason', exc)}") from exc
|
|
332
|
+
return status, body[:FETCH_MAX_BYTES].decode("utf-8", errors="replace")
|
|
333
|
+
|
|
334
|
+
def _cache_read(self, cache_file: Path) -> dict[str, Any] | None:
|
|
335
|
+
try:
|
|
336
|
+
return IO.read_cache_entry(cache_file)
|
|
337
|
+
except (OSError, ValueError, EOFError):
|
|
338
|
+
IO.drop_cache_entry(cache_file)
|
|
339
|
+
return None
|
|
340
|
+
|
|
341
|
+
def _cache_write(self, cache_file: Path, entry: dict[str, Any]) -> None:
|
|
342
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
343
|
+
IO.write_cache_entry(cache_file, entry)
|
|
344
|
+
|
|
345
|
+
def _journal_start(self, digest: str) -> None:
|
|
346
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
347
|
+
with (self.cache_dir / "starts.log").open("a", encoding="utf-8") as handle:
|
|
348
|
+
handle.write(f"{date.today().isoformat()} {digest}\n")
|
|
349
|
+
|
|
350
|
+
def _prior_start_count(self, digest: str) -> int:
|
|
351
|
+
try:
|
|
352
|
+
lines = (self.cache_dir / "starts.log").read_text(encoding="utf-8").splitlines()
|
|
353
|
+
except OSError:
|
|
354
|
+
return 0
|
|
355
|
+
return sum(1 for line in lines if line.endswith(f" {digest}"))
|
|
356
|
+
|
|
357
|
+
def _metered(self, kind: str, path: str, payload: dict[str, Any], timeout: float, *, budget_attr: str, budget_max: int, label: str) -> tuple[dict[str, Any], str]:
|
|
358
|
+
key_raw = json.dumps({"kind": kind, "path": path, "payload": payload}, sort_keys=True)
|
|
359
|
+
digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
|
|
360
|
+
cache_file = self.cache_dir / f"{digest}.json"
|
|
361
|
+
self.touched.add(digest)
|
|
362
|
+
entry = self._cache_read(cache_file)
|
|
363
|
+
if entry is not None:
|
|
364
|
+
self.cache_hits += 1
|
|
365
|
+
response = entry.get("response")
|
|
366
|
+
return (response if isinstance(response, dict) else {}), entry.get("retrieved_on") or date.today().isoformat()
|
|
367
|
+
|
|
368
|
+
transport = self.request or self._http
|
|
369
|
+
attempts = 0
|
|
370
|
+
wire_key = f"{digest}-{self._prior_start_count(digest)}"
|
|
371
|
+
while True:
|
|
372
|
+
# Metered starts count BEFORE the transport call (fail closed) and
|
|
373
|
+
# are journaled durably: a failed start may still have billed.
|
|
374
|
+
if getattr(self, budget_attr) >= budget_max:
|
|
375
|
+
raise BudgetExhausted(
|
|
376
|
+
f"{label} budget exhausted ({budget_max} per attempt); rerun the same command to "
|
|
377
|
+
"continue from the cache"
|
|
378
|
+
)
|
|
379
|
+
setattr(self, budget_attr, getattr(self, budget_attr) + 1)
|
|
380
|
+
self._journal_start(digest)
|
|
381
|
+
try:
|
|
382
|
+
response = transport(path, payload, timeout, idempotency_key=wire_key)
|
|
383
|
+
break
|
|
384
|
+
except CompetitorSetError as exc:
|
|
385
|
+
if (kind == "dataforseo" and _TASK_ERROR_RE.search(str(exc))) or (
|
|
386
|
+
kind in ("spyfu", "meta") and re.search(r"HTTP 4(?:00|22)\b", str(exc))
|
|
387
|
+
):
|
|
388
|
+
# A rejected task (bad input, partial result pages, a
|
|
389
|
+
# market SpyFu does not cover) is a fact about this query,
|
|
390
|
+
# not a transient failure: cache it so replays stay
|
|
391
|
+
# byte-stable and never re-spend.
|
|
392
|
+
response = {"unavailable": str(exc)[:300]}
|
|
393
|
+
break
|
|
394
|
+
if attempts >= 1 or not re.search(r"HTTP 5\d\d", str(exc)):
|
|
395
|
+
raise
|
|
396
|
+
attempts += 1
|
|
397
|
+
(self.sleep or time.sleep)(RETRY_DELAY_SECONDS)
|
|
398
|
+
retrieved_on = date.today().isoformat()
|
|
399
|
+
self._cache_write(cache_file, {"kind": kind, "path": path, "payload": payload, "response": response, "retrieved_on": retrieved_on})
|
|
400
|
+
return (response if isinstance(response, dict) else {}), retrieved_on
|
|
401
|
+
|
|
402
|
+
def dataforseo(self, operation: str, payload: dict[str, Any]) -> tuple[dict[str, Any], str]:
|
|
403
|
+
return self._metered(
|
|
404
|
+
"dataforseo", f"/tools/dataforseo/{operation}", payload, 180.0,
|
|
405
|
+
budget_attr="dataforseo_tasks", budget_max=self.max_dataforseo_tasks, label="DataForSEO task",
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
def brave(self, query: str, count: int, country: str) -> tuple[dict[str, Any], str]:
|
|
409
|
+
return self._metered(
|
|
410
|
+
"search", "/search/web/brave", {"query": query, "count": count, "country": country}, 60.0,
|
|
411
|
+
budget_attr="search_calls", budget_max=self.max_search_calls, label="web search",
|
|
412
|
+
)
|
|
413
|
+
|
|
414
|
+
def spyfu(self, operation: str, payload: dict[str, Any]) -> tuple[dict[str, Any], str]:
|
|
415
|
+
return self._metered(
|
|
416
|
+
"spyfu", f"/tools/spyfu/{operation}", payload, 90.0,
|
|
417
|
+
budget_attr="spyfu_calls", budget_max=self.max_spyfu_calls, label="SpyFu lookup",
|
|
418
|
+
)
|
|
419
|
+
|
|
420
|
+
def meta_library(self, payload: dict[str, Any]) -> tuple[dict[str, Any], str]:
|
|
421
|
+
return self._metered(
|
|
422
|
+
"meta", "/tools/meta-ad-library/search", payload, 90.0,
|
|
423
|
+
budget_attr="meta_searches", budget_max=self.max_meta_searches, label="Meta Ad Library search",
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
def fetch_page(self, url: str) -> tuple[dict[str, Any], str]:
|
|
427
|
+
"""Free, cached GET of a public page. A timeout, a rate limit or a
|
|
428
|
+
server error is reported for this attempt and never cached."""
|
|
429
|
+
key_raw = json.dumps({"kind": "fetch", "url": url}, sort_keys=True)
|
|
430
|
+
digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
|
|
431
|
+
cache_file = self.cache_dir / f"{digest}.json"
|
|
432
|
+
self.touched.add(digest)
|
|
433
|
+
entry = self._cache_read(cache_file)
|
|
434
|
+
if entry is not None:
|
|
435
|
+
self.cache_hits += 1
|
|
436
|
+
cached = entry.get("response")
|
|
437
|
+
return (cached if isinstance(cached, dict) else {}), entry.get("retrieved_on") or date.today().isoformat()
|
|
438
|
+
if self.fetches >= self.max_fetches:
|
|
439
|
+
raise BudgetExhausted(f"page fetch budget exhausted ({self.max_fetches} per attempt); rerun the same command to continue from the cache")
|
|
440
|
+
self.fetches += 1
|
|
441
|
+
transport = self.fetch or self._fetch_url
|
|
442
|
+
retrieved_on = date.today().isoformat()
|
|
443
|
+
try:
|
|
444
|
+
status, body = transport(url)
|
|
445
|
+
except CompetitorSetError as exc:
|
|
446
|
+
return {"status": None, "body": "", "error": str(exc)[:200]}, retrieved_on
|
|
447
|
+
if status == 429 or status >= 500:
|
|
448
|
+
return {"status": status, "body": "", "error": f"HTTP {status}"}, retrieved_on
|
|
449
|
+
response = {"status": status, "body": body}
|
|
450
|
+
self._cache_write(cache_file, {"kind": "fetch", "url": url, "response": response, "retrieved_on": retrieved_on})
|
|
451
|
+
return response, retrieved_on
|
|
452
|
+
|
|
453
|
+
def ledger_totals(self) -> dict[str, Any]:
|
|
454
|
+
"""Durable provenance over entries THIS execution touched. A cached
|
|
455
|
+
provider rejection is a journaled start, not a successful call: it
|
|
456
|
+
never counts as evidence that the market was searched."""
|
|
457
|
+
counts = {"dataforseo": 0, "search": 0, "spyfu": 0, "meta": 0, "fetch": 0}
|
|
458
|
+
rejected = 0
|
|
459
|
+
dates: list[str] = []
|
|
460
|
+
if self.cache_dir.is_dir():
|
|
461
|
+
for digest in sorted(self.touched):
|
|
462
|
+
try:
|
|
463
|
+
entry = IO.read_cache_entry(self.cache_dir / f"{digest}.json")
|
|
464
|
+
except (OSError, ValueError, EOFError):
|
|
465
|
+
continue
|
|
466
|
+
if not isinstance(entry, dict):
|
|
467
|
+
continue
|
|
468
|
+
kind = entry.get("kind")
|
|
469
|
+
response = entry.get("response") if isinstance(entry.get("response"), dict) else {}
|
|
470
|
+
if response.get("unavailable"):
|
|
471
|
+
rejected += 1
|
|
472
|
+
elif kind in counts:
|
|
473
|
+
counts[kind] += 1
|
|
474
|
+
if isinstance(entry.get("retrieved_on"), str):
|
|
475
|
+
dates.append(entry["retrieved_on"])
|
|
476
|
+
today = date.today().isoformat()
|
|
477
|
+
try:
|
|
478
|
+
starts_recorded = sum(1 for line in (self.cache_dir / "starts.log").read_text(encoding="utf-8").splitlines() if line)
|
|
479
|
+
except OSError:
|
|
480
|
+
starts_recorded = 0
|
|
481
|
+
provider_calls = counts["dataforseo"] + counts["search"] + counts["spyfu"] + counts["meta"]
|
|
482
|
+
return {
|
|
483
|
+
"dataforseo_calls": counts["dataforseo"],
|
|
484
|
+
"search_calls": counts["search"],
|
|
485
|
+
"spyfu_calls": counts["spyfu"],
|
|
486
|
+
"meta_searches": counts["meta"],
|
|
487
|
+
"page_fetches": counts["fetch"],
|
|
488
|
+
"provider_calls": provider_calls,
|
|
489
|
+
"rejected_calls": rejected,
|
|
490
|
+
"provider_starts_recorded": starts_recorded,
|
|
491
|
+
"first_retrieved_on": min(dates) if dates else today,
|
|
492
|
+
"last_retrieved_on": max(dates) if dates else today,
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def _call_with_context(client: OsClient, subject: str, label: str, fn: Callable[[], Any]) -> Any:
|
|
497
|
+
try:
|
|
498
|
+
return fn()
|
|
499
|
+
except BudgetExhausted:
|
|
500
|
+
raise
|
|
501
|
+
except CompetitorSetError as exc:
|
|
502
|
+
raise CompetitorSetError(
|
|
503
|
+
f"{label} for `{subject}` failed: {exc} — this attempt had already started "
|
|
504
|
+
f"{client.dataforseo_tasks}/{client.max_dataforseo_tasks} DataForSEO task(s) and "
|
|
505
|
+
f"{client.search_calls}/{client.max_search_calls} web search(es); see "
|
|
506
|
+
"GET /me/limits?resource=dataforseo:tasks for remaining daily quota."
|
|
507
|
+
) from exc
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
# ── inputs: the offer card and the two kinds of named competitors ───────────
|
|
511
|
+
|
|
512
|
+
def parse_terms(raw: str, cap: int) -> list[str]:
|
|
513
|
+
seen: list[str] = []
|
|
514
|
+
for term in (raw or "").split(","):
|
|
515
|
+
value = " ".join(term.split()).strip().lower()
|
|
516
|
+
if value and value not in seen:
|
|
517
|
+
seen.append(value)
|
|
518
|
+
return seen[:cap]
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def parse_domains(raw: Any, cap: int) -> list[str]:
|
|
522
|
+
terms = raw if isinstance(raw, list) else str(raw or "").split(",")
|
|
523
|
+
domains: list[str] = []
|
|
524
|
+
for term in terms:
|
|
525
|
+
domain = _domain(term)
|
|
526
|
+
if domain and domain not in domains:
|
|
527
|
+
domains.append(domain)
|
|
528
|
+
return domains[:cap]
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def normalize_market(value: Any) -> str:
|
|
532
|
+
code = str(value or "").strip().upper()
|
|
533
|
+
return "UK" if code == "GB" else code
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def _brave_country(market: str) -> str:
|
|
537
|
+
return "gb" if market == "UK" else market.lower()
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def load_intake(path: Path) -> dict[str, Any] | None:
|
|
541
|
+
"""Read optional typed operation inputs; never infer labels from prose."""
|
|
542
|
+
if not path.is_file():
|
|
543
|
+
return None
|
|
544
|
+
try:
|
|
545
|
+
raw = json.loads(path.read_text(encoding="utf-8"))
|
|
546
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
547
|
+
raise CompetitorSetError(f"{path.name} is not valid JSON: {exc}") from exc
|
|
548
|
+
return validate_intake(raw)
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def validate_intake(raw: Any) -> dict[str, Any]:
|
|
552
|
+
"""Validate supplied fields, not the completeness of a research brief."""
|
|
553
|
+
if not isinstance(raw, dict) or type(raw.get("schema_version")) is not int or raw["schema_version"] != 1:
|
|
554
|
+
raise CompetitorSetError("intake must be an object with schema_version 1")
|
|
555
|
+
|
|
556
|
+
def text(value, field):
|
|
557
|
+
if value is not None and not isinstance(value, str):
|
|
558
|
+
raise CompetitorSetError(f"intake {field} must be text or null")
|
|
559
|
+
return " ".join((value or "").split()) or None
|
|
560
|
+
|
|
561
|
+
def strings(value, field):
|
|
562
|
+
if value is None:
|
|
563
|
+
return []
|
|
564
|
+
if not isinstance(value, list) or any(not isinstance(item, str) or not item.strip() for item in value):
|
|
565
|
+
raise CompetitorSetError(f"intake {field} must be a list of nonempty strings")
|
|
566
|
+
return list(dict.fromkeys(item.strip() for item in value))
|
|
567
|
+
|
|
568
|
+
def boolean(value, field):
|
|
569
|
+
if value is not None and type(value) is not bool:
|
|
570
|
+
raise CompetitorSetError(f"intake {field} must be boolean or null")
|
|
571
|
+
return value
|
|
572
|
+
|
|
573
|
+
def domains(value, field, cap):
|
|
574
|
+
values = strings(value, field)
|
|
575
|
+
if any(not _domain(item) for item in values):
|
|
576
|
+
raise CompetitorSetError(f"intake {field} contains an invalid domain")
|
|
577
|
+
normalized = parse_domains(values, len(values))
|
|
578
|
+
if len(normalized) > cap:
|
|
579
|
+
raise CompetitorSetError(f"intake {field} exceeds the collection limit of {cap}")
|
|
580
|
+
return normalized
|
|
581
|
+
|
|
582
|
+
def languages(value, field):
|
|
583
|
+
values = list(dict.fromkeys(item.lower() for item in strings(value, field)))
|
|
584
|
+
if len(values) > 40 or any(not re.fullmatch(r"[a-z]{2,3}(?:-[a-z0-9]{2,8})*", item) for item in values):
|
|
585
|
+
raise CompetitorSetError(f"intake {field} must contain explicit ISO/BCP-47 language codes")
|
|
586
|
+
return values
|
|
587
|
+
|
|
588
|
+
offer_raw = raw.get("offer")
|
|
589
|
+
if offer_raw is not None and not isinstance(offer_raw, dict):
|
|
590
|
+
raise CompetitorSetError("intake offer must be an object or null")
|
|
591
|
+
offer_raw = offer_raw or {}
|
|
592
|
+
offer: dict[str, Any] = {}
|
|
593
|
+
for key in ("name", "job", "buyer", "offer_class", "comparable_classes", "duration", "price_text", "price_floor_basis", "basis"):
|
|
594
|
+
value = text(offer_raw.get(key), f"offer.{key}")
|
|
595
|
+
if value is not None:
|
|
596
|
+
offer[key] = value
|
|
597
|
+
offer["url"] = _url(offer_raw.get("url"))
|
|
598
|
+
if offer_raw.get("url") and not offer["url"]:
|
|
599
|
+
raise CompetitorSetError("intake offer.url must be an HTTP(S) address")
|
|
600
|
+
for key in ("price_usd", "price_floor_usd"):
|
|
601
|
+
value = offer_raw.get(key)
|
|
602
|
+
if value is None:
|
|
603
|
+
offer[key] = None
|
|
604
|
+
continue
|
|
605
|
+
try:
|
|
606
|
+
offer[key] = _number(value)
|
|
607
|
+
except ValueError:
|
|
608
|
+
raise CompetitorSetError(f"intake offer.{key} must be a finite non-negative number or null") from None
|
|
609
|
+
phrases = strings(offer_raw.get("category_phrases"), "offer.category_phrases")
|
|
610
|
+
phrases = parse_terms(",".join(p.replace(",", " ") for p in phrases), len(phrases))
|
|
611
|
+
if len(phrases) > MAX_CATEGORY_PHRASES:
|
|
612
|
+
raise CompetitorSetError(f"intake offer.category_phrases exceeds the paid-query limit of {MAX_CATEGORY_PHRASES}")
|
|
613
|
+
offer["category_phrases"] = phrases
|
|
614
|
+
offer["confirmed_by_operator"] = boolean(offer_raw.get("confirmed_by_operator"), "offer.confirmed_by_operator")
|
|
615
|
+
offer["inferred"] = boolean(offer_raw.get("inferred"), "offer.inferred")
|
|
616
|
+
if offer["confirmed_by_operator"] and offer["inferred"]:
|
|
617
|
+
raise CompetitorSetError("intake offer cannot be both confirmed and inferred")
|
|
618
|
+
# Without a stated basis, neither a confirmation nor an inference is established.
|
|
619
|
+
if not offer.get("basis"):
|
|
620
|
+
offer["confirmed_by_operator"] = None
|
|
621
|
+
offer["inferred"] = None
|
|
622
|
+
markets: list[str] = []
|
|
623
|
+
for market in strings(raw.get("markets"), "markets"):
|
|
624
|
+
code = normalize_market(market)
|
|
625
|
+
if not re.fullmatch(r"[A-Z]{2}", code) or code == "EU":
|
|
626
|
+
raise CompetitorSetError("intake markets must contain explicit two-letter country codes")
|
|
627
|
+
if code not in markets:
|
|
628
|
+
markets.append(code)
|
|
629
|
+
market_languages = raw.get("market_languages") or {}
|
|
630
|
+
if not isinstance(market_languages, dict):
|
|
631
|
+
raise CompetitorSetError("intake market_languages must be an object")
|
|
632
|
+
by_market = {}
|
|
633
|
+
for country, values in market_languages.items():
|
|
634
|
+
code = normalize_market(country).replace("UK", "GB")
|
|
635
|
+
if not re.fullmatch(r"[A-Z]{2}", code) or code == "EU" or code in by_market:
|
|
636
|
+
raise CompetitorSetError("intake market_languages needs unique explicit country codes")
|
|
637
|
+
by_market[code] = languages(values, f"market_languages.{country}")
|
|
638
|
+
if not by_market[code]:
|
|
639
|
+
raise CompetitorSetError(f"intake market_languages.{country} cannot be empty")
|
|
640
|
+
named = domains(raw.get("operator_named"), "operator_named", MAX_NAMED_DOMAINS)
|
|
641
|
+
asked = boolean(raw.get("operator_named_asked"), "operator_named_asked")
|
|
642
|
+
if named and asked is False:
|
|
643
|
+
raise CompetitorSetError("intake operator_named contradicts operator_named_asked=false")
|
|
644
|
+
ltv = raw.get("ltv_usd")
|
|
645
|
+
if ltv is not None:
|
|
646
|
+
try:
|
|
647
|
+
ltv = _number(ltv)
|
|
648
|
+
except ValueError:
|
|
649
|
+
raise CompetitorSetError("intake ltv_usd must be a finite non-negative number or null") from None
|
|
650
|
+
ltv_provenance = raw.get("ltv_provenance")
|
|
651
|
+
if ltv_provenance not in (None, "given", "estimated", "unknown"):
|
|
652
|
+
raise CompetitorSetError("intake ltv_provenance must be given, estimated, unknown or null")
|
|
653
|
+
if ltv is None and ltv_provenance in ("given", "estimated"):
|
|
654
|
+
raise CompetitorSetError("intake ltv_provenance has no amount")
|
|
655
|
+
ltv_basis = text(raw.get("ltv_basis"), "ltv_basis")
|
|
656
|
+
if not ltv_basis:
|
|
657
|
+
ltv_provenance = None
|
|
658
|
+
return {
|
|
659
|
+
"offer": offer,
|
|
660
|
+
"operator_named": named,
|
|
661
|
+
"operator_named_asked": asked,
|
|
662
|
+
"page_mentioned": domains(raw.get("page_mentioned"), "page_mentioned", MAX_PAGE_DOMAINS),
|
|
663
|
+
"product_domain": _domain(raw.get("product_domain")),
|
|
664
|
+
"markets": markets,
|
|
665
|
+
"languages": languages(raw.get("languages"), "languages"),
|
|
666
|
+
"market_languages": by_market,
|
|
667
|
+
"ltv_usd": ltv,
|
|
668
|
+
"ltv_provenance": ltv_provenance,
|
|
669
|
+
"ltv_basis": ltv_basis,
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
def domains_from_keyword_evidence(workspace: Path, cap: int) -> list[str]:
|
|
674
|
+
evidence = _load_script("research-evidence.py", "pmr_evidence_for_competitor_set")
|
|
675
|
+
return evidence.keyword_inputs(workspace, consumer=SECTION_ID, row_type="competitor_domain", field="domain", cap=cap)
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
def category_queries(phrases: list[str]) -> list[str]:
|
|
679
|
+
"""Per category phrase: the phrase itself (who pays for and ranks on the
|
|
680
|
+
offer's own search) and its "best" list (listicles)."""
|
|
681
|
+
queries: list[str] = []
|
|
682
|
+
for phrase in phrases:
|
|
683
|
+
for query in (phrase, f"best {phrase}"):
|
|
684
|
+
if query not in queries:
|
|
685
|
+
queries.append(query)
|
|
686
|
+
return queries
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
def brand_queries(named_domains: list[str]) -> list[str]:
|
|
690
|
+
"""Alternatives queries for competitors named by the founder only, with their
|
|
691
|
+
own budget so they never displace the category queries."""
|
|
692
|
+
queries: list[str] = []
|
|
693
|
+
for domain in named_domains:
|
|
694
|
+
brand = brand_token(domain)
|
|
695
|
+
query = f"{brand} alternatives" if brand else ""
|
|
696
|
+
if query and query not in queries:
|
|
697
|
+
queries.append(query)
|
|
698
|
+
return queries[:MAX_BRAND_QUERIES]
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
@dataclass
|
|
702
|
+
class RunInputs:
|
|
703
|
+
named_domains: list[str]
|
|
704
|
+
page_domains: list[str]
|
|
705
|
+
keyword_domains: list[str]
|
|
706
|
+
product_domain: str
|
|
707
|
+
category_phrases: list[str]
|
|
708
|
+
category_queries: list[str]
|
|
709
|
+
brand_queries: list[str]
|
|
710
|
+
offer: dict[str, Any]
|
|
711
|
+
price_floor_usd: float | None
|
|
712
|
+
markets: list[str]
|
|
713
|
+
location_code: int
|
|
714
|
+
language_code: str
|
|
715
|
+
country: str
|
|
716
|
+
spyfu_market: str
|
|
717
|
+
eu_country: str
|
|
718
|
+
intake_path: str | None
|
|
719
|
+
intake_sha256: str | None
|
|
720
|
+
input_annotations: dict[str, Any] = field(default_factory=dict)
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def _arg(args: argparse.Namespace, name: str, default: Any) -> Any:
|
|
724
|
+
value = getattr(args, name, None)
|
|
725
|
+
return default if value is None else value
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
def resolve_inputs(workspace: Path, args: argparse.Namespace) -> RunInputs:
|
|
729
|
+
intake_arg = str(_arg(args, "intake", "") or "")
|
|
730
|
+
intake_path = Path(intake_arg) if intake_arg else workspace / INTAKE_REL
|
|
731
|
+
if not intake_path.is_absolute():
|
|
732
|
+
intake_path = workspace / intake_path
|
|
733
|
+
if intake_arg and not intake_path.is_file():
|
|
734
|
+
raise CompetitorSetError(f"the supplied intake file does not exist: {intake_arg}")
|
|
735
|
+
intake = load_intake(intake_path) or {}
|
|
736
|
+
offer = dict(intake.get("offer") or {})
|
|
737
|
+
|
|
738
|
+
product_domain = _domain(_arg(args, "product_domain", "")) or intake.get("product_domain") or _domain(offer.get("url"))
|
|
739
|
+
named_flag = str(_arg(args, "domains", "") or "")
|
|
740
|
+
named = parse_domains(named_flag, MAX_NAMED_DOMAINS) if named_flag.strip() else list(intake.get("operator_named") or [])
|
|
741
|
+
page_flag = str(_arg(args, "page_domains", "") or "")
|
|
742
|
+
page = parse_domains(page_flag, MAX_PAGE_DOMAINS) if page_flag.strip() else list(intake.get("page_mentioned") or [])
|
|
743
|
+
named = [d for d in named if d != product_domain][:MAX_NAMED_DOMAINS]
|
|
744
|
+
page = [d for d in page if d != product_domain and d not in named][:MAX_PAGE_DOMAINS]
|
|
745
|
+
keyword_domains = [d for d in domains_from_keyword_evidence(workspace, MAX_KEYWORD_DOMAINS) if d != product_domain and d not in named and d not in page]
|
|
746
|
+
phrases_flag = str(_arg(args, "seed_keywords", "") or "")
|
|
747
|
+
phrases = parse_terms(phrases_flag, MAX_CATEGORY_PHRASES) if phrases_flag.strip() else list(offer.get("category_phrases") or [])
|
|
748
|
+
if not named and not phrases:
|
|
749
|
+
raise CompetitorSetError(
|
|
750
|
+
"no competitor domains or seed queries available: supply typed operation inputs "
|
|
751
|
+
f"in {INTAKE_REL.as_posix()}, or pass --seed-keywords and/or --domains "
|
|
752
|
+
"for the assigned comparison (see the workflow doc)"
|
|
753
|
+
)
|
|
754
|
+
floor_flag = _arg(args, "price_floor_usd", None)
|
|
755
|
+
try:
|
|
756
|
+
floor = _number(floor_flag) if floor_flag is not None else offer.get("price_floor_usd")
|
|
757
|
+
except ValueError:
|
|
758
|
+
raise CompetitorSetError("--price-floor-usd must be a non-negative number") from None
|
|
759
|
+
offer["category_phrases"] = phrases
|
|
760
|
+
|
|
761
|
+
markets = list(intake.get("markets") or [])
|
|
762
|
+
country_flag = normalize_market(_arg(args, "country", ""))
|
|
763
|
+
if country_flag:
|
|
764
|
+
markets = [country_flag] + [m for m in markets if m != country_flag]
|
|
765
|
+
if not markets:
|
|
766
|
+
markets = [DEFAULT_MARKET]
|
|
767
|
+
primary = markets[0]
|
|
768
|
+
location_flag = _arg(args, "location_code", None)
|
|
769
|
+
location_code = int(location_flag) if location_flag is not None else LOCATION_CODES.get(primary, DEFAULT_LOCATION_CODE)
|
|
770
|
+
eu_flag = normalize_market(_arg(args, "eu_country", ""))
|
|
771
|
+
eu_country = eu_flag or next((m for m in markets if m in EU_COUNTRIES), DEFAULT_EU_COUNTRY)
|
|
772
|
+
intake_sha = hashlib.sha256(intake_path.read_bytes()).hexdigest() if intake_path.is_file() else None
|
|
773
|
+
try:
|
|
774
|
+
intake_rel = intake_path.relative_to(workspace).as_posix()
|
|
775
|
+
except ValueError:
|
|
776
|
+
intake_rel = intake_path.as_posix()
|
|
777
|
+
return RunInputs(
|
|
778
|
+
named_domains=named,
|
|
779
|
+
page_domains=page,
|
|
780
|
+
keyword_domains=keyword_domains,
|
|
781
|
+
product_domain=product_domain,
|
|
782
|
+
category_phrases=phrases,
|
|
783
|
+
category_queries=category_queries(phrases),
|
|
784
|
+
brand_queries=brand_queries(named),
|
|
785
|
+
offer=offer,
|
|
786
|
+
price_floor_usd=floor,
|
|
787
|
+
markets=markets,
|
|
788
|
+
location_code=location_code,
|
|
789
|
+
language_code=str(_arg(args, "language_code", DEFAULT_LANGUAGE_CODE) or DEFAULT_LANGUAGE_CODE),
|
|
790
|
+
country=_brave_country(primary),
|
|
791
|
+
spyfu_market=primary,
|
|
792
|
+
eu_country=eu_country,
|
|
793
|
+
intake_path=intake_rel if intake_sha else None,
|
|
794
|
+
intake_sha256=intake_sha,
|
|
795
|
+
input_annotations={
|
|
796
|
+
key: intake.get(key)
|
|
797
|
+
for key in ("operator_named_asked", "languages", "market_languages", "ltv_usd", "ltv_provenance", "ltv_basis")
|
|
798
|
+
} | {"operator_named_asked": True if named_flag.strip() else intake.get("operator_named_asked")},
|
|
799
|
+
)
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
def validate_resume_inputs(workspace: Path, args: argparse.Namespace, inputs: RunInputs) -> None:
|
|
803
|
+
"""Check explicit changes without rebuilding the saved collection scope."""
|
|
804
|
+
comparisons = {
|
|
805
|
+
"domains": (inputs.named_domains, lambda value: [d for d in parse_domains(value, MAX_NAMED_DOMAINS) if d != inputs.product_domain]),
|
|
806
|
+
"page_domains": (inputs.page_domains, lambda value: [d for d in parse_domains(value, MAX_PAGE_DOMAINS) if d != inputs.product_domain and d not in inputs.named_domains]),
|
|
807
|
+
"product_domain": (inputs.product_domain, _domain),
|
|
808
|
+
"seed_keywords": (inputs.category_phrases, lambda value: parse_terms(value, MAX_CATEGORY_PHRASES)),
|
|
809
|
+
"price_floor_usd": (inputs.price_floor_usd, _number),
|
|
810
|
+
"location_code": (inputs.location_code, int),
|
|
811
|
+
"language_code": (inputs.language_code, str),
|
|
812
|
+
"country": (inputs.markets[0], normalize_market),
|
|
813
|
+
"eu_country": (inputs.eu_country, normalize_market),
|
|
814
|
+
}
|
|
815
|
+
for name, (expected, normalize) in comparisons.items():
|
|
816
|
+
supplied = _arg(args, name, None)
|
|
817
|
+
if supplied in (None, ""):
|
|
818
|
+
continue
|
|
819
|
+
try:
|
|
820
|
+
matches = normalize(supplied) == expected
|
|
821
|
+
except (TypeError, ValueError):
|
|
822
|
+
matches = False
|
|
823
|
+
if not matches:
|
|
824
|
+
raise CompetitorSetError("competitor collection inputs changed; retain this study and use a new research ID")
|
|
825
|
+
|
|
826
|
+
# Source applicability is separate from replay identity. Retained sources
|
|
827
|
+
# must still match, but a clone need not carry the original source file.
|
|
828
|
+
if inputs.intake_path:
|
|
829
|
+
source = Path(inputs.intake_path)
|
|
830
|
+
source = source if source.is_absolute() else workspace / source
|
|
831
|
+
if source.is_file() and hashlib.sha256(source.read_bytes()).hexdigest() != inputs.intake_sha256:
|
|
832
|
+
raise CompetitorSetError("competitor collection inputs changed: declared intake source changed; review saved evidence applicability")
|
|
833
|
+
intake_arg = str(_arg(args, "intake", "") or "")
|
|
834
|
+
if intake_arg:
|
|
835
|
+
source = Path(intake_arg)
|
|
836
|
+
source = source if source.is_absolute() else workspace / source
|
|
837
|
+
if not source.is_file():
|
|
838
|
+
raise CompetitorSetError(f"the supplied intake file does not exist: {intake_arg}")
|
|
839
|
+
if hashlib.sha256(source.read_bytes()).hexdigest() != inputs.intake_sha256:
|
|
840
|
+
raise CompetitorSetError("competitor collection inputs changed; retain this study and use a new research ID")
|
|
841
|
+
|
|
842
|
+
|
|
843
|
+
def bind_inputs(workspace: Path, args: argparse.Namespace) -> RunInputs:
|
|
844
|
+
"""Freeze the collection inputs before provider calls; output caps are not inputs."""
|
|
845
|
+
path = workspace / RUNTIME_REL / "inputs.json"
|
|
846
|
+
saved = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else None
|
|
847
|
+
if saved is not None:
|
|
848
|
+
if not isinstance(saved, dict) or saved.get("schema_version") != 1 or not isinstance(saved.get("inputs"), dict):
|
|
849
|
+
raise CompetitorSetError("invalid saved competitor inputs; preserve the state and inspect it")
|
|
850
|
+
digest = hashlib.sha256(json.dumps(saved["inputs"], sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
|
851
|
+
if saved.get("sha256") != digest:
|
|
852
|
+
raise CompetitorSetError("saved competitor inputs failed their integrity check")
|
|
853
|
+
inputs = RunInputs(**saved["inputs"])
|
|
854
|
+
validate_resume_inputs(workspace, args, inputs)
|
|
855
|
+
return inputs
|
|
856
|
+
inputs = resolve_inputs(workspace, args)
|
|
857
|
+
payload = asdict(inputs)
|
|
858
|
+
current = {"schema_version": 1, "inputs": payload,
|
|
859
|
+
"sha256": hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest()}
|
|
860
|
+
_write_atomic(path, json.dumps(current, indent=2, sort_keys=True, ensure_ascii=False) + "\n")
|
|
861
|
+
return inputs
|
|
862
|
+
|
|
863
|
+
|
|
864
|
+
# ── candidates ──────────────────────────────────────────────────────────────
|
|
865
|
+
|
|
866
|
+
@dataclass
|
|
867
|
+
class Candidate:
|
|
868
|
+
domain: str
|
|
869
|
+
provenance: list[str] = field(default_factory=list)
|
|
870
|
+
sources: list[str] = field(default_factory=list)
|
|
871
|
+
paid_queries: list[str] = field(default_factory=list)
|
|
872
|
+
organic_queries: list[str] = field(default_factory=list)
|
|
873
|
+
search_queries: list[str] = field(default_factory=list)
|
|
874
|
+
brand_pairs: list[str] = field(default_factory=list)
|
|
875
|
+
keyword_ads_phrases: list[str] = field(default_factory=list)
|
|
876
|
+
keyword_ads_rows: int = 0
|
|
877
|
+
keyword_ads_last_seen: str | None = None
|
|
878
|
+
listicle_pages: list[str] = field(default_factory=list)
|
|
879
|
+
overlap_seeds: list[str] = field(default_factory=list)
|
|
880
|
+
overlap_intersections: int = 0
|
|
881
|
+
ppc_seeds: list[str] = field(default_factory=list)
|
|
882
|
+
spyfu_common_keywords: int = 0
|
|
883
|
+
listed_for: list[str] = field(default_factory=list)
|
|
884
|
+
found_in: str | None = None
|
|
885
|
+
round: int = 1
|
|
886
|
+
aliases: list[str] = field(default_factory=list)
|
|
887
|
+
pages: list[str] = field(default_factory=list)
|
|
888
|
+
|
|
889
|
+
def add_provenance(self, kind: str) -> None:
|
|
890
|
+
if kind not in self.provenance:
|
|
891
|
+
self.provenance.append(kind)
|
|
892
|
+
|
|
893
|
+
def add_source(self, source: str) -> None:
|
|
894
|
+
if source not in self.sources:
|
|
895
|
+
self.sources.append(source)
|
|
896
|
+
|
|
897
|
+
def listed(self, query: str) -> None:
|
|
898
|
+
if query not in self.listed_for:
|
|
899
|
+
self.listed_for.append(query)
|
|
900
|
+
|
|
901
|
+
def seen_on(self, url: str) -> None:
|
|
902
|
+
"""The first pages that ranked or advertised: profiling starts there
|
|
903
|
+
(a reseller's or host's page shows what the domain really sells)."""
|
|
904
|
+
page = _url(url.split("#", 1)[0])
|
|
905
|
+
if page and page not in self.pages and len(self.pages) < PAGES_PER_CANDIDATE:
|
|
906
|
+
self.pages.append(page)
|
|
907
|
+
|
|
908
|
+
@staticmethod
|
|
909
|
+
def _add(values: list[str], value: str) -> None:
|
|
910
|
+
if value not in values:
|
|
911
|
+
values.append(value)
|
|
912
|
+
|
|
913
|
+
@property
|
|
914
|
+
def serp_paid_hits(self) -> int:
|
|
915
|
+
return len(self.paid_queries)
|
|
916
|
+
|
|
917
|
+
@property
|
|
918
|
+
def serp_organic_hits(self) -> int:
|
|
919
|
+
return len(self.organic_queries)
|
|
920
|
+
|
|
921
|
+
@property
|
|
922
|
+
def search_hits(self) -> int:
|
|
923
|
+
return len(self.search_queries)
|
|
924
|
+
|
|
925
|
+
@property
|
|
926
|
+
def brand_hits(self) -> int:
|
|
927
|
+
return len(self.brand_pairs)
|
|
928
|
+
|
|
929
|
+
def offer_signals(self) -> int:
|
|
930
|
+
"""Distinct offer signals: one per query and engine, one per phrase a
|
|
931
|
+
recent advertiser bought."""
|
|
932
|
+
return self.serp_paid_hits + self.serp_organic_hits + self.search_hits + len(self.keyword_ads_phrases)
|
|
933
|
+
|
|
934
|
+
def relationship_hits(self) -> int:
|
|
935
|
+
return len(set(self.overlap_seeds)) + len(set(self.ppc_seeds))
|
|
936
|
+
|
|
937
|
+
def round1_score(self) -> float:
|
|
938
|
+
score = SCORE_CATEGORY_PAID * min(self.serp_paid_hits, CAP_CATEGORY_PAID)
|
|
939
|
+
score += SCORE_KEYWORD_ADS * min(len(self.keyword_ads_phrases), CAP_KEYWORD_ADS)
|
|
940
|
+
score += SCORE_CATEGORY_ORGANIC * min(self.serp_organic_hits, CAP_CATEGORY_ORGANIC)
|
|
941
|
+
score += SCORE_SECOND_ENGINE * min(self.search_hits, CAP_SECOND_ENGINE)
|
|
942
|
+
score += SCORE_BRAND_QUERY * min(self.brand_hits, CAP_BRAND_QUERY)
|
|
943
|
+
score += SCORE_LISTICLE * min(len(self.listicle_pages), CAP_LISTICLE)
|
|
944
|
+
if "keyword_economics" in self.provenance:
|
|
945
|
+
score += SCORE_KEYWORD_ECONOMICS
|
|
946
|
+
return round(score, 2)
|
|
947
|
+
|
|
948
|
+
def score(self) -> float:
|
|
949
|
+
return round(self.round1_score() + SCORE_RELATIONSHIP * min(self.relationship_hits(), CAP_RELATIONSHIP), 2)
|
|
950
|
+
|
|
951
|
+
def signal_count(self) -> int:
|
|
952
|
+
"""Independent places the domain showed up: offer signals, listicle
|
|
953
|
+
pages and alternatives-query hits."""
|
|
954
|
+
return self.offer_signals() + len(self.listicle_pages) + self.brand_hits
|
|
955
|
+
|
|
956
|
+
def offer_signal(self) -> bool:
|
|
957
|
+
"""Worth profiling: ranks or pays on Google for the offer's own
|
|
958
|
+
search, or shows up in two independent places (two queries, two
|
|
959
|
+
engines, an advertised phrase plus a listicle, two listicles); a
|
|
960
|
+
single second-engine result, a single advertised phrase or a single
|
|
961
|
+
listicle mention alone is a weak hit."""
|
|
962
|
+
return bool(self.serp_paid_hits or self.serp_organic_hits or self.signal_count() >= 2)
|
|
963
|
+
|
|
964
|
+
def absorb(self, alias: "Candidate") -> None:
|
|
965
|
+
"""Take over the discovery facts of another domain of the same
|
|
966
|
+
company (idempotent: every list is a set)."""
|
|
967
|
+
for name in ("provenance", "sources", "paid_queries", "organic_queries", "search_queries", "brand_pairs", "keyword_ads_phrases",
|
|
968
|
+
"listicle_pages", "overlap_seeds", "ppc_seeds", "listed_for"):
|
|
969
|
+
values = getattr(self, name)
|
|
970
|
+
for value in getattr(alias, name):
|
|
971
|
+
Candidate._add(values, value)
|
|
972
|
+
for page in alias.pages:
|
|
973
|
+
self.seen_on(page)
|
|
974
|
+
if alias.keyword_ads_last_seen and (self.keyword_ads_last_seen is None or alias.keyword_ads_last_seen > self.keyword_ads_last_seen):
|
|
975
|
+
self.keyword_ads_last_seen = alias.keyword_ads_last_seen
|
|
976
|
+
self.overlap_intersections = max(self.overlap_intersections, alias.overlap_intersections)
|
|
977
|
+
self.spyfu_common_keywords = max(self.spyfu_common_keywords, alias.spyfu_common_keywords)
|
|
978
|
+
Candidate._add(self.aliases, alias.domain)
|
|
979
|
+
|
|
980
|
+
def round1_signal(self) -> bool:
|
|
981
|
+
return bool(self.serp_paid_hits or self.keyword_ads_phrases or self.serp_organic_hits or self.search_hits
|
|
982
|
+
or self.brand_hits or self.listicle_pages or "keyword_economics" in self.provenance)
|
|
983
|
+
|
|
984
|
+
def primary_provenance(self) -> str:
|
|
985
|
+
return next((p for p in PROVENANCE_ORDER if p in self.provenance), "discovered")
|
|
986
|
+
|
|
987
|
+
def ordered_provenance(self) -> list[str]:
|
|
988
|
+
return [p for p in PROVENANCE_ORDER if p in self.provenance]
|
|
989
|
+
|
|
990
|
+
def row(self) -> dict[str, Any]:
|
|
991
|
+
return {
|
|
992
|
+
"type": "candidate",
|
|
993
|
+
"domain": self.domain,
|
|
994
|
+
"provenance": self.ordered_provenance(),
|
|
995
|
+
"round": self.round,
|
|
996
|
+
"sources": list(self.sources),
|
|
997
|
+
"source_count": len(self.sources),
|
|
998
|
+
"serp_paid_hits": self.serp_paid_hits,
|
|
999
|
+
"serp_organic_hits": self.serp_organic_hits,
|
|
1000
|
+
"search_hits": self.search_hits,
|
|
1001
|
+
"brand_hits": self.brand_hits,
|
|
1002
|
+
"keyword_ads_hits": len(self.keyword_ads_phrases),
|
|
1003
|
+
"listicle_pages": len(self.listicle_pages),
|
|
1004
|
+
"relationship_hits": self.relationship_hits(),
|
|
1005
|
+
"overlap_intersections": self.overlap_intersections,
|
|
1006
|
+
"spyfu_ppc_hits": len(set(self.ppc_seeds)),
|
|
1007
|
+
"spyfu_common_keywords": self.spyfu_common_keywords,
|
|
1008
|
+
"listed_for": list(self.listed_for[:8]),
|
|
1009
|
+
"pages": list(self.pages),
|
|
1010
|
+
"found_in": self.found_in,
|
|
1011
|
+
"aliases": list(self.aliases),
|
|
1012
|
+
"score": self.score(),
|
|
1013
|
+
"tier": "B",
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
|
|
1017
|
+
def _candidate(pool: dict[str, Candidate], domain: str, round_no: int = 1) -> Candidate:
|
|
1018
|
+
if domain not in pool:
|
|
1019
|
+
pool[domain] = Candidate(domain=domain, round=round_no)
|
|
1020
|
+
return pool[domain]
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
def _serp_items(response: dict[str, Any]) -> list[dict[str, Any]]:
|
|
1024
|
+
result = response.get("result")
|
|
1025
|
+
entry = (result or [{}])[0] if isinstance(result, list) else {}
|
|
1026
|
+
items = entry.get("items") if isinstance(entry, dict) else None
|
|
1027
|
+
return [item for item in (items or []) if isinstance(item, dict)]
|
|
1028
|
+
|
|
1029
|
+
|
|
1030
|
+
def _listicle(url: str, title: str) -> bool:
|
|
1031
|
+
return bool(_LISTICLE_RE.search(title or "")) or bool(_LISTICLE_RE.search(url or ""))
|
|
1032
|
+
|
|
1033
|
+
|
|
1034
|
+
def _summary(round_no: int, query: str, kind: str, source: str, retrieved: int, found: int, listicles: list[str], unavailable: str | None = None) -> dict[str, Any]:
|
|
1035
|
+
row = {"type": "search_summary", "round": round_no, "query": query, "query_kind": kind, "source": source,
|
|
1036
|
+
"retrieved": retrieved, "candidates": found, "listicles": listicles, "tier": "A"}
|
|
1037
|
+
if unavailable:
|
|
1038
|
+
row["unavailable"] = unavailable[:200]
|
|
1039
|
+
return row
|
|
1040
|
+
|
|
1041
|
+
|
|
1042
|
+
@dataclass
|
|
1043
|
+
class Discovery:
|
|
1044
|
+
pool: dict[str, Candidate]
|
|
1045
|
+
directories: dict[str, list[str]]
|
|
1046
|
+
summaries: list[dict[str, Any]]
|
|
1047
|
+
|
|
1048
|
+
|
|
1049
|
+
def run_round1(client: OsClient, inputs: RunInputs) -> Discovery:
|
|
1050
|
+
"""Discovery from the offer: category phrases first, then the founder's
|
|
1051
|
+
brands; core Google results fail the attempt, SpyFu and Brave are best
|
|
1052
|
+
effort."""
|
|
1053
|
+
pool: dict[str, Candidate] = {}
|
|
1054
|
+
directories: dict[str, list[str]] = {}
|
|
1055
|
+
summaries: list[dict[str, Any]] = []
|
|
1056
|
+
for domain in inputs.named_domains:
|
|
1057
|
+
_candidate(pool, domain).add_provenance("operator_named")
|
|
1058
|
+
for domain in inputs.page_domains:
|
|
1059
|
+
_candidate(pool, domain).add_provenance("product_page")
|
|
1060
|
+
for domain in inputs.keyword_domains:
|
|
1061
|
+
candidate = _candidate(pool, domain)
|
|
1062
|
+
candidate.add_provenance("keyword_economics")
|
|
1063
|
+
candidate.add_source("keyword_economics")
|
|
1064
|
+
|
|
1065
|
+
locale = {"location_code": inputs.location_code, "language_code": inputs.language_code}
|
|
1066
|
+
queries = [(q, "category") for q in inputs.category_queries] + [(q, "brand") for q in inputs.brand_queries]
|
|
1067
|
+
|
|
1068
|
+
def admit(domain: str, query: str) -> str:
|
|
1069
|
+
"""`candidate`, `skip` or `directory` for a result domain."""
|
|
1070
|
+
if not domain or domain == inputs.product_domain:
|
|
1071
|
+
return "skip"
|
|
1072
|
+
kind = classify_domain(domain)
|
|
1073
|
+
if kind == "noise":
|
|
1074
|
+
return "skip"
|
|
1075
|
+
if kind == "directory":
|
|
1076
|
+
directories.setdefault(domain, []).append(query)
|
|
1077
|
+
return "directory"
|
|
1078
|
+
return "candidate"
|
|
1079
|
+
|
|
1080
|
+
for query, kind in queries:
|
|
1081
|
+
response, _ = _call_with_context(client, query, "DataForSEO serp", lambda q=query: client.dataforseo(
|
|
1082
|
+
"serp", {"keyword": q, "depth": SERP_DEPTH, **locale}
|
|
1083
|
+
))
|
|
1084
|
+
if response.get("unavailable"):
|
|
1085
|
+
summaries.append(_summary(1, query, kind, "dataforseo_serp", 0, 0, [], str(response["unavailable"])))
|
|
1086
|
+
continue
|
|
1087
|
+
retrieved = found = 0
|
|
1088
|
+
listicles: list[str] = []
|
|
1089
|
+
for item in _serp_items(response):
|
|
1090
|
+
item_type = item.get("type")
|
|
1091
|
+
if item_type not in ("paid", "organic"):
|
|
1092
|
+
continue
|
|
1093
|
+
retrieved += 1
|
|
1094
|
+
url = str(item.get("url") or "")
|
|
1095
|
+
domain = _domain(item.get("domain") or url)
|
|
1096
|
+
verdict = admit(domain, query)
|
|
1097
|
+
if verdict == "skip":
|
|
1098
|
+
continue
|
|
1099
|
+
if verdict == "directory" or (item_type == "organic" and _listicle(url, str(item.get("title") or ""))):
|
|
1100
|
+
if url and len(listicles) < LISTICLES_PER_QUERY and url not in listicles:
|
|
1101
|
+
listicles.append(url)
|
|
1102
|
+
continue
|
|
1103
|
+
candidate = _candidate(pool, domain)
|
|
1104
|
+
candidate.add_provenance("discovered")
|
|
1105
|
+
if kind == "category":
|
|
1106
|
+
if item_type == "paid":
|
|
1107
|
+
candidate.add_source("serp_paid")
|
|
1108
|
+
Candidate._add(candidate.paid_queries, query)
|
|
1109
|
+
else:
|
|
1110
|
+
candidate.add_source("serp_organic")
|
|
1111
|
+
Candidate._add(candidate.organic_queries, query)
|
|
1112
|
+
else:
|
|
1113
|
+
candidate.add_source("brand_query")
|
|
1114
|
+
Candidate._add(candidate.brand_pairs, f"serp:{query}")
|
|
1115
|
+
candidate.listed(query)
|
|
1116
|
+
candidate.seen_on(url)
|
|
1117
|
+
found += 1
|
|
1118
|
+
summaries.append(_summary(1, query, kind, "dataforseo_serp", retrieved, found, listicles))
|
|
1119
|
+
|
|
1120
|
+
# SpyFu: the domains that bought Google ads on the offer's own phrases.
|
|
1121
|
+
for phrase in inputs.category_phrases if client.max_spyfu_calls > 0 else []:
|
|
1122
|
+
query = f"ads:{phrase}"
|
|
1123
|
+
try:
|
|
1124
|
+
response, retrieved_on = client.spyfu("keyword-ads", {"keyword": phrase, "country_code": inputs.spyfu_market, "max_rows": KEYWORD_ADS_ROWS})
|
|
1125
|
+
except BudgetExhausted:
|
|
1126
|
+
raise
|
|
1127
|
+
except CompetitorSetError as exc:
|
|
1128
|
+
summaries.append(_summary(1, query, "category", "spyfu_keyword_ads", 0, 0, [], str(exc)))
|
|
1129
|
+
continue
|
|
1130
|
+
if response.get("unavailable"):
|
|
1131
|
+
summaries.append(_summary(1, query, "category", "spyfu_keyword_ads", 0, 0, [], str(response["unavailable"])))
|
|
1132
|
+
continue
|
|
1133
|
+
# Recency is measured from the day the answer was retrieved, so a
|
|
1134
|
+
# cached replay classifies the same advertisers the same way.
|
|
1135
|
+
try:
|
|
1136
|
+
cutoff = (date.fromisoformat(retrieved_on) - timedelta(days=KEYWORD_ADS_RECENT_DAYS)).isoformat()
|
|
1137
|
+
except ValueError:
|
|
1138
|
+
cutoff = (date.today() - timedelta(days=KEYWORD_ADS_RECENT_DAYS)).isoformat()
|
|
1139
|
+
retrieved = 0
|
|
1140
|
+
advertisers: list[str] = []
|
|
1141
|
+
for item in response.get("rows") or []:
|
|
1142
|
+
if not isinstance(item, dict):
|
|
1143
|
+
continue
|
|
1144
|
+
retrieved += 1
|
|
1145
|
+
seen = str(item.get("seen_on") or "")[:10]
|
|
1146
|
+
if not _DATE_RE.match(seen) or seen < cutoff:
|
|
1147
|
+
continue # an advertiser of years ago does not buy this traffic now
|
|
1148
|
+
urls = item.get("urls") if isinstance(item.get("urls"), list) else []
|
|
1149
|
+
domain = _domain(item.get("advertiser") or (urls[0] if urls else ""))
|
|
1150
|
+
if admit(domain, query) != "candidate":
|
|
1151
|
+
continue
|
|
1152
|
+
candidate = _candidate(pool, domain)
|
|
1153
|
+
candidate.add_provenance("discovered")
|
|
1154
|
+
candidate.add_source("spyfu_keyword_ads")
|
|
1155
|
+
Candidate._add(candidate.keyword_ads_phrases, phrase)
|
|
1156
|
+
candidate.keyword_ads_rows += 1
|
|
1157
|
+
if candidate.keyword_ads_last_seen is None or seen > candidate.keyword_ads_last_seen:
|
|
1158
|
+
candidate.keyword_ads_last_seen = seen
|
|
1159
|
+
candidate.listed(query)
|
|
1160
|
+
for ad_url in urls[:1]:
|
|
1161
|
+
candidate.seen_on(str(ad_url))
|
|
1162
|
+
Candidate._add(advertisers, domain)
|
|
1163
|
+
summaries.append(_summary(1, query, "category", "spyfu_keyword_ads", retrieved, len(advertisers), []))
|
|
1164
|
+
|
|
1165
|
+
for query, kind in queries:
|
|
1166
|
+
try:
|
|
1167
|
+
response, _ = client.brave(query, BRAVE_COUNT, inputs.country)
|
|
1168
|
+
except BudgetExhausted:
|
|
1169
|
+
raise # the attempt stops; the next invocation finishes the queries from the cache
|
|
1170
|
+
except CompetitorSetError as exc:
|
|
1171
|
+
# Best-effort second engine: an undeployed provider or a provider
|
|
1172
|
+
# rate limit is recorded, never a failed section.
|
|
1173
|
+
summaries.append(_summary(1, query, kind, "brave", 0, 0, [], str(exc)))
|
|
1174
|
+
continue
|
|
1175
|
+
retrieved = found = 0
|
|
1176
|
+
listicles = []
|
|
1177
|
+
for item in response.get("results") or []:
|
|
1178
|
+
if not isinstance(item, dict):
|
|
1179
|
+
continue
|
|
1180
|
+
retrieved += 1
|
|
1181
|
+
url = str(item.get("url") or "")
|
|
1182
|
+
domain = _domain(url)
|
|
1183
|
+
verdict = admit(domain, query)
|
|
1184
|
+
if verdict == "skip":
|
|
1185
|
+
continue
|
|
1186
|
+
if verdict == "directory" or _listicle(url, str(item.get("title") or "")):
|
|
1187
|
+
if url and len(listicles) < LISTICLES_PER_QUERY and url not in listicles:
|
|
1188
|
+
listicles.append(url)
|
|
1189
|
+
continue
|
|
1190
|
+
candidate = _candidate(pool, domain)
|
|
1191
|
+
candidate.add_provenance("discovered")
|
|
1192
|
+
if kind == "category":
|
|
1193
|
+
candidate.add_source("brave")
|
|
1194
|
+
Candidate._add(candidate.search_queries, query)
|
|
1195
|
+
else:
|
|
1196
|
+
candidate.add_source("brand_query")
|
|
1197
|
+
Candidate._add(candidate.brand_pairs, f"brave:{query}")
|
|
1198
|
+
candidate.listed(query)
|
|
1199
|
+
candidate.seen_on(url)
|
|
1200
|
+
found += 1
|
|
1201
|
+
summaries.append(_summary(1, query, kind, "brave", retrieved, found, listicles))
|
|
1202
|
+
return Discovery(pool=pool, directories=directories, summaries=summaries)
|
|
1203
|
+
|
|
1204
|
+
|
|
1205
|
+
def listicles_to_read(summaries: list[dict[str, Any]], product_domain: str) -> list[str]:
|
|
1206
|
+
"""The bounded reading list: "best" lists of the category phrases first,
|
|
1207
|
+
then the other category results, then the brand queries; directories and
|
|
1208
|
+
the product's own pages are never on it."""
|
|
1209
|
+
def priority(row: dict[str, Any]) -> tuple[int, int]:
|
|
1210
|
+
if row.get("query_kind") == "brand":
|
|
1211
|
+
group = 2
|
|
1212
|
+
else:
|
|
1213
|
+
group = 0 if str(row.get("query") or "").startswith("best ") else 1
|
|
1214
|
+
return group, 0 if row.get("source") == "dataforseo_serp" else 1
|
|
1215
|
+
|
|
1216
|
+
urls: list[str] = []
|
|
1217
|
+
per_site: dict[str, int] = {}
|
|
1218
|
+
for row in sorted((s for s in summaries if s.get("round") == 1), key=priority):
|
|
1219
|
+
for url in row.get("listicles") or []:
|
|
1220
|
+
domain = _domain(url)
|
|
1221
|
+
if not domain or domain == product_domain or classify_domain(domain) == "directory" or url in urls:
|
|
1222
|
+
continue
|
|
1223
|
+
if _search_page(url) or per_site.get(domain, 0) >= LISTICLES_PER_SITE:
|
|
1224
|
+
continue # a site's own search results are no list; two pages per site keep the views varied
|
|
1225
|
+
per_site[domain] = per_site.get(domain, 0) + 1
|
|
1226
|
+
urls.append(url)
|
|
1227
|
+
return urls[:MAX_LISTICLES_TO_READ]
|
|
1228
|
+
|
|
1229
|
+
|
|
1230
|
+
def _search_page(url: str) -> bool:
|
|
1231
|
+
parsed = urllib.parse.urlparse(url)
|
|
1232
|
+
keys = {key.lower() for key in urllib.parse.parse_qs(parsed.query)}
|
|
1233
|
+
return bool(keys & _SEARCH_PAGE_KEYS) or "/search" in parsed.path.lower()
|
|
1234
|
+
|
|
1235
|
+
|
|
1236
|
+
def run_round2(client: OsClient, inputs: RunInputs, discovery: Discovery, seeds: list[str]) -> None:
|
|
1237
|
+
"""Keyword overlap and PPC competitors of the confirmed direct domains.
|
|
1238
|
+
The overlap report is core evidence; SpyFu is best effort."""
|
|
1239
|
+
pool, summaries = discovery.pool, discovery.summaries
|
|
1240
|
+
locale = {"location_code": inputs.location_code, "language_code": inputs.language_code}
|
|
1241
|
+
|
|
1242
|
+
def admit(domain: str, seed: str, query: str) -> Candidate | None:
|
|
1243
|
+
if not domain or domain in (seed, inputs.product_domain):
|
|
1244
|
+
return None
|
|
1245
|
+
kind = classify_domain(domain)
|
|
1246
|
+
if kind == "noise":
|
|
1247
|
+
return None
|
|
1248
|
+
if kind == "directory":
|
|
1249
|
+
discovery.directories.setdefault(domain, []).append(query)
|
|
1250
|
+
return None
|
|
1251
|
+
candidate = _candidate(pool, domain, 2)
|
|
1252
|
+
candidate.add_provenance("discovered")
|
|
1253
|
+
return candidate
|
|
1254
|
+
|
|
1255
|
+
for seed in seeds:
|
|
1256
|
+
query = f"overlap:{seed}"
|
|
1257
|
+
response, _ = _call_with_context(client, seed, "DataForSEO competitors-domain", lambda s=seed: client.dataforseo(
|
|
1258
|
+
"competitors-domain", {"target": s, "limit": COMPETITORS_DOMAIN_LIMIT, **locale}
|
|
1259
|
+
))
|
|
1260
|
+
if response.get("unavailable"):
|
|
1261
|
+
summaries.append(_summary(2, query, "relationship", "dataforseo_competitors_domain", 0, 0, [], str(response["unavailable"])))
|
|
1262
|
+
continue
|
|
1263
|
+
retrieved = found = 0
|
|
1264
|
+
for item in _serp_items(response):
|
|
1265
|
+
retrieved += 1
|
|
1266
|
+
candidate = admit(_domain(item.get("domain")), seed, query)
|
|
1267
|
+
if candidate is None:
|
|
1268
|
+
continue
|
|
1269
|
+
candidate.add_source("dataforseo_overlap")
|
|
1270
|
+
if seed not in candidate.overlap_seeds:
|
|
1271
|
+
candidate.overlap_seeds.append(seed)
|
|
1272
|
+
intersections = item.get("intersections")
|
|
1273
|
+
if isinstance(intersections, int) and not isinstance(intersections, bool):
|
|
1274
|
+
candidate.overlap_intersections = max(candidate.overlap_intersections, intersections)
|
|
1275
|
+
candidate.listed(query)
|
|
1276
|
+
found += 1
|
|
1277
|
+
summaries.append(_summary(2, query, "relationship", "dataforseo_competitors_domain", retrieved, found, []))
|
|
1278
|
+
|
|
1279
|
+
for seed in seeds if client.max_spyfu_calls > 0 else []:
|
|
1280
|
+
query = f"ppc:{seed}"
|
|
1281
|
+
try:
|
|
1282
|
+
response, _ = client.spyfu("ppc-competitors", {"domain": seed, "country_code": inputs.spyfu_market, "max_rows": SPYFU_COMPETITORS})
|
|
1283
|
+
except BudgetExhausted:
|
|
1284
|
+
raise
|
|
1285
|
+
except CompetitorSetError as exc:
|
|
1286
|
+
summaries.append(_summary(2, query, "relationship", "spyfu_ppc_competitors", 0, 0, [], str(exc)))
|
|
1287
|
+
continue
|
|
1288
|
+
if response.get("unavailable"):
|
|
1289
|
+
summaries.append(_summary(2, query, "relationship", "spyfu_ppc_competitors", 0, 0, [], str(response["unavailable"])))
|
|
1290
|
+
continue
|
|
1291
|
+
retrieved = found = 0
|
|
1292
|
+
for item in response.get("rows") or []:
|
|
1293
|
+
if not isinstance(item, dict):
|
|
1294
|
+
continue
|
|
1295
|
+
retrieved += 1
|
|
1296
|
+
candidate = admit(_domain(item.get("domain")), seed, query)
|
|
1297
|
+
if candidate is None:
|
|
1298
|
+
continue
|
|
1299
|
+
candidate.add_source("spyfu_ppc")
|
|
1300
|
+
if seed not in candidate.ppc_seeds:
|
|
1301
|
+
candidate.ppc_seeds.append(seed)
|
|
1302
|
+
common = item.get("common_paid_keywords")
|
|
1303
|
+
if isinstance(common, int) and not isinstance(common, bool):
|
|
1304
|
+
candidate.spyfu_common_keywords = max(candidate.spyfu_common_keywords, common)
|
|
1305
|
+
candidate.listed(query)
|
|
1306
|
+
found += 1
|
|
1307
|
+
summaries.append(_summary(2, query, "relationship", "spyfu_ppc_competitors", retrieved, found, []))
|
|
1308
|
+
|
|
1309
|
+
|
|
1310
|
+
# ── the agent's enrichment files ────────────────────────────────────────────
|
|
1311
|
+
|
|
1312
|
+
def _load_json(path: Path) -> Any:
|
|
1313
|
+
if not path.is_file():
|
|
1314
|
+
return None
|
|
1315
|
+
try:
|
|
1316
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
1317
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
1318
|
+
raise CompetitorSetError(f"{path.name} is not valid JSON: {exc}") from exc
|
|
1319
|
+
|
|
1320
|
+
|
|
1321
|
+
def load_listicles(path: Path) -> dict[str, dict[str, Any]]:
|
|
1322
|
+
raw = _load_json(path)
|
|
1323
|
+
if raw is None:
|
|
1324
|
+
return {}
|
|
1325
|
+
if not isinstance(raw, dict) or raw.get("schema_version") != 1 or not isinstance(raw.get("pages"), list):
|
|
1326
|
+
raise CompetitorSetError(f"{path.name} must be an object with schema_version 1 and a pages list")
|
|
1327
|
+
pages: dict[str, dict[str, Any]] = {}
|
|
1328
|
+
for index, page in enumerate(raw["pages"]):
|
|
1329
|
+
if not isinstance(page, dict):
|
|
1330
|
+
raise CompetitorSetError(f"{path.name} page #{index + 1} must be an object")
|
|
1331
|
+
url = _url(page.get("url"))
|
|
1332
|
+
if not url:
|
|
1333
|
+
raise CompetitorSetError(f"{path.name} page #{index + 1} needs the listicle address as url")
|
|
1334
|
+
if url in pages:
|
|
1335
|
+
raise CompetitorSetError(f"{path.name} records {url} twice")
|
|
1336
|
+
status = page.get("status")
|
|
1337
|
+
if status not in ("read", "unreadable"):
|
|
1338
|
+
raise CompetitorSetError(f"{path.name} page {url} needs status read or unreadable")
|
|
1339
|
+
reason = _text(page.get("reason"), 200)
|
|
1340
|
+
if status == "unreadable" and not reason:
|
|
1341
|
+
raise CompetitorSetError(f"{path.name} page {url} is unreadable without a reason")
|
|
1342
|
+
entries_raw = page.get("entries") or []
|
|
1343
|
+
if not isinstance(entries_raw, list) or len(entries_raw) > MAX_LISTICLE_ENTRIES:
|
|
1344
|
+
raise CompetitorSetError(f"{path.name} page {url} needs at most {MAX_LISTICLE_ENTRIES} entries in a list")
|
|
1345
|
+
entries = []
|
|
1346
|
+
for entry in entries_raw:
|
|
1347
|
+
if not isinstance(entry, dict) or not _text(entry.get("name")):
|
|
1348
|
+
raise CompetitorSetError(f"{path.name} page {url} has an entry without a name")
|
|
1349
|
+
domain_raw = entry.get("domain")
|
|
1350
|
+
domain = _domain(domain_raw) if domain_raw else ""
|
|
1351
|
+
if domain_raw and not domain:
|
|
1352
|
+
raise CompetitorSetError(f"{path.name} page {url} has an entry with an invalid domain {domain_raw!r}")
|
|
1353
|
+
entries.append({
|
|
1354
|
+
"name": _text(entry.get("name"), 80), "domain": domain or None,
|
|
1355
|
+
**{key: _text(entry.get(key), 160) or None for key in ("offer", "format", "price_text", "duration", "audience")},
|
|
1356
|
+
})
|
|
1357
|
+
pages[url] = {"url": url, "status": status, "reason": reason or None, "entries": entries}
|
|
1358
|
+
return pages
|
|
1359
|
+
|
|
1360
|
+
|
|
1361
|
+
def load_profiles(path: Path) -> dict[str, dict[str, Any]]:
|
|
1362
|
+
raw = _load_json(path)
|
|
1363
|
+
if raw is None:
|
|
1364
|
+
return {}
|
|
1365
|
+
if not isinstance(raw, dict) or raw.get("schema_version") != 1 or not isinstance(raw.get("profiles"), list):
|
|
1366
|
+
raise CompetitorSetError(f"{path.name} must be an object with schema_version 1 and a profiles list")
|
|
1367
|
+
profiles: dict[str, dict[str, Any]] = {}
|
|
1368
|
+
for index, item in enumerate(raw["profiles"]):
|
|
1369
|
+
if not isinstance(item, dict):
|
|
1370
|
+
raise CompetitorSetError(f"{path.name} profile #{index + 1} must be an object")
|
|
1371
|
+
domain = _domain(item.get("domain"))
|
|
1372
|
+
if not domain:
|
|
1373
|
+
raise CompetitorSetError(f"{path.name} profile #{index + 1} has no valid domain")
|
|
1374
|
+
if domain in profiles:
|
|
1375
|
+
raise CompetitorSetError(f"{path.name} profiles `{domain}` twice")
|
|
1376
|
+
for flag in ("operating", "same_buyer", "same_job"):
|
|
1377
|
+
if not isinstance(item.get(flag), bool):
|
|
1378
|
+
raise CompetitorSetError(f"{path.name} profile for `{domain}` needs a boolean {flag}")
|
|
1379
|
+
if item.get("offer_match") not in OFFER_MATCHES:
|
|
1380
|
+
raise CompetitorSetError(f"{path.name} profile for `{domain}` needs offer_match in {', '.join(OFFER_MATCHES)}")
|
|
1381
|
+
price = item.get("price_usd")
|
|
1382
|
+
try:
|
|
1383
|
+
price_usd = None if price is None else _number(price)
|
|
1384
|
+
except ValueError:
|
|
1385
|
+
raise CompetitorSetError(f"{path.name} profile for `{domain}` needs price_usd as a non-negative number or null") from None
|
|
1386
|
+
for flag in ("marketplace", "focused"):
|
|
1387
|
+
if item.get(flag) is not None and not isinstance(item.get(flag), bool):
|
|
1388
|
+
raise CompetitorSetError(f"{path.name} profile for `{domain}` has a non-boolean {flag}")
|
|
1389
|
+
urls = item.get("evidence_urls")
|
|
1390
|
+
evidence = [u for u in (_url(x) for x in urls) if u] if isinstance(urls, list) else []
|
|
1391
|
+
if not evidence:
|
|
1392
|
+
raise CompetitorSetError(f"{path.name} profile for `{domain}` needs at least one evidence URL (the pages read)")
|
|
1393
|
+
found_raw = item.get("found_in")
|
|
1394
|
+
found_in = _url(found_raw) if found_raw else ""
|
|
1395
|
+
if found_raw and not found_in:
|
|
1396
|
+
raise CompetitorSetError(f"{path.name} profile for `{domain}` has found_in that is not an address")
|
|
1397
|
+
profiles[domain] = {
|
|
1398
|
+
"domain": domain,
|
|
1399
|
+
"name": _text(item.get("name"), 80) or None,
|
|
1400
|
+
"operating": item["operating"],
|
|
1401
|
+
"offer": _text(item.get("offer"), 200) or None,
|
|
1402
|
+
"offer_url": _url(item.get("offer_url")) or None,
|
|
1403
|
+
"offer_class": _text(item.get("offer_class"), 120) or None,
|
|
1404
|
+
"offer_match": item["offer_match"],
|
|
1405
|
+
"price_text": _text(item.get("price_text"), 160) or None,
|
|
1406
|
+
"price_usd": price_usd,
|
|
1407
|
+
"duration": _text(item.get("duration"), 120) or None,
|
|
1408
|
+
"buyer": _text(item.get("buyer"), 200) or None,
|
|
1409
|
+
"same_buyer": item["same_buyer"],
|
|
1410
|
+
"job": _text(item.get("job"), 200) or None,
|
|
1411
|
+
"same_job": item["same_job"],
|
|
1412
|
+
"marketplace": bool(item.get("marketplace")),
|
|
1413
|
+
# False when the offer is a side line of a big multi-product domain (a
|
|
1414
|
+
# university, a large platform): its neighbours are not the offer's.
|
|
1415
|
+
"focused": item.get("focused") is not False,
|
|
1416
|
+
"main_business": _text(item.get("main_business"), 200) or None,
|
|
1417
|
+
"markets": _text(item.get("markets"), 120) or None,
|
|
1418
|
+
"evidence_urls": evidence[:6],
|
|
1419
|
+
"found_in": found_in or None,
|
|
1420
|
+
}
|
|
1421
|
+
return profiles
|
|
1422
|
+
|
|
1423
|
+
|
|
1424
|
+
def load_decisions(path: Path) -> dict[str, dict[str, Any]]:
|
|
1425
|
+
"""Validate the agent's tiering file. Every decision names a domain the
|
|
1426
|
+
pool knows (discovered, named, listicle-backed or profiled with the page
|
|
1427
|
+
where the agent found it)."""
|
|
1428
|
+
raw = _load_json(path)
|
|
1429
|
+
if raw is None:
|
|
1430
|
+
return {}
|
|
1431
|
+
if isinstance(raw, dict) and raw.get("schema_version") == 1:
|
|
1432
|
+
raise CompetitorSetError(
|
|
1433
|
+
f"{path.name} has schema_version 1, which predates the offer-led rules: profile the candidates and "
|
|
1434
|
+
f"decide them again with schema_version {DECISIONS_SCHEMA_VERSION} (see the workflow doc)"
|
|
1435
|
+
)
|
|
1436
|
+
if not isinstance(raw, dict) or raw.get("schema_version") != DECISIONS_SCHEMA_VERSION or not isinstance(raw.get("decisions"), list):
|
|
1437
|
+
raise CompetitorSetError(f"{path.name} must be an object with schema_version {DECISIONS_SCHEMA_VERSION} and a decisions list")
|
|
1438
|
+
decisions: dict[str, dict[str, Any]] = {}
|
|
1439
|
+
for index, item in enumerate(raw["decisions"]):
|
|
1440
|
+
if not isinstance(item, dict):
|
|
1441
|
+
raise CompetitorSetError(f"{path.name} decision #{index + 1} must be an object")
|
|
1442
|
+
domain = _domain(item.get("domain"))
|
|
1443
|
+
tier = item.get("competitor_tier")
|
|
1444
|
+
reason = _text(item.get("reason"), 240)
|
|
1445
|
+
if not domain:
|
|
1446
|
+
raise CompetitorSetError(f"{path.name} decision #{index + 1} has no valid domain")
|
|
1447
|
+
if domain in decisions:
|
|
1448
|
+
raise CompetitorSetError(f"{path.name} decides `{domain}` twice")
|
|
1449
|
+
if tier not in DECISION_TIERS:
|
|
1450
|
+
raise CompetitorSetError(f"{path.name} decision for `{domain}` needs competitor_tier in {', '.join(DECISION_TIERS)}")
|
|
1451
|
+
if not reason:
|
|
1452
|
+
raise CompetitorSetError(f"{path.name} decision for `{domain}` needs a reason")
|
|
1453
|
+
name = item.get("name")
|
|
1454
|
+
same_raw = item.get("same_as")
|
|
1455
|
+
same_as = _domain(same_raw) if same_raw else ""
|
|
1456
|
+
if same_raw and (not same_as or same_as == domain):
|
|
1457
|
+
raise CompetitorSetError(f"{path.name} decision for `{domain}` has same_as that is not another domain")
|
|
1458
|
+
if same_as and tier != "excluded":
|
|
1459
|
+
raise CompetitorSetError(f"{path.name} decision for `{domain}` folds it into `{same_as}`: its tier must be excluded")
|
|
1460
|
+
decisions[domain] = {
|
|
1461
|
+
"competitor_tier": tier,
|
|
1462
|
+
"reason": reason,
|
|
1463
|
+
"name": _text(name, 80) if isinstance(name, str) and name.strip() else None,
|
|
1464
|
+
"override_reason": _text(item.get("override_reason"), 240) or None,
|
|
1465
|
+
"same_as": same_as or None,
|
|
1466
|
+
}
|
|
1467
|
+
for domain, decision in decisions.items():
|
|
1468
|
+
target = decisions.get(decision["same_as"] or "")
|
|
1469
|
+
if target is not None and target["same_as"]:
|
|
1470
|
+
raise CompetitorSetError(f"{path.name} folds `{domain}` into `{decision['same_as']}`, which is folded itself; name the final domain")
|
|
1471
|
+
return decisions
|
|
1472
|
+
|
|
1473
|
+
|
|
1474
|
+
def load_review(path: Path) -> dict[str, Any]:
|
|
1475
|
+
raw = _load_json(path)
|
|
1476
|
+
empty = {"rounds": [], "checked": set(), "objections": [], "missing": [], "checked_by_round": {1: set(), 2: set()}}
|
|
1477
|
+
if raw is None:
|
|
1478
|
+
return empty
|
|
1479
|
+
if not isinstance(raw, dict) or raw.get("schema_version") != 1 or not isinstance(raw.get("rounds"), list):
|
|
1480
|
+
raise CompetitorSetError(f"{path.name} must be an object with schema_version 1 and a rounds list")
|
|
1481
|
+
review = empty
|
|
1482
|
+
for index, item in enumerate(raw["rounds"]):
|
|
1483
|
+
if not isinstance(item, dict):
|
|
1484
|
+
raise CompetitorSetError(f"{path.name} round #{index + 1} must be an object")
|
|
1485
|
+
reviewer = _text(item.get("reviewer"), 120)
|
|
1486
|
+
if not reviewer:
|
|
1487
|
+
raise CompetitorSetError(f"{path.name} round #{index + 1} needs the reviewer (a separate agent or pass)")
|
|
1488
|
+
checked = item.get("checked")
|
|
1489
|
+
if not isinstance(checked, list) or not checked:
|
|
1490
|
+
raise CompetitorSetError(f"{path.name} round #{index + 1} needs the checked domains")
|
|
1491
|
+
domains = parse_domains(checked, 1000)
|
|
1492
|
+
review["checked"].update(domains)
|
|
1493
|
+
# The discovery round whose pool this review round looked through (1 when omitted).
|
|
1494
|
+
discovery_round = item.get("discovery_round", 1)
|
|
1495
|
+
if type(discovery_round) is not int or discovery_round not in (1, 2):
|
|
1496
|
+
raise CompetitorSetError(f"{path.name} round #{index + 1} has discovery_round {discovery_round!r}; use 1 or 2")
|
|
1497
|
+
review["checked_by_round"][discovery_round].update(domains)
|
|
1498
|
+
objections = item.get("objections") or []
|
|
1499
|
+
missing = item.get("missing") or []
|
|
1500
|
+
if not isinstance(objections, list) or not isinstance(missing, list):
|
|
1501
|
+
raise CompetitorSetError(f"{path.name} round #{index + 1} needs objections and missing as lists")
|
|
1502
|
+
for objection in objections:
|
|
1503
|
+
domain = _domain(objection.get("domain")) if isinstance(objection, dict) else ""
|
|
1504
|
+
if not domain or not _text(objection.get("objection")):
|
|
1505
|
+
raise CompetitorSetError(f"{path.name} round #{index + 1} has an objection without a domain or text")
|
|
1506
|
+
resolution = objection.get("resolution")
|
|
1507
|
+
if resolution not in (None, "", *RESOLUTIONS):
|
|
1508
|
+
raise CompetitorSetError(f"{path.name} objection on `{domain}` needs resolution accepted or rejected")
|
|
1509
|
+
note = _text(objection.get("note"), 240)
|
|
1510
|
+
review["objections"].append({"round": index + 1, "domain": domain, "objection": _text(objection.get("objection"), 240),
|
|
1511
|
+
"resolution": resolution or None, "note": note or None})
|
|
1512
|
+
for entry in missing:
|
|
1513
|
+
domain = _domain(entry.get("domain")) if isinstance(entry, dict) else ""
|
|
1514
|
+
if not domain:
|
|
1515
|
+
raise CompetitorSetError(f"{path.name} round #{index + 1} lists a missing rival without a valid domain")
|
|
1516
|
+
resolution = entry.get("resolution")
|
|
1517
|
+
if resolution not in (None, "", *MISSING_RESOLUTIONS):
|
|
1518
|
+
raise CompetitorSetError(f"{path.name} missing rival `{domain}` needs resolution added or rejected")
|
|
1519
|
+
review["missing"].append({"round": index + 1, "domain": domain, "why": _text(entry.get("why"), 240),
|
|
1520
|
+
"resolution": resolution or None, "note": _text(entry.get("note"), 240) or None})
|
|
1521
|
+
review["rounds"].append({"round": index + 1, "reviewer": reviewer, "reviewed_on": _text(item.get("reviewed_on"), 10) or None,
|
|
1522
|
+
"discovery_round": discovery_round, "checked": len(domains), "objections": len(objections),
|
|
1523
|
+
"missing": len(missing)})
|
|
1524
|
+
return review
|
|
1525
|
+
|
|
1526
|
+
|
|
1527
|
+
@dataclass
|
|
1528
|
+
class AgentFiles:
|
|
1529
|
+
listicles: dict[str, dict[str, Any]]
|
|
1530
|
+
profiles: dict[str, dict[str, Any]]
|
|
1531
|
+
decisions: dict[str, dict[str, Any]]
|
|
1532
|
+
review: dict[str, Any]
|
|
1533
|
+
paths: dict[str, Path]
|
|
1534
|
+
|
|
1535
|
+
|
|
1536
|
+
def load_agent_files(runtime: Path, decisions_path: Path) -> AgentFiles:
|
|
1537
|
+
paths = {
|
|
1538
|
+
"listicles": runtime / LISTICLES_FILENAME,
|
|
1539
|
+
"profiles": runtime / PROFILES_FILENAME,
|
|
1540
|
+
"decisions": decisions_path,
|
|
1541
|
+
"review": runtime / REVIEW_FILENAME,
|
|
1542
|
+
}
|
|
1543
|
+
return AgentFiles(
|
|
1544
|
+
listicles=load_listicles(paths["listicles"]),
|
|
1545
|
+
profiles=load_profiles(paths["profiles"]),
|
|
1546
|
+
decisions=load_decisions(paths["decisions"]),
|
|
1547
|
+
review=load_review(paths["review"]),
|
|
1548
|
+
paths=paths,
|
|
1549
|
+
)
|
|
1550
|
+
|
|
1551
|
+
|
|
1552
|
+
def add_agent_research(pool: dict[str, Candidate], files: AgentFiles, to_read: list[str], product_domain: str) -> list[str]:
|
|
1553
|
+
"""Listicle entries and profiles the agent recorded become candidates
|
|
1554
|
+
with provenance `agent_research` and the page they came from; a profile
|
|
1555
|
+
of a domain no source recorded must say where it was found."""
|
|
1556
|
+
recorded = set(to_read)
|
|
1557
|
+
stray = sorted(url for url in files.listicles if url not in recorded)
|
|
1558
|
+
if stray:
|
|
1559
|
+
raise CompetitorSetError(
|
|
1560
|
+
"listicles.json reads pages the runner did not record: " + ", ".join(stray[:5])
|
|
1561
|
+
+ " — keep the reading to the addresses in candidates.json `listicles_to_read`"
|
|
1562
|
+
)
|
|
1563
|
+
for url in to_read:
|
|
1564
|
+
page = files.listicles.get(url)
|
|
1565
|
+
if not page or page["status"] != "read":
|
|
1566
|
+
continue
|
|
1567
|
+
for entry in page["entries"]:
|
|
1568
|
+
domain = entry["domain"]
|
|
1569
|
+
if not domain or domain == product_domain or classify_domain(domain) in ("noise", "directory"):
|
|
1570
|
+
continue
|
|
1571
|
+
if domain not in pool:
|
|
1572
|
+
# Only a domain no source recorded gets the agent's provenance;
|
|
1573
|
+
# for a known one the listicle page is one more source.
|
|
1574
|
+
candidate = _candidate(pool, domain)
|
|
1575
|
+
candidate.add_provenance("agent_research")
|
|
1576
|
+
candidate.found_in = url
|
|
1577
|
+
candidate = pool[domain]
|
|
1578
|
+
candidate.add_source("listicle")
|
|
1579
|
+
if url not in candidate.listicle_pages:
|
|
1580
|
+
candidate.listicle_pages.append(url)
|
|
1581
|
+
additions: list[str] = []
|
|
1582
|
+
for domain, profile in sorted(files.profiles.items()):
|
|
1583
|
+
if domain == product_domain:
|
|
1584
|
+
raise CompetitorSetError(f"profiles.json profiles the product's own domain `{domain}`")
|
|
1585
|
+
if domain in pool or not profile["found_in"]:
|
|
1586
|
+
continue
|
|
1587
|
+
candidate = _candidate(pool, domain)
|
|
1588
|
+
candidate.add_provenance("agent_research")
|
|
1589
|
+
candidate.add_source("agent_research")
|
|
1590
|
+
candidate.found_in = profile["found_in"]
|
|
1591
|
+
additions.append(domain)
|
|
1592
|
+
return additions
|
|
1593
|
+
|
|
1594
|
+
|
|
1595
|
+
def own_additions(pool: dict[str, Candidate], additions: list[str]) -> list[str]:
|
|
1596
|
+
"""The agent's own finds, after round 2: a profiled domain that round 2
|
|
1597
|
+
also discovered is a discovery, whatever page the profile names."""
|
|
1598
|
+
own = [domain for domain in additions if "discovered" not in pool[domain].provenance]
|
|
1599
|
+
if len(own) > MAX_AGENT_ADDITIONS:
|
|
1600
|
+
raise CompetitorSetError(f"profiles.json adds {len(own)} domains the discovery never saw; at most {MAX_AGENT_ADDITIONS} per run")
|
|
1601
|
+
return own
|
|
1602
|
+
|
|
1603
|
+
|
|
1604
|
+
# ── profiling list, tier rules and completeness ─────────────────────────────
|
|
1605
|
+
|
|
1606
|
+
def _capped(rows: list[Candidate], cap: int, decisions: dict[str, dict[str, Any]]) -> list[str]:
|
|
1607
|
+
"""The first `cap` domains, not counting the ones already decided
|
|
1608
|
+
`excluded`: an obvious non-competitor costs no profile, so it frees its
|
|
1609
|
+
place for the next candidate."""
|
|
1610
|
+
chosen: list[str] = []
|
|
1611
|
+
counted = 0
|
|
1612
|
+
for candidate in rows:
|
|
1613
|
+
excluded = (decisions.get(candidate.domain) or {}).get("competitor_tier") == "excluded"
|
|
1614
|
+
if not excluded:
|
|
1615
|
+
if counted >= cap:
|
|
1616
|
+
continue
|
|
1617
|
+
counted += 1
|
|
1618
|
+
chosen.append(candidate.domain)
|
|
1619
|
+
return chosen
|
|
1620
|
+
|
|
1621
|
+
|
|
1622
|
+
def required_round1(pool: dict[str, Candidate], inputs: RunInputs, decisions: dict[str, dict[str, Any]] | None = None) -> list[str]:
|
|
1623
|
+
"""Domains the agent must profile and decide in round 1: every named
|
|
1624
|
+
competitor (founder or product page), then every domain worth
|
|
1625
|
+
profiling (see `offer_signal`), strongest first, up to the cap."""
|
|
1626
|
+
decisions = decisions or {}
|
|
1627
|
+
aliases = {d for d, dec in decisions.items() if dec.get("same_as")}
|
|
1628
|
+
named = [d for d in inputs.named_domains if d in pool]
|
|
1629
|
+
page = [d for d in inputs.page_domains if d in pool and d not in named]
|
|
1630
|
+
fixed = set(named) | set(page)
|
|
1631
|
+
discovered = [c for c in pool.values() if c.domain not in fixed and c.domain not in aliases and c.offer_signal()]
|
|
1632
|
+
# Advertisers on the offer's own search first: the list exists for the ones that buy ads.
|
|
1633
|
+
discovered.sort(key=lambda c: (not (c.serp_paid_hits or c.keyword_ads_phrases), -c.round1_score(), -c.signal_count(), -len(c.sources), c.domain))
|
|
1634
|
+
return named + page + _capped(discovered, MAX_PROFILES_ROUND1, decisions)
|
|
1635
|
+
|
|
1636
|
+
|
|
1637
|
+
def required_round2(pool: dict[str, Candidate], required1: list[str], decisions: dict[str, dict[str, Any]] | None = None) -> list[str]:
|
|
1638
|
+
"""A bounded stable-domain sample of relationship results; single hits remain eligible."""
|
|
1639
|
+
decisions = decisions or {}
|
|
1640
|
+
first = set(required1)
|
|
1641
|
+
rows = [
|
|
1642
|
+
c for c in pool.values()
|
|
1643
|
+
if c.domain not in first and not (decisions.get(c.domain) or {}).get("same_as")
|
|
1644
|
+
and c.relationship_hits()
|
|
1645
|
+
]
|
|
1646
|
+
rows.sort(key=lambda c: c.domain)
|
|
1647
|
+
return _capped(rows, MAX_PROFILES_ROUND2, decisions)
|
|
1648
|
+
|
|
1649
|
+
|
|
1650
|
+
def apply_aliases(pool: dict[str, Candidate], decisions: dict[str, dict[str, Any]]) -> None:
|
|
1651
|
+
"""A decision with `same_as` folds a second domain of the same offer (a
|
|
1652
|
+
redirect, an academy domain, a university brand on a partner's program)
|
|
1653
|
+
into the canonical one: its discovery facts count there, and it is
|
|
1654
|
+
recorded as excluded."""
|
|
1655
|
+
for domain, decision in sorted(decisions.items()):
|
|
1656
|
+
canonical = decision.get("same_as")
|
|
1657
|
+
if not canonical or domain not in pool:
|
|
1658
|
+
continue
|
|
1659
|
+
if canonical not in pool:
|
|
1660
|
+
raise CompetitorSetError(
|
|
1661
|
+
f"decisions.json folds `{domain}` into `{canonical}`, which no source recorded: profile `{canonical}` with found_in first"
|
|
1662
|
+
)
|
|
1663
|
+
pool[canonical].absorb(pool[domain])
|
|
1664
|
+
|
|
1665
|
+
|
|
1666
|
+
def price_band(price_usd: float | None, floor: float | None) -> str:
|
|
1667
|
+
if floor is None:
|
|
1668
|
+
return "no_floor"
|
|
1669
|
+
if price_usd is None:
|
|
1670
|
+
return "unknown"
|
|
1671
|
+
return "at_or_above_floor" if price_usd >= floor else "below_floor"
|
|
1672
|
+
|
|
1673
|
+
|
|
1674
|
+
@dataclass
|
|
1675
|
+
class Assessment:
|
|
1676
|
+
domain: str
|
|
1677
|
+
tier: str
|
|
1678
|
+
named: bool
|
|
1679
|
+
profile: dict[str, Any] | None
|
|
1680
|
+
decision: dict[str, Any]
|
|
1681
|
+
price_band: str
|
|
1682
|
+
meets_direct: bool
|
|
1683
|
+
comparable: bool
|
|
1684
|
+
unmet: list[str]
|
|
1685
|
+
group: int | None
|
|
1686
|
+
reason_code: str
|
|
1687
|
+
|
|
1688
|
+
|
|
1689
|
+
def assess(domain: str, decision: dict[str, Any], profile: dict[str, Any] | None, named: bool, floor: float | None) -> Assessment:
|
|
1690
|
+
"""Record caller-supplied comparison roles; prices and ad activity are facts,
|
|
1691
|
+
not automatic eligibility or business-priority rules."""
|
|
1692
|
+
tier = decision["competitor_tier"]
|
|
1693
|
+
band = price_band(profile["price_usd"] if profile else None, floor)
|
|
1694
|
+
group = None if tier == "excluded" else {"direct": 0, "indirect": 1}.get(tier, 2)
|
|
1695
|
+
code = ("same_as" if decision.get("same_as") else "excluded") if group is None else tier
|
|
1696
|
+
return Assessment(domain=domain, tier=tier, named=named, profile=profile, decision=decision, price_band=band,
|
|
1697
|
+
meets_direct=tier == "direct", comparable=group is not None, unmet=[], group=group, reason_code=code)
|
|
1698
|
+
|
|
1699
|
+
|
|
1700
|
+
def check_consistency(assessments: dict[str, Assessment]) -> None:
|
|
1701
|
+
"""Decisions require reasons and sourced profiles in load_agent_files.
|
|
1702
|
+
No built-in price/buyer/offer formula overrides the caller's scope."""
|
|
1703
|
+
return None
|
|
1704
|
+
|
|
1705
|
+
|
|
1706
|
+
def missing_items(*, domains: set[str], required: list[str], inputs: RunInputs, files: AgentFiles, to_read: list[str],
|
|
1707
|
+
assessments: dict[str, Assessment]) -> dict[str, list[str]]:
|
|
1708
|
+
"""Identify missing source records for this chosen collection operation."""
|
|
1709
|
+
listicles = [url for url in to_read if url not in files.listicles]
|
|
1710
|
+
profiles: list[str] = []
|
|
1711
|
+
decisions: list[str] = []
|
|
1712
|
+
for domain in required:
|
|
1713
|
+
decision = files.decisions.get(domain)
|
|
1714
|
+
if decision is None:
|
|
1715
|
+
decisions.append(domain)
|
|
1716
|
+
needs_profile = domain in inputs.named_domains or (decision is not None and decision["competitor_tier"] != "excluded") or decision is None
|
|
1717
|
+
if needs_profile and domain not in files.profiles:
|
|
1718
|
+
profiles.append(domain)
|
|
1719
|
+
for domain in sorted(d for d in files.profiles if d in domains):
|
|
1720
|
+
if domain not in files.decisions and domain not in decisions:
|
|
1721
|
+
decisions.append(domain)
|
|
1722
|
+
for domain in sorted(d for d, dec in files.decisions.items() if d in domains and dec["competitor_tier"] != "excluded"):
|
|
1723
|
+
if domain not in files.profiles and domain not in profiles:
|
|
1724
|
+
profiles.append(domain)
|
|
1725
|
+
missing = {"listicles": listicles, "profiles": profiles, "decisions": decisions}
|
|
1726
|
+
return {key: value for key, value in missing.items() if value}
|
|
1727
|
+
|
|
1728
|
+
|
|
1729
|
+
def build_assessments(pool: dict[str, Candidate], files: AgentFiles, inputs: RunInputs) -> dict[str, Assessment]:
|
|
1730
|
+
return {
|
|
1731
|
+
domain: assess(domain, decision, files.profiles.get(domain), domain in inputs.named_domains, inputs.price_floor_usd)
|
|
1732
|
+
for domain, decision in files.decisions.items() if domain in pool
|
|
1733
|
+
}
|
|
1734
|
+
|
|
1735
|
+
|
|
1736
|
+
def round2_seeds(pool: dict[str, Candidate], assessments: dict[str, Assessment], inputs: RunInputs) -> list[str]:
|
|
1737
|
+
"""Up to three confirmed direct competitors (those named by the founder first, then
|
|
1738
|
+
by ads on the offer's phrases), topped up with comparable indirect ones.
|
|
1739
|
+
A domain whose profile says the offer is a side line of a big
|
|
1740
|
+
multi-product site (`focused: false`) is never a seed: its keyword and
|
|
1741
|
+
PPC neighbours are the rest of that site's business, not this offer's.
|
|
1742
|
+
Nor is a seller's subdomain on a course host: the tools measure the host."""
|
|
1743
|
+
def key(item: Assessment) -> tuple:
|
|
1744
|
+
candidate = pool[item.domain]
|
|
1745
|
+
ads = len(candidate.keyword_ads_phrases) + candidate.serp_paid_hits
|
|
1746
|
+
return (0 if item.named else 1, -ads, -candidate.round1_score(), item.domain)
|
|
1747
|
+
|
|
1748
|
+
def usable(item: Assessment) -> bool:
|
|
1749
|
+
return bool(item.profile and item.profile.get("focused", True)) and not hosted_seller(item.domain)
|
|
1750
|
+
|
|
1751
|
+
direct = sorted((a for a in assessments.values() if a.group == 0 and usable(a)), key=key)
|
|
1752
|
+
seeds = [a.domain for a in direct[:ROUND2_SEEDS]]
|
|
1753
|
+
if len(seeds) < ROUND2_SEEDS:
|
|
1754
|
+
comparable = sorted((a for a in assessments.values() if a.group == 1 and usable(a)), key=key)
|
|
1755
|
+
seeds += [a.domain for a in comparable[: ROUND2_SEEDS - len(seeds)]]
|
|
1756
|
+
return seeds
|
|
1757
|
+
|
|
1758
|
+
|
|
1759
|
+
# ── ad check ────────────────────────────────────────────────────────────────
|
|
1760
|
+
|
|
1761
|
+
def scan_home(client: OsClient, domain: str) -> dict[str, Any]:
|
|
1762
|
+
"""Home page plus up to two Google Tag Manager containers; the ad tags
|
|
1763
|
+
of the Channel Footprint signature list."""
|
|
1764
|
+
cf = _channel_footprint()
|
|
1765
|
+
page, _ = client.fetch_page(f"https://{domain}/")
|
|
1766
|
+
status = page.get("status")
|
|
1767
|
+
body = str(page.get("body") or "")
|
|
1768
|
+
if not isinstance(status, int):
|
|
1769
|
+
return {"fetch_status": "error", "tags": []}
|
|
1770
|
+
if status >= 400:
|
|
1771
|
+
return {"fetch_status": "blocked" if status in (401, 403, 429, 503) else "error", "tags": []}
|
|
1772
|
+
if not body.strip():
|
|
1773
|
+
return {"fetch_status": "empty", "tags": []}
|
|
1774
|
+
blobs = [body]
|
|
1775
|
+
for container in list(dict.fromkeys(cf.GTM_ID_RE.findall(body)))[:GTM_CONTAINERS_PER_SITE]:
|
|
1776
|
+
gtm, _ = client.fetch_page(f"https://www.googletagmanager.com/gtm.js?id={container}")
|
|
1777
|
+
blobs.append(str(gtm.get("body") or ""))
|
|
1778
|
+
tags = sorted({t["tag"] for t in cf.detect_tags(blobs) if t["channel"] in AD_TAG_CHANNELS})
|
|
1779
|
+
return {"fetch_status": "ok", "tags": tags}
|
|
1780
|
+
|
|
1781
|
+
|
|
1782
|
+
def _meta_matches(rows: list[Any], domain: str) -> int:
|
|
1783
|
+
"""Ads whose destination is the domain, or whose page name is the brand
|
|
1784
|
+
when the ad carries no usable destination (the Channel Footprint rule)."""
|
|
1785
|
+
cf = _channel_footprint()
|
|
1786
|
+
brand = brand_token(domain)
|
|
1787
|
+
matched = 0
|
|
1788
|
+
for row in rows:
|
|
1789
|
+
if not isinstance(row, dict):
|
|
1790
|
+
continue
|
|
1791
|
+
links = row.get("ad_creative_link_captions") or []
|
|
1792
|
+
if isinstance(links, str):
|
|
1793
|
+
links = [links]
|
|
1794
|
+
destinations = {cf.usable_destination(x) for x in links if x} - {""} if isinstance(links, list) else set()
|
|
1795
|
+
if destinations:
|
|
1796
|
+
matched += int(domain in destinations)
|
|
1797
|
+
elif cf._name_matches(str(row.get("page_name") or ""), brand):
|
|
1798
|
+
matched += 1
|
|
1799
|
+
return matched
|
|
1800
|
+
|
|
1801
|
+
|
|
1802
|
+
def check_ads(client: OsClient, inputs: RunInputs, candidate: Candidate, *, focused: bool = True) -> dict[str, Any]:
|
|
1803
|
+
"""Does the competitor buy media? The round-1 ad facts on the offer's
|
|
1804
|
+
phrases, SpyFu's modeled Google spend (the primary market, then up to two
|
|
1805
|
+
more target markets), ad pixels on the home page and, when nothing on
|
|
1806
|
+
Google shows, the official Meta Ad Library (EU delivery).
|
|
1807
|
+
|
|
1808
|
+
For an offer that is a side line of a multi-product domain (`focused:
|
|
1809
|
+
false`) only the ads on the offer's own phrases count: the domain's
|
|
1810
|
+
overall spend, pixels and Meta ads promote the rest of its business."""
|
|
1811
|
+
cf = _channel_footprint()
|
|
1812
|
+
kinds: list[str] = []
|
|
1813
|
+
facts: list[str] = []
|
|
1814
|
+
lookups: list[str] = []
|
|
1815
|
+
budget: float | None = None
|
|
1816
|
+
answered = False
|
|
1817
|
+
if candidate.keyword_ads_phrases:
|
|
1818
|
+
kinds.append("category_keyword_ads")
|
|
1819
|
+
seen = f", last seen {candidate.keyword_ads_last_seen}" if candidate.keyword_ads_last_seen else ""
|
|
1820
|
+
facts.append("Google ads on " + ", ".join(f"'{p}'" for p in candidate.keyword_ads_phrases) + f" (SpyFu{seen})")
|
|
1821
|
+
if candidate.serp_paid_hits:
|
|
1822
|
+
kinds.append("category_paid_result")
|
|
1823
|
+
facts.append("paid result for " + ", ".join(f"'{q}'" for q in candidate.paid_queries[:3]))
|
|
1824
|
+
if not focused:
|
|
1825
|
+
lookups.append("domain-wide checks skipped: the offer is a side line of a multi-product domain")
|
|
1826
|
+
return {"ad_activity": "ads_seen" if kinds else "unknown", "ad_evidence": facts[:6], "ad_kinds": kinds,
|
|
1827
|
+
"ad_strength": len(kinds), "google_budget_usd": None, "pixel_tags": [], "lookup_gaps": lookups}
|
|
1828
|
+
markets = [m for m in inputs.markets if m in SPYFU_MARKETS][: 1 + AD_CHECK_EXTRA_MARKETS]
|
|
1829
|
+
if hosted_seller(candidate.domain):
|
|
1830
|
+
# SpyFu models the host's domain; that spend is not this seller's.
|
|
1831
|
+
markets = []
|
|
1832
|
+
lookups.append("spyfu: a seller's subdomain on a course host, not measured")
|
|
1833
|
+
for market in markets if client.max_spyfu_calls > 0 else []:
|
|
1834
|
+
try:
|
|
1835
|
+
data, retrieved_on = client.spyfu("domain-stats", {"domain": candidate.domain, "country_code": market, "months": AD_CHECK_MONTHS})
|
|
1836
|
+
except BudgetExhausted:
|
|
1837
|
+
raise
|
|
1838
|
+
except CompetitorSetError as exc:
|
|
1839
|
+
lookups.append(f"spyfu {market}: {str(exc)[:80]}")
|
|
1840
|
+
continue
|
|
1841
|
+
if data.get("unavailable"):
|
|
1842
|
+
lookups.append(f"spyfu {market}: rejected")
|
|
1843
|
+
continue
|
|
1844
|
+
row = cf._paid_search_row(candidate.domain, market, data, retrieved_on)
|
|
1845
|
+
if row["found"]:
|
|
1846
|
+
answered = True
|
|
1847
|
+
if row["months_with_spend"]:
|
|
1848
|
+
kinds.append("google_spend")
|
|
1849
|
+
budget = row["avg_monthly_budget_usd"]
|
|
1850
|
+
amount = f", about {_money(float(budget))}/month" if budget else ""
|
|
1851
|
+
facts.append(f"Google Ads spend in {row['months_with_spend']} of {row['months_checked']} month(s) in {market} (SpyFu{amount})")
|
|
1852
|
+
break
|
|
1853
|
+
scan = scan_home(client, candidate.domain)
|
|
1854
|
+
if scan["fetch_status"] == "ok":
|
|
1855
|
+
answered = True
|
|
1856
|
+
else:
|
|
1857
|
+
lookups.append(f"home page: {scan['fetch_status']}")
|
|
1858
|
+
if not kinds and client.max_meta_searches > 0:
|
|
1859
|
+
try:
|
|
1860
|
+
response, _ = client.meta_library({"query": brand_token(candidate.domain), "countries": [inputs.eu_country],
|
|
1861
|
+
"activeStatus": "ACTIVE", "limit": META_ADS_LIMIT})
|
|
1862
|
+
except BudgetExhausted:
|
|
1863
|
+
raise
|
|
1864
|
+
except CompetitorSetError as exc:
|
|
1865
|
+
lookups.append(f"meta: {str(exc)[:80]}")
|
|
1866
|
+
else:
|
|
1867
|
+
if response.get("unavailable"):
|
|
1868
|
+
lookups.append("meta: rejected")
|
|
1869
|
+
else:
|
|
1870
|
+
answered = True
|
|
1871
|
+
matched = _meta_matches(response.get("data") or [], candidate.domain)
|
|
1872
|
+
if matched:
|
|
1873
|
+
kinds.append("meta_eu")
|
|
1874
|
+
facts.append(f"{matched} active Meta ad(s) delivered in {inputs.eu_country} (EU library)")
|
|
1875
|
+
if scan["tags"]:
|
|
1876
|
+
facts.append("ad pixels on the home page: " + ", ".join(scan["tags"]))
|
|
1877
|
+
verdict = "ads_seen" if kinds else "pixels_only" if scan["tags"] else "none_detected" if answered else "unknown"
|
|
1878
|
+
return {"ad_activity": verdict, "ad_evidence": facts[:6], "ad_kinds": kinds, "ad_strength": len(kinds),
|
|
1879
|
+
"google_budget_usd": budget, "pixel_tags": scan["tags"], "lookup_gaps": lookups[:4]}
|
|
1880
|
+
|
|
1881
|
+
|
|
1882
|
+
# ── composition ─────────────────────────────────────────────────────────────
|
|
1883
|
+
|
|
1884
|
+
def _order_key(item: Assessment, ads: dict[str, dict[str, Any]], pool: dict[str, Candidate]) -> tuple:
|
|
1885
|
+
# Stable inspection order, never a recommendation ranking.
|
|
1886
|
+
return (item.domain,)
|
|
1887
|
+
|
|
1888
|
+
|
|
1889
|
+
def compose(eligible: list[Assessment], ads: dict[str, dict[str, Any]], pool: dict[str, Candidate], cap: int) -> tuple[list[Assessment], list[Assessment]]:
|
|
1890
|
+
"""Preserve explicitly named scope, then take a stable-domain bounded sample."""
|
|
1891
|
+
ordered = sorted(eligible, key=lambda a: _order_key(a, ads, pool))
|
|
1892
|
+
reserved = [a for a in ordered if a.named]
|
|
1893
|
+
others = [a for a in ordered if not a.named]
|
|
1894
|
+
room = max(cap - len(reserved), 0)
|
|
1895
|
+
chosen = sorted(reserved + others[:room], key=lambda a: _order_key(a, ads, pool))
|
|
1896
|
+
return chosen, others[room:]
|
|
1897
|
+
|
|
1898
|
+
|
|
1899
|
+
def _selection_text(item: Assessment, ad: dict[str, Any], cap: int, direct_count: int) -> str:
|
|
1900
|
+
return f"{item.tier}; caller-scoped inspection sample; {AD_LABELS[ad['ad_activity']]}"
|
|
1901
|
+
|
|
1902
|
+
|
|
1903
|
+
def _not_eligible_reason(item: Assessment, floor: float | None) -> str:
|
|
1904
|
+
if item.reason_code == "same_as":
|
|
1905
|
+
return f"same offer as {item.decision['same_as']}: {item.decision['reason']}"
|
|
1906
|
+
return item.decision["reason"]
|
|
1907
|
+
|
|
1908
|
+
|
|
1909
|
+
# ── rendering ───────────────────────────────────────────────────────────────
|
|
1910
|
+
|
|
1911
|
+
def _cell(value: Any, cap: int = 150) -> str:
|
|
1912
|
+
text = " ".join(str(value if value is not None and value != "" else "—").split()).replace("|", "\\|")
|
|
1913
|
+
return text if len(text) <= cap else text[: cap - 1] + "…"
|
|
1914
|
+
|
|
1915
|
+
|
|
1916
|
+
def _offer_lines(inputs: RunInputs) -> list[str]:
|
|
1917
|
+
offer = inputs.offer
|
|
1918
|
+
lines = []
|
|
1919
|
+
if offer.get("name") or offer.get("url"):
|
|
1920
|
+
lines.append(f"- Offer: {_cell(offer.get('name') or 'the product', 120)}" + (f" ({offer['url']})" if offer.get("url") else ""))
|
|
1921
|
+
for key, label in (("job", "Job"), ("buyer", "Buyer"), ("offer_class", "Offer class"), ("comparable_classes", "Comparable offer classes"),
|
|
1922
|
+
("duration", "Duration"), ("price_text", "Price")):
|
|
1923
|
+
if offer.get(key):
|
|
1924
|
+
lines.append(f"- {label}: {_cell(offer[key], 300)}")
|
|
1925
|
+
if inputs.price_floor_usd is not None:
|
|
1926
|
+
basis = f" ({offer['price_floor_basis']})" if offer.get("price_floor_basis") else ""
|
|
1927
|
+
lines.append(f"- Caller-supplied comparison floor: {_money(inputs.price_floor_usd)}{basis}")
|
|
1928
|
+
else:
|
|
1929
|
+
lines.append("- Caller-supplied comparison floor: not set; no default is inferred")
|
|
1930
|
+
lines.append(f"- Category phrases: {', '.join(inputs.category_phrases) or 'none'}")
|
|
1931
|
+
named = ", ".join(inputs.named_domains) or ("explicitly none" if inputs.input_annotations.get("operator_named_asked") is True else "unknown")
|
|
1932
|
+
lines.append(f"- Competitors named by the founder: {named}; mentioned on the product page: {', '.join(inputs.page_domains) or 'not supplied'}")
|
|
1933
|
+
lines.append(f"- Markets: {', '.join(inputs.markets)}; product domain {inputs.product_domain or 'not given'} (excluded, never a search seed)")
|
|
1934
|
+
if inputs.offer.get("confirmed_by_operator"):
|
|
1935
|
+
lines.append("- The founder confirmed this offer: " + _cell(inputs.offer.get("basis"), 300))
|
|
1936
|
+
elif inputs.offer.get("inferred"):
|
|
1937
|
+
lines.append("- The offer is an inference: " + _cell(inputs.offer.get("basis"), 300))
|
|
1938
|
+
elif inputs.intake_path:
|
|
1939
|
+
lines.append("- The offer card is not marked as confirmed by the founder.")
|
|
1940
|
+
else:
|
|
1941
|
+
lines.append(f"- No offer card was recorded ({INTAKE_REL.as_posix()} is missing); the inputs came from the command line.")
|
|
1942
|
+
return lines
|
|
1943
|
+
|
|
1944
|
+
|
|
1945
|
+
def render_section(*, inputs: RunInputs, cap: int, competitors: list[dict[str, Any]], profiled_out: list[dict[str, Any]],
|
|
1946
|
+
excluded: list[dict[str, Any]], summaries: list[dict[str, Any]], pool_size: int, coverage: dict[str, Any],
|
|
1947
|
+
totals: dict[str, Any], retrieved_on: str) -> str:
|
|
1948
|
+
lines: list[str] = []
|
|
1949
|
+
add = lines.append
|
|
1950
|
+
groups = {g: sum(1 for c in competitors if c["selection_group"] == g) for g in GROUP_LABELS.values()}
|
|
1951
|
+
add(
|
|
1952
|
+
f"Competitor discovery retrieved {retrieved_on}: {pool_size} candidate domain(s) from {len(summaries)} provider "
|
|
1953
|
+
f"query/report(s) in two rounds; {coverage['profiles']} profiled; {len(competitors)} competitor(s) kept "
|
|
1954
|
+
f"({groups['direct']} direct, {groups['indirect']} indirect, {groups['other scoped']} "
|
|
1955
|
+
f"other scoped); {len(excluded)} candidate(s) recorded as not included."
|
|
1956
|
+
)
|
|
1957
|
+
add("")
|
|
1958
|
+
add("### Offer searched from")
|
|
1959
|
+
add("")
|
|
1960
|
+
lines.extend(_offer_lines(inputs))
|
|
1961
|
+
add("")
|
|
1962
|
+
add("### Inspection scope")
|
|
1963
|
+
add("")
|
|
1964
|
+
add(f"- Up to {cap} caller-classified competitors, with explicitly named domains retained and the rest in stable domain order.")
|
|
1965
|
+
add("- Direct, indirect and substitute are recorded comparison roles. Price, ad activity and signal counts do not choose a business priority or exclude a substitute.")
|
|
1966
|
+
add("")
|
|
1967
|
+
add("### Competitor set")
|
|
1968
|
+
add("")
|
|
1969
|
+
if competitors:
|
|
1970
|
+
add("| # | Competitor | Tier | Provenance | Offer and format | Price | Ads | Why in the list |")
|
|
1971
|
+
add("| ---: | --- | --- | --- | --- | --- | --- | --- |")
|
|
1972
|
+
for row in competitors:
|
|
1973
|
+
offer = " — ".join(x for x in (row.get("offer"), row.get("offer_class")) if x) or None
|
|
1974
|
+
price = row.get("price_text") or _money(row.get("price_usd"))
|
|
1975
|
+
add(f"| {row['rank']} | {_cell(row['name'], 40)} ({_cell(row['domain'], 40)}) | {row['competitor_tier']} | {_cell(row['provenance_detail'], 60)} "
|
|
1976
|
+
f"| {_cell(offer, 110)} | {_cell(price, 50)} | {_cell('; '.join([AD_LABELS[row['ad_activity']]] + row['ad_evidence'][:2]), 120)} | {_cell(row['selection'], 140)} |")
|
|
1977
|
+
else:
|
|
1978
|
+
add("No competitor was confirmed in this run.")
|
|
1979
|
+
overrides = [row for row in competitors + profiled_out if row.get("override_reason")]
|
|
1980
|
+
if overrides:
|
|
1981
|
+
add("")
|
|
1982
|
+
add("Additional classification rationale: " + "; ".join(f"{r['domain']}: {_cell(r['override_reason'], 160)}" for r in overrides))
|
|
1983
|
+
add("")
|
|
1984
|
+
add("### Profiled, not included")
|
|
1985
|
+
add("")
|
|
1986
|
+
if profiled_out:
|
|
1987
|
+
add("| Domain | Tier | Offer and format | Price | Ads | Why not included |")
|
|
1988
|
+
add("| --- | --- | --- | --- | --- | --- |")
|
|
1989
|
+
for row in profiled_out[:PROFILED_ROWS_RENDERED]:
|
|
1990
|
+
offer = " — ".join(x for x in (row.get("offer"), row.get("offer_class")) if x) or None
|
|
1991
|
+
price = row.get("price_text") or _money(row.get("price_usd"))
|
|
1992
|
+
ads = AD_LABELS.get(row.get("ad_activity") or "", "not checked")
|
|
1993
|
+
add(f"| {_cell(row['domain'], 40)} | {row.get('competitor_tier') or '—'} | {_cell(offer, 110)} | {_cell(price, 50)} | {ads} | {_cell(row['reason'], 160)} |")
|
|
1994
|
+
if len(profiled_out) > PROFILED_ROWS_RENDERED:
|
|
1995
|
+
add("")
|
|
1996
|
+
add(f"{len(profiled_out) - PROFILED_ROWS_RENDERED} more profiled candidate(s) are recorded in the evidence file.")
|
|
1997
|
+
else:
|
|
1998
|
+
add("Every profiled candidate is in the list.")
|
|
1999
|
+
others = [row for row in excluded if not row.get("profiled") and row["reason_code"] not in ("directory", "over_cap")]
|
|
2000
|
+
if others:
|
|
2001
|
+
add("")
|
|
2002
|
+
counts: dict[str, int] = {}
|
|
2003
|
+
for row in others:
|
|
2004
|
+
counts[row["reason_code"]] = counts.get(row["reason_code"], 0) + 1
|
|
2005
|
+
add("Not profiled: " + ", ".join(f"{n} {code.replace('_', ' ')}" for code, n in sorted(counts.items())) + "; examples: "
|
|
2006
|
+
+ ", ".join(row["domain"] for row in others[:EXCLUDED_ROWS_RENDERED]) + ".")
|
|
2007
|
+
directories = [row["domain"] for row in excluded if row["reason_code"] == "directory"]
|
|
2008
|
+
if directories:
|
|
2009
|
+
add("")
|
|
2010
|
+
add("Review and listing directories (recorded, never promoted): " + ", ".join(directories[:EXCLUDED_ROWS_RENDERED]) + ".")
|
|
2011
|
+
add("")
|
|
2012
|
+
add("### Discovery and enrichment coverage")
|
|
2013
|
+
add("")
|
|
2014
|
+
add("| Round | Query | Source | Items | Candidates |")
|
|
2015
|
+
add("| ---: | --- | --- | ---: | ---: |")
|
|
2016
|
+
for row in summaries:
|
|
2017
|
+
items = "unavailable" if row.get("unavailable") else str(row["retrieved"])
|
|
2018
|
+
add(f"| {row['round']} | {_cell(row['query'], 80)} | {row['source']} | {items} | {row['candidates']} |")
|
|
2019
|
+
if any(row.get("unavailable") for row in summaries):
|
|
2020
|
+
add("")
|
|
2021
|
+
add("Unavailable queries were rejected by the provider or failed; they are recorded, not retried silently.")
|
|
2022
|
+
add("")
|
|
2023
|
+
add(f"- Listicles: {coverage['listicles_read']} page(s) read, {coverage['listicles_unreadable']} unreadable, "
|
|
2024
|
+
f"{coverage['listicle_entries']} program entries recorded.")
|
|
2025
|
+
add(f"- Profiles: {coverage['profiles']} domain(s) profiled on their own sites; {coverage['agent_additions']} added by the agent "
|
|
2026
|
+
f"from its reading, each with the page it came from.")
|
|
2027
|
+
seeds = ", ".join(coverage["round2_seeds"]) or "none (no confirmed direct or comparable competitor)"
|
|
2028
|
+
add(f"- Round 2 seeds (confirmed direct competitors): {seeds}.")
|
|
2029
|
+
add(f"- Review: {coverage['review_rounds']} optional recorded review round(s); {coverage['objections_accepted']} objection(s) "
|
|
2030
|
+
f"accepted, {coverage['objections_rejected']} rejected; {coverage['missing_added']} missing rival(s) added, "
|
|
2031
|
+
f"{coverage['missing_rejected']} rejected.")
|
|
2032
|
+
add(f"- Ad check: {coverage['ad_checks']} domain(s); " + ", ".join(f"{n} {AD_LABELS[k]}" for k, n in coverage["ad_counts"].items() if n) + ".")
|
|
2033
|
+
add("")
|
|
2034
|
+
add("### Method and coverage")
|
|
2035
|
+
add("")
|
|
2036
|
+
add(
|
|
2037
|
+
f"- Spend recorded: {totals['dataforseo_calls']} answered DataForSEO task(s), {totals['spyfu_calls']} SpyFu lookup(s), "
|
|
2038
|
+
f"{totals['search_calls']} web search(es), {totals['meta_searches']} Meta Ad Library search(es) and {totals['page_fetches']} free page fetch(es)"
|
|
2039
|
+
+ (f", plus {totals['rejected_calls']} rejected query(ies) that count as spend but not as evidence" if totals.get("rejected_calls") else "")
|
|
2040
|
+
+ ", accumulated across attempts; cached replays add no new spend."
|
|
2041
|
+
)
|
|
2042
|
+
add(
|
|
2043
|
+
"- Signal strength orders the profiling list and breaks ties, never a tier: a paid result on a category query "
|
|
2044
|
+
f"{SCORE_CATEGORY_PAID:g} (up to {CAP_CATEGORY_PAID}), a Google advertiser on a category phrase {SCORE_KEYWORD_ADS:g} "
|
|
2045
|
+
f"(up to {CAP_KEYWORD_ADS}), an organic category result {SCORE_CATEGORY_ORGANIC:g} (up to {CAP_CATEGORY_ORGANIC}), a second-engine "
|
|
2046
|
+
f"result {SCORE_SECOND_ENGINE:g} (up to {CAP_SECOND_ENGINE}), an alternatives-query hit {SCORE_BRAND_QUERY:g} (up to {CAP_BRAND_QUERY}), "
|
|
2047
|
+
f"a listicle page {SCORE_LISTICLE:g} (up to {CAP_LISTICLE}), a round-2 relationship hit {SCORE_RELATIONSHIP:g} (up to {CAP_RELATIONSHIP}). "
|
|
2048
|
+
"Being named by the founder or on the product page adds nothing."
|
|
2049
|
+
)
|
|
2050
|
+
add(
|
|
2051
|
+
f"- Caps: {MAX_CATEGORY_PHRASES} category phrases (two queries each), {MAX_BRAND_QUERIES} alternatives queries for competitors named by the founder; "
|
|
2052
|
+
f"{MAX_LISTICLES_TO_READ} listicle pages read, {MAX_PROFILES_ROUND1} profiles in round 1 and {MAX_PROFILES_ROUND2} in round 2, "
|
|
2053
|
+
f"{MAX_AGENT_ADDITIONS} agent additions, {ROUND2_SEEDS} round-2 seeds, {MAX_AD_CHECKS} ad checks, {cap} competitors."
|
|
2054
|
+
)
|
|
2055
|
+
add("")
|
|
2056
|
+
add("### Sources")
|
|
2057
|
+
add("")
|
|
2058
|
+
add(
|
|
2059
|
+
"- DataForSEO Google SERP (paid and organic) and Labs `competitors_domain`, SpyFu keyword ad history, PPC competitors and "
|
|
2060
|
+
"domain stats, the official Meta Ad Library, Brave web search via OS tools, free home-page fetches, and the executing "
|
|
2061
|
+
f"agent's listicle reading, site profiles and optional review — retrieved {retrieved_on}; the candidate, competitor and "
|
|
2062
|
+
f"exclusion rows in `{EVIDENCE_REL.as_posix()}` are exactly the set every table above is computed from."
|
|
2063
|
+
)
|
|
2064
|
+
return "\n".join(lines).strip() + "\n"
|
|
2065
|
+
|
|
2066
|
+
|
|
2067
|
+
# ── orchestration ───────────────────────────────────────────────────────────
|
|
2068
|
+
|
|
2069
|
+
def _write_atomic(path: Path, content: str) -> None:
|
|
2070
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
2071
|
+
tmp = path.with_suffix(path.suffix + ".tmp")
|
|
2072
|
+
tmp.write_text(content, encoding="utf-8")
|
|
2073
|
+
tmp.replace(path)
|
|
2074
|
+
|
|
2075
|
+
|
|
2076
|
+
def _sha(path: Path) -> str | None:
|
|
2077
|
+
return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None
|
|
2078
|
+
|
|
2079
|
+
|
|
2080
|
+
def run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
|
|
2081
|
+
workspace = workspace.resolve()
|
|
2082
|
+
io = _load_script("research-artifact-io.py", "pmr_artifact_io")
|
|
2083
|
+
with io.section_attempt(workspace, SECTION_ID, context_file=getattr(args, "context_file", None)):
|
|
2084
|
+
return _run(workspace, args)
|
|
2085
|
+
|
|
2086
|
+
|
|
2087
|
+
def _inputs_block(inputs: RunInputs, cap: int) -> dict[str, Any]:
|
|
2088
|
+
return {
|
|
2089
|
+
"offer": inputs.offer,
|
|
2090
|
+
"operator_named": inputs.named_domains,
|
|
2091
|
+
"page_mentioned": inputs.page_domains,
|
|
2092
|
+
"keyword_domains": inputs.keyword_domains,
|
|
2093
|
+
"product_domain": inputs.product_domain,
|
|
2094
|
+
"category_phrases": inputs.category_phrases,
|
|
2095
|
+
"queries": inputs.category_queries + inputs.brand_queries,
|
|
2096
|
+
"price_floor_usd": inputs.price_floor_usd,
|
|
2097
|
+
"markets": inputs.markets,
|
|
2098
|
+
"location_code": inputs.location_code,
|
|
2099
|
+
"language_code": inputs.language_code,
|
|
2100
|
+
"max_competitors": cap,
|
|
2101
|
+
"intake_path": inputs.intake_path,
|
|
2102
|
+
"input_annotations": inputs.input_annotations,
|
|
2103
|
+
}
|
|
2104
|
+
|
|
2105
|
+
|
|
2106
|
+
def _pending(workspace: Path, runtime: Path, decisions_path: Path, client: OsClient, reason: str, next_step: str, missing: dict[str, list[str]]) -> dict[str, Any]:
|
|
2107
|
+
state = {
|
|
2108
|
+
"schema_version": SCHEMA_VERSION,
|
|
2109
|
+
"section_id": SECTION_ID,
|
|
2110
|
+
"status": "pending",
|
|
2111
|
+
"inputs_path": (RUNTIME_REL / "inputs.json").as_posix(),
|
|
2112
|
+
"inputs_sha256": _sha(runtime / "inputs.json"),
|
|
2113
|
+
"research_context": _load_script("research-artifact-io.py", "pmr_input_io").context_binding(workspace, SECTION_ID),
|
|
2114
|
+
"error": None,
|
|
2115
|
+
"reason": reason,
|
|
2116
|
+
"next_step": next_step,
|
|
2117
|
+
"missing": missing,
|
|
2118
|
+
"candidates_path": (RUNTIME_REL / CANDIDATES_FILENAME).as_posix(),
|
|
2119
|
+
"decisions_path": decisions_path.as_posix(),
|
|
2120
|
+
"undecided": missing.get("decisions", []),
|
|
2121
|
+
"attempt_dataforseo_tasks": client.dataforseo_tasks,
|
|
2122
|
+
"attempt_search_calls": client.search_calls,
|
|
2123
|
+
"attempt_spyfu_calls": client.spyfu_calls,
|
|
2124
|
+
"attempt_cache_hits": client.cache_hits,
|
|
2125
|
+
}
|
|
2126
|
+
_write_atomic(runtime / "request.json", json.dumps(state, indent=1))
|
|
2127
|
+
return state
|
|
2128
|
+
|
|
2129
|
+
|
|
2130
|
+
def _write_candidates(runtime: Path, *, inputs: RunInputs, cap: int, files: AgentFiles, discovery: Discovery, to_read: list[str],
|
|
2131
|
+
required1: list[str], required2: list[str] | None, seeds: list[str] | None, missing: dict[str, list[str]], status: str) -> None:
|
|
2132
|
+
profiled = set(files.profiles)
|
|
2133
|
+
rows = sorted(discovery.pool.values(), key=lambda c: (-c.score(), c.domain))
|
|
2134
|
+
payload = {
|
|
2135
|
+
"schema_version": SCHEMA_VERSION,
|
|
2136
|
+
"section_id": SECTION_ID,
|
|
2137
|
+
"status": status,
|
|
2138
|
+
"files": {name: path.as_posix() for name, path in files.paths.items()},
|
|
2139
|
+
"inputs": _inputs_block(inputs, cap),
|
|
2140
|
+
"listicles_to_read": to_read,
|
|
2141
|
+
"profile_required": {"round1": required1, "round2": required2 if required2 is not None else "after round 1 is complete"},
|
|
2142
|
+
"round2_seeds": seeds,
|
|
2143
|
+
"missing": missing,
|
|
2144
|
+
"rules": {
|
|
2145
|
+
"roles": "caller-supplied roles with sourced profile and reason",
|
|
2146
|
+
"price": "recorded comparison only; no automatic floor",
|
|
2147
|
+
"order": "explicitly named scope retained; stable domain inspection sample",
|
|
2148
|
+
},
|
|
2149
|
+
"candidates": [{**c.row(), "required": c.domain in required1 or c.domain in (required2 or []), "profiled": c.domain in profiled} for c in rows],
|
|
2150
|
+
"directories": sorted(discovery.directories),
|
|
2151
|
+
"search_summaries": discovery.summaries,
|
|
2152
|
+
}
|
|
2153
|
+
_write_atomic(runtime / CANDIDATES_FILENAME, json.dumps(payload, indent=1, ensure_ascii=False))
|
|
2154
|
+
|
|
2155
|
+
|
|
2156
|
+
def _pending_message(missing: dict[str, list[str]]) -> str:
|
|
2157
|
+
parts = []
|
|
2158
|
+
for key, label in (("listicles", "listicle page(s) to read"), ("profiles", "domain(s) to profile"), ("decisions", "domain(s) to decide"),
|
|
2159
|
+
("review", "domain(s) for the reviewer"), ("objections", "objection(s) to resolve"), ("missing_rivals", "missing rival(s) to settle"),
|
|
2160
|
+
("round2_review", "round-2 domain(s) for a review round of the round-2 pool")):
|
|
2161
|
+
if missing.get(key):
|
|
2162
|
+
parts.append(f"{len(missing[key])} {label}")
|
|
2163
|
+
return ", ".join(parts)
|
|
2164
|
+
|
|
2165
|
+
|
|
2166
|
+
def _run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
|
|
2167
|
+
workspace = workspace.resolve()
|
|
2168
|
+
runtime = workspace / RUNTIME_REL
|
|
2169
|
+
cache_dir = runtime / "cache"
|
|
2170
|
+
runtime.mkdir(parents=True, exist_ok=True)
|
|
2171
|
+
|
|
2172
|
+
api_key = (os.environ.get("OS_API_KEY") or "").strip()
|
|
2173
|
+
if not api_key:
|
|
2174
|
+
raise CompetitorSetError("OS_API_KEY is required")
|
|
2175
|
+
os_url = (os.environ.get("OS_URL") or DEFAULT_OS_URL).strip() or DEFAULT_OS_URL
|
|
2176
|
+
|
|
2177
|
+
cap = max(1, int(_arg(args, "max_competitors", MAX_COMPETITORS)))
|
|
2178
|
+
inputs = bind_inputs(workspace, args)
|
|
2179
|
+
client = OsClient(
|
|
2180
|
+
os_url=os_url,
|
|
2181
|
+
api_key=api_key,
|
|
2182
|
+
cache_dir=cache_dir,
|
|
2183
|
+
max_dataforseo_tasks=int(_arg(args, "max_dataforseo_tasks", MAX_DATAFORSEO_TASKS_PER_ATTEMPT)),
|
|
2184
|
+
max_search_calls=int(_arg(args, "max_search_calls", MAX_SEARCH_CALLS_PER_ATTEMPT)),
|
|
2185
|
+
max_spyfu_calls=int(_arg(args, "max_spyfu_calls", MAX_SPYFU_CALLS_PER_ATTEMPT)),
|
|
2186
|
+
max_meta_searches=int(_arg(args, "max_meta_searches", MAX_META_SEARCHES_PER_ATTEMPT)),
|
|
2187
|
+
max_fetches=int(_arg(args, "max_fetches", MAX_FETCHES_PER_ATTEMPT)),
|
|
2188
|
+
)
|
|
2189
|
+
|
|
2190
|
+
discovery = run_round1(client, inputs)
|
|
2191
|
+
if not discovery.summaries:
|
|
2192
|
+
raise CompetitorSetError("no discovery query could be formed from the inputs: give the offer's category phrases or competitors named by the founder")
|
|
2193
|
+
if not any(not s.get("unavailable") for s in discovery.summaries):
|
|
2194
|
+
# Nothing proves the market was searched: a provider that rejected or
|
|
2195
|
+
# failed every query must not become a "completed" section with an
|
|
2196
|
+
# empty or named-only competitor list.
|
|
2197
|
+
rejected = sum(1 for s in discovery.summaries if s.get("unavailable"))
|
|
2198
|
+
raise CompetitorSetError(
|
|
2199
|
+
f"discovery unavailable: none of the {len(discovery.summaries)} provider queries answered "
|
|
2200
|
+
f"({rejected} rejected or failed); the section cannot complete without a successful search. "
|
|
2201
|
+
"Rejected DataForSEO queries are cached and will not be retried unchanged: change the category phrases "
|
|
2202
|
+
"or retry later if the web search was rate-limited."
|
|
2203
|
+
)
|
|
2204
|
+
|
|
2205
|
+
decisions_arg = str(_arg(args, "decisions", "") or "")
|
|
2206
|
+
decisions_path = Path(decisions_arg) if decisions_arg else runtime / DECISIONS_FILENAME
|
|
2207
|
+
if not decisions_path.is_absolute():
|
|
2208
|
+
decisions_path = workspace / decisions_path
|
|
2209
|
+
files = load_agent_files(runtime, decisions_path)
|
|
2210
|
+
to_read = listicles_to_read(discovery.summaries, inputs.product_domain)
|
|
2211
|
+
additions = add_agent_research(discovery.pool, files, to_read, inputs.product_domain)
|
|
2212
|
+
apply_aliases(discovery.pool, files.decisions)
|
|
2213
|
+
required1 = required_round1(discovery.pool, inputs, files.decisions)
|
|
2214
|
+
round1_domains = set(discovery.pool)
|
|
2215
|
+
assessments = build_assessments(discovery.pool, files, inputs)
|
|
2216
|
+
check_consistency(assessments)
|
|
2217
|
+
missing1 = missing_items(domains=round1_domains, required=required1, inputs=inputs, files=files, to_read=to_read, assessments=assessments)
|
|
2218
|
+
if missing1:
|
|
2219
|
+
_write_candidates(runtime, inputs=inputs, cap=cap, files=files, discovery=discovery, to_read=to_read, required1=required1,
|
|
2220
|
+
required2=None, seeds=None, missing=missing1, status="enrich_round1")
|
|
2221
|
+
return _pending(workspace, runtime, decisions_path, client, "enrichment of round 1 incomplete: " + _pending_message(missing1), "enrich_round1", missing1)
|
|
2222
|
+
|
|
2223
|
+
seeds = round2_seeds(discovery.pool, assessments, inputs)
|
|
2224
|
+
if seeds:
|
|
2225
|
+
run_round2(client, inputs, discovery, seeds)
|
|
2226
|
+
additions = own_additions(discovery.pool, additions)
|
|
2227
|
+
unknown = sorted(d for d in set(files.decisions) | {d for d, p in files.profiles.items() if not p["found_in"]} if d not in discovery.pool)
|
|
2228
|
+
if unknown:
|
|
2229
|
+
raise CompetitorSetError(
|
|
2230
|
+
"decisions or profiles name domains no source recorded: " + ", ".join(unknown[:10])
|
|
2231
|
+
+ " — give a profile with found_in (the page where the agent found it), or remove them (the round-2 seeds may have changed)"
|
|
2232
|
+
)
|
|
2233
|
+
apply_aliases(discovery.pool, files.decisions)
|
|
2234
|
+
required2 = required_round2(discovery.pool, required1, files.decisions)
|
|
2235
|
+
assessments = build_assessments(discovery.pool, files, inputs)
|
|
2236
|
+
check_consistency(assessments)
|
|
2237
|
+
missing2 = missing_items(domains=set(discovery.pool), required=required1 + required2, inputs=inputs, files=files, to_read=to_read, assessments=assessments)
|
|
2238
|
+
if missing2:
|
|
2239
|
+
_write_candidates(runtime, inputs=inputs, cap=cap, files=files, discovery=discovery, to_read=to_read, required1=required1,
|
|
2240
|
+
required2=required2, seeds=seeds, missing=missing2, status="enrich_round2")
|
|
2241
|
+
return _pending(workspace, runtime, decisions_path, client, "enrichment of round 2 incomplete: " + _pending_message(missing2), "enrich_round2", missing2)
|
|
2242
|
+
|
|
2243
|
+
# Ad check for every domain that can enter the list (those named by the founder first, then in tier order).
|
|
2244
|
+
eligible = [a for a in assessments.values() if a.group is not None]
|
|
2245
|
+
eligible.sort(key=lambda a: (0 if a.named else 1, a.domain))
|
|
2246
|
+
ads: dict[str, dict[str, Any]] = {}
|
|
2247
|
+
checked_ads: list[str] = []
|
|
2248
|
+
for item in eligible[:max(MAX_AD_CHECKS, sum(1 for a in eligible if a.named))]:
|
|
2249
|
+
ads[item.domain] = check_ads(client, inputs, discovery.pool[item.domain], focused=(item.profile or {}).get("focused") is not False)
|
|
2250
|
+
checked_ads.append(item.domain)
|
|
2251
|
+
for item in eligible:
|
|
2252
|
+
ads.setdefault(item.domain, {"ad_activity": "unknown", "ad_evidence": [f"not checked: over the ad-check cap of {MAX_AD_CHECKS}"],
|
|
2253
|
+
"ad_kinds": [], "ad_strength": 0, "google_budget_usd": None, "pixel_tags": [], "lookup_gaps": []})
|
|
2254
|
+
chosen, over = compose(eligible, ads, discovery.pool, cap)
|
|
2255
|
+
_write_candidates(runtime, inputs=inputs, cap=cap, files=files, discovery=discovery, to_read=to_read, required1=required1,
|
|
2256
|
+
required2=required2, seeds=seeds, missing={}, status="complete")
|
|
2257
|
+
|
|
2258
|
+
direct_count = sum(1 for a in eligible if a.group == 0)
|
|
2259
|
+
pool = discovery.pool
|
|
2260
|
+
|
|
2261
|
+
def profile_fields(domain: str) -> dict[str, Any]:
|
|
2262
|
+
profile = files.profiles.get(domain) or {}
|
|
2263
|
+
return {key: profile.get(key) for key in ("offer", "offer_url", "offer_class", "offer_match", "price_text", "price_usd",
|
|
2264
|
+
"duration", "buyer", "job", "operating", "marketplace", "evidence_urls")}
|
|
2265
|
+
|
|
2266
|
+
def provenance_detail(candidate: Candidate) -> str:
|
|
2267
|
+
primary = candidate.primary_provenance()
|
|
2268
|
+
if primary == "agent_research" and candidate.found_in:
|
|
2269
|
+
return f"agent research ({candidate.found_in})"
|
|
2270
|
+
if primary == "discovered":
|
|
2271
|
+
return "discovered (" + ", ".join(candidate.sources[:3]) + ")"
|
|
2272
|
+
return primary.replace("_", " ")
|
|
2273
|
+
|
|
2274
|
+
competitors: list[dict[str, Any]] = []
|
|
2275
|
+
for rank, item in enumerate(chosen, start=1):
|
|
2276
|
+
candidate = pool[item.domain]
|
|
2277
|
+
ad = ads[item.domain]
|
|
2278
|
+
profile = files.profiles.get(item.domain) or {}
|
|
2279
|
+
competitors.append({
|
|
2280
|
+
"type": "competitor",
|
|
2281
|
+
"rank": rank,
|
|
2282
|
+
"domain": item.domain,
|
|
2283
|
+
"name": item.decision["name"] or profile.get("name") or default_name(item.domain),
|
|
2284
|
+
"competitor_tier": item.tier,
|
|
2285
|
+
"reason": item.decision["reason"],
|
|
2286
|
+
"icp_match": profile.get("same_buyer"),
|
|
2287
|
+
"jtbd_match": profile.get("same_job"),
|
|
2288
|
+
"provenance": candidate.ordered_provenance(),
|
|
2289
|
+
"provenance_detail": provenance_detail(candidate),
|
|
2290
|
+
"found_in": candidate.found_in,
|
|
2291
|
+
**profile_fields(item.domain),
|
|
2292
|
+
"price_band": item.price_band,
|
|
2293
|
+
"comparable": item.comparable,
|
|
2294
|
+
"ad_activity": ad["ad_activity"],
|
|
2295
|
+
"ad_evidence": ad["ad_evidence"],
|
|
2296
|
+
"selection_group": GROUP_LABELS[item.group],
|
|
2297
|
+
"selection": _selection_text(item, ad, cap, direct_count),
|
|
2298
|
+
"override_reason": item.decision.get("override_reason"),
|
|
2299
|
+
"reviewed": item.domain in files.review["checked"],
|
|
2300
|
+
"sources": list(candidate.sources),
|
|
2301
|
+
"source_count": len(candidate.sources),
|
|
2302
|
+
"score": candidate.score(),
|
|
2303
|
+
"decided_by": "agent",
|
|
2304
|
+
"tier": "B" if (candidate.sources and len(candidate.sources) >= 2) or ad["ad_activity"] == "ads_seen" else "C",
|
|
2305
|
+
})
|
|
2306
|
+
|
|
2307
|
+
profiled_out: list[dict[str, Any]] = []
|
|
2308
|
+
excluded: list[dict[str, Any]] = []
|
|
2309
|
+
for item in sorted(over, key=lambda a: _order_key(a, ads, pool)):
|
|
2310
|
+
ad = ads[item.domain]
|
|
2311
|
+
excluded.append({
|
|
2312
|
+
"type": "excluded_candidate", "domain": item.domain, "competitor_tier": item.tier, "reason_code": "over_cap",
|
|
2313
|
+
"reason": f"over the cap of {cap}: {GROUP_LABELS[item.group]}, {AD_LABELS[ad['ad_activity']]}",
|
|
2314
|
+
"decided_by": "cap", "provenance": pool[item.domain].ordered_provenance(), "profiled": item.domain in files.profiles,
|
|
2315
|
+
**profile_fields(item.domain), "price_band": item.price_band, "ad_activity": ad["ad_activity"], "ad_evidence": ad["ad_evidence"],
|
|
2316
|
+
"override_reason": item.decision.get("override_reason"), "listed_for": [], "score": pool[item.domain].score(), "tier": "C",
|
|
2317
|
+
})
|
|
2318
|
+
for domain in sorted(assessments, key=lambda d: (assessments[d].reason_code, -pool[d].score(), d)):
|
|
2319
|
+
item = assessments[domain]
|
|
2320
|
+
if item.group is not None:
|
|
2321
|
+
continue
|
|
2322
|
+
excluded.append({
|
|
2323
|
+
"type": "excluded_candidate", "domain": domain, "competitor_tier": item.tier, "reason_code": item.reason_code,
|
|
2324
|
+
"reason": _not_eligible_reason(item, inputs.price_floor_usd), "decided_by": "agent" if item.tier == "excluded" else "rule",
|
|
2325
|
+
"provenance": pool[domain].ordered_provenance(), "profiled": domain in files.profiles, **profile_fields(domain),
|
|
2326
|
+
"price_band": item.price_band, "override_reason": item.decision.get("override_reason"),
|
|
2327
|
+
"listed_for": list(pool[domain].listed_for[:8]), "score": pool[domain].score(), "tier": "C",
|
|
2328
|
+
})
|
|
2329
|
+
profiled_out = [row for row in excluded if row.get("profiled")]
|
|
2330
|
+
decided = set(assessments)
|
|
2331
|
+
required_all = set(required1) | set(required2)
|
|
2332
|
+
for candidate in sorted(pool.values(), key=lambda c: (-c.score(), c.domain)):
|
|
2333
|
+
if candidate.domain in decided:
|
|
2334
|
+
continue
|
|
2335
|
+
if candidate.domain in required_all:
|
|
2336
|
+
continue # unreachable: completeness requires a decision
|
|
2337
|
+
if candidate.round == 2 and not candidate.round1_signal():
|
|
2338
|
+
code, why = "round2_single_hit", "one relationship hit in round 2; not profiled"
|
|
2339
|
+
elif candidate.offer_signal():
|
|
2340
|
+
code, why = "over_profile_cap", f"over the profiling cap of {MAX_PROFILES_ROUND1}; not profiled"
|
|
2341
|
+
else:
|
|
2342
|
+
code, why = "weak_signal", "a single weak discovery hit; not profiled"
|
|
2343
|
+
excluded.append({
|
|
2344
|
+
"type": "excluded_candidate", "domain": candidate.domain, "reason_code": code, "reason": why, "decided_by": "profiling_list",
|
|
2345
|
+
"provenance": candidate.ordered_provenance(), "profiled": False, "listed_for": list(candidate.listed_for[:8]),
|
|
2346
|
+
"score": candidate.score(), "tier": "C",
|
|
2347
|
+
})
|
|
2348
|
+
for domain, queries in sorted(discovery.directories.items()):
|
|
2349
|
+
excluded.append({"type": "excluded_candidate", "domain": domain, "reason_code": "directory", "reason": "directory",
|
|
2350
|
+
"decided_by": "noise_filter", "profiled": False, "listed_for": sorted(set(queries))[:8], "score": None, "tier": "C"})
|
|
2351
|
+
|
|
2352
|
+
# Every pool domain already has a competitor or exclusion row; the
|
|
2353
|
+
# candidate rows (discovery facts) keep the strongest MAX_CANDIDATES.
|
|
2354
|
+
candidate_rows = [c.row() for c in sorted(pool.values(), key=lambda c: (-c.score(), c.domain))][:MAX_CANDIDATES]
|
|
2355
|
+
totals = client.ledger_totals()
|
|
2356
|
+
retrieved_on = totals["first_retrieved_on"] if totals["first_retrieved_on"] == totals["last_retrieved_on"] else f"{totals['first_retrieved_on']}..{totals['last_retrieved_on']}"
|
|
2357
|
+
listicle_pages = [files.listicles[url] for url in to_read if url in files.listicles]
|
|
2358
|
+
ad_counts = {key: sum(1 for ad in ads.values() if ad["ad_activity"] == key) for key in AD_ORDER}
|
|
2359
|
+
coverage = {
|
|
2360
|
+
"listicles_read": sum(1 for p in listicle_pages if p["status"] == "read"),
|
|
2361
|
+
"listicles_unreadable": sum(1 for p in listicle_pages if p["status"] == "unreadable"),
|
|
2362
|
+
"listicle_entries": sum(len(p["entries"]) for p in listicle_pages),
|
|
2363
|
+
"profiles": sum(1 for d in files.profiles if d in pool),
|
|
2364
|
+
"agent_additions": len(additions),
|
|
2365
|
+
"round2_seeds": seeds,
|
|
2366
|
+
"review_rounds": len(files.review["rounds"]),
|
|
2367
|
+
"objections_accepted": sum(1 for o in files.review["objections"] if o["resolution"] == "accepted"),
|
|
2368
|
+
"objections_rejected": sum(1 for o in files.review["objections"] if o["resolution"] == "rejected"),
|
|
2369
|
+
"missing_added": sum(1 for m in files.review["missing"] if m["resolution"] == "added"),
|
|
2370
|
+
"missing_rejected": sum(1 for m in files.review["missing"] if m["resolution"] == "rejected"),
|
|
2371
|
+
"ad_checks": len(checked_ads),
|
|
2372
|
+
"ad_counts": ad_counts,
|
|
2373
|
+
}
|
|
2374
|
+
evidence = {
|
|
2375
|
+
"schema_version": SCHEMA_VERSION,
|
|
2376
|
+
"section_id": SECTION_ID,
|
|
2377
|
+
"generated_by": Path(__file__).name,
|
|
2378
|
+
"retrieved_on": retrieved_on,
|
|
2379
|
+
"inputs": _inputs_block(inputs, cap),
|
|
2380
|
+
"composition_rule": {
|
|
2381
|
+
"roles": "caller-supplied roles with sourced profile and reason",
|
|
2382
|
+
"price": "recorded comparison only; no automatic floor",
|
|
2383
|
+
"order": "stable domain inspection order, not business priority",
|
|
2384
|
+
"operator_named": "explicitly named scope retained within the requested sample",
|
|
2385
|
+
"max_competitors": cap,
|
|
2386
|
+
},
|
|
2387
|
+
"enrichment": {key: value for key, value in coverage.items() if key != "ad_counts"} | {"ad_counts": ad_counts},
|
|
2388
|
+
"rows": discovery.summaries + candidate_rows + competitors + excluded,
|
|
2389
|
+
"totals": {key: totals[key] for key in ("dataforseo_calls", "search_calls", "spyfu_calls", "meta_searches", "page_fetches", "provider_calls", "rejected_calls")}
|
|
2390
|
+
| {"answered_queries": sum(1 for s in discovery.summaries if not s.get("unavailable"))},
|
|
2391
|
+
}
|
|
2392
|
+
section_markdown = render_section(
|
|
2393
|
+
inputs=inputs, cap=cap, competitors=competitors, profiled_out=profiled_out, excluded=excluded, summaries=discovery.summaries,
|
|
2394
|
+
pool_size=len(pool), coverage=coverage, totals=totals, retrieved_on=retrieved_on,
|
|
2395
|
+
)
|
|
2396
|
+
|
|
2397
|
+
evidence_path = workspace / EVIDENCE_REL
|
|
2398
|
+
evidence_json = json.dumps(evidence, indent=1, ensure_ascii=False)
|
|
2399
|
+
_write_atomic(evidence_path, evidence_json)
|
|
2400
|
+
section_file = runtime / "result.md"
|
|
2401
|
+
_write_atomic(section_file, section_markdown)
|
|
2402
|
+
|
|
2403
|
+
upsert = _load_script("upsert-markdown-section.py", "pmr_upsert_markdown_section")
|
|
2404
|
+
report_path = workspace / REPORT_REL
|
|
2405
|
+
report_path.parent.mkdir(parents=True, exist_ok=True)
|
|
2406
|
+
upsert.upsert_markdown_section(report_path, SECTION_HEADING, section_markdown)
|
|
2407
|
+
|
|
2408
|
+
persist = _load_script("persist-research-report.py", "pmr_persist_research_report")
|
|
2409
|
+
persistence = persist.persist_research_report(
|
|
2410
|
+
workspace,
|
|
2411
|
+
commit_message=f"research: update {SECTION_HEADING.lower()} section",
|
|
2412
|
+
extra_paths=(Path(_artifact_path("egbe-docs/research/market/evidence/competitors.json")),),
|
|
2413
|
+
)
|
|
2414
|
+
|
|
2415
|
+
legacy_source = {"operator_named": "explicit"}
|
|
2416
|
+
state = {
|
|
2417
|
+
"schema_version": SCHEMA_VERSION,
|
|
2418
|
+
"section_id": SECTION_ID,
|
|
2419
|
+
"status": "completed",
|
|
2420
|
+
"domains": [row["domain"] for row in competitors],
|
|
2421
|
+
"domain_sources": {row["domain"]: legacy_source.get(row["provenance"][0], row["provenance"][0]) if row["provenance"] else "discovered" for row in competitors},
|
|
2422
|
+
"explicit_domains": inputs.named_domains,
|
|
2423
|
+
"page_domains": inputs.page_domains,
|
|
2424
|
+
"product_domain": inputs.product_domain,
|
|
2425
|
+
"category_phrases": inputs.category_phrases,
|
|
2426
|
+
"queries": inputs.category_queries + inputs.brand_queries,
|
|
2427
|
+
"price_floor_usd": inputs.price_floor_usd,
|
|
2428
|
+
"round2_seeds": seeds,
|
|
2429
|
+
"evidence_path": EVIDENCE_REL.as_posix(),
|
|
2430
|
+
"evidence_sha256": hashlib.sha256(evidence_json.encode("utf-8")).hexdigest(),
|
|
2431
|
+
"result_path": (RUNTIME_REL / "result.md").as_posix(),
|
|
2432
|
+
"result_sha256": hashlib.sha256(section_markdown.encode("utf-8")).hexdigest(),
|
|
2433
|
+
"intake_sha256": inputs.intake_sha256,
|
|
2434
|
+
"intake_path": inputs.intake_path,
|
|
2435
|
+
"inputs_path": (RUNTIME_REL / "inputs.json").as_posix(),
|
|
2436
|
+
"inputs_sha256": _sha(runtime / "inputs.json"),
|
|
2437
|
+
"decisions_sha256": _sha(files.paths["decisions"]),
|
|
2438
|
+
"profiles_sha256": _sha(files.paths["profiles"]),
|
|
2439
|
+
"listicles_sha256": _sha(files.paths["listicles"]),
|
|
2440
|
+
"review_sha256": _sha(files.paths["review"]),
|
|
2441
|
+
"provider_calls": totals["provider_calls"],
|
|
2442
|
+
"dataforseo_calls": totals["dataforseo_calls"],
|
|
2443
|
+
"search_calls": totals["search_calls"],
|
|
2444
|
+
"spyfu_calls": totals["spyfu_calls"],
|
|
2445
|
+
"meta_searches": totals["meta_searches"],
|
|
2446
|
+
"page_fetches": totals["page_fetches"],
|
|
2447
|
+
"rejected_calls": totals["rejected_calls"],
|
|
2448
|
+
"answered_queries": evidence["totals"]["answered_queries"],
|
|
2449
|
+
"provider_starts_recorded": totals["provider_starts_recorded"],
|
|
2450
|
+
"attempt_dataforseo_tasks": client.dataforseo_tasks,
|
|
2451
|
+
"attempt_search_calls": client.search_calls,
|
|
2452
|
+
"attempt_spyfu_calls": client.spyfu_calls,
|
|
2453
|
+
"attempt_meta_searches": client.meta_searches,
|
|
2454
|
+
"attempt_cache_hits": client.cache_hits,
|
|
2455
|
+
"report_commit_sha": persistence.commit_sha,
|
|
2456
|
+
"publication_mode": persistence.publication_mode,
|
|
2457
|
+
"research_context": _load_script("research-artifact-io.py", "pmr_input_io").context_binding(workspace, SECTION_ID),
|
|
2458
|
+
"error": None,
|
|
2459
|
+
}
|
|
2460
|
+
_write_atomic(runtime / "request.json", json.dumps(state, indent=1))
|
|
2461
|
+
return state
|
|
2462
|
+
|
|
2463
|
+
|
|
2464
|
+
def default_name(domain: str) -> str:
|
|
2465
|
+
stem = registrable_domain(domain).split(".")[0]
|
|
2466
|
+
return stem[:1].upper() + stem[1:] if stem else domain
|
|
2467
|
+
|
|
2468
|
+
|
|
2469
|
+
def main() -> int:
|
|
2470
|
+
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
2471
|
+
parser.add_argument("--workspace", type=Path, required=True)
|
|
2472
|
+
parser.add_argument("--context-file", default=None, help="Optional explicitly selected provider-safe research context.")
|
|
2473
|
+
parser.add_argument("--intake", default="", help=f"Optional typed operation inputs (default: {INTAKE_REL.as_posix()}).")
|
|
2474
|
+
parser.add_argument("--domains", default="", help="Competitor domains named by the founder; overrides the intake's list.")
|
|
2475
|
+
parser.add_argument("--page-domains", default="", help="Competitors the product page mentions; ordinary candidates. Overrides the intake's list.")
|
|
2476
|
+
parser.add_argument("--product-domain", default="", help="The product's own domain; never a candidate and never a search seed.")
|
|
2477
|
+
parser.add_argument("--seed-keywords", default="", help="1-3 category phrases of the offer; override the intake's offer card.")
|
|
2478
|
+
parser.add_argument("--price-floor-usd", type=float, default=None, help="Comparable price floor; overrides the intake's offer card.")
|
|
2479
|
+
parser.add_argument("--decisions", default="", help=f"Path to the tiering decisions file (default: {RUNTIME_REL.as_posix()}/{DECISIONS_FILENAME}).")
|
|
2480
|
+
parser.add_argument("--max-competitors", type=int, default=MAX_COMPETITORS)
|
|
2481
|
+
parser.add_argument("--location-code", type=int, default=None, help="DataForSEO location (default: the intake's first market, else US).")
|
|
2482
|
+
parser.add_argument("--language-code", default="", help="Provider query language (default: en); retain the saved language on resume.")
|
|
2483
|
+
parser.add_argument("--country", default="", help="Primary market as a two-letter code (default: the intake's first market, else US).")
|
|
2484
|
+
parser.add_argument("--eu-country", default="", help="EU country for the official Meta Ad Library check (default: the first EU market, else DE).")
|
|
2485
|
+
parser.add_argument("--max-dataforseo-tasks", type=int, default=MAX_DATAFORSEO_TASKS_PER_ATTEMPT)
|
|
2486
|
+
parser.add_argument("--max-search-calls", type=int, default=MAX_SEARCH_CALLS_PER_ATTEMPT)
|
|
2487
|
+
parser.add_argument("--max-spyfu-calls", type=int, default=MAX_SPYFU_CALLS_PER_ATTEMPT, help="SpyFu lookups per attempt (0 disables the source).")
|
|
2488
|
+
parser.add_argument("--max-meta-searches", type=int, default=MAX_META_SEARCHES_PER_ATTEMPT)
|
|
2489
|
+
parser.add_argument("--max-fetches", type=int, default=MAX_FETCHES_PER_ATTEMPT)
|
|
2490
|
+
args = parser.parse_args()
|
|
2491
|
+
|
|
2492
|
+
try:
|
|
2493
|
+
state = run(args.workspace, args)
|
|
2494
|
+
except (CompetitorSetError, ValueError) as exc:
|
|
2495
|
+
print(f"{DISPLAY_NAME}: {exc}", file=sys.stderr)
|
|
2496
|
+
return 1
|
|
2497
|
+
if state["status"] == "pending":
|
|
2498
|
+
print(
|
|
2499
|
+
f"{DISPLAY_NAME}: {state['reason']} — read {state['candidates_path']} (`missing`, `profile_required`, "
|
|
2500
|
+
f"`listicles_to_read`), write the enrichment files next to it (see the workflow doc), then rerun the same "
|
|
2501
|
+
"command; cached discovery adds no new spend.",
|
|
2502
|
+
file=sys.stderr,
|
|
2503
|
+
)
|
|
2504
|
+
return 2
|
|
2505
|
+
print(
|
|
2506
|
+
f"{DISPLAY_NAME}: completed — {len(state['domains'])} competitor(s), {state['dataforseo_calls']} DataForSEO "
|
|
2507
|
+
f"task(s), {state['spyfu_calls']} SpyFu lookup(s), {state['search_calls']} web search(es), evidence at "
|
|
2508
|
+
f"{state['evidence_path']}, report commit {(state['report_commit_sha'] or '')[:12] or 'n/a'}"
|
|
2509
|
+
)
|
|
2510
|
+
return 0
|
|
2511
|
+
|
|
2512
|
+
|
|
2513
|
+
if __name__ == "__main__":
|
|
2514
|
+
raise SystemExit(main())
|