@egbe-ai/opencode 0.0.0-stage → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +124 -2
- package/index.js +5 -0
- package/package.json +55 -4
- package/plugin.js +230 -0
- package/session-log.js +315 -0
- package/skills/.egbe-catalog/manifest.json +1 -0
- package/skills/egbe-os-analytics/SKILL.md +499 -0
- package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
- package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
- package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
- package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
- package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
- package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
- package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
- package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
- package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
- package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
- package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
- package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
- package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
- package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
- package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
- package/skills/egbe-os-db/SKILL.md +174 -0
- package/skills/egbe-os-deploy/SKILL.md +412 -0
- package/skills/egbe-os-design/SKILL.md +182 -0
- package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
- package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
- package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
- package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
- package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
- package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
- package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
- package/skills/egbe-os-destination-readiness/references/report.md +116 -0
- package/skills/egbe-os-domains/SKILL.md +189 -0
- package/skills/egbe-os-errors/SKILL.md +259 -0
- package/skills/egbe-os-git/SKILL.md +275 -0
- package/skills/egbe-os-git/references/issues.md +33 -0
- package/skills/egbe-os-google-ads/SKILL.md +163 -0
- package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
- package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
- package/skills/egbe-os-google-ads/references/conversions.md +153 -0
- package/skills/egbe-os-google-ads/references/delivery.md +128 -0
- package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
- package/skills/egbe-os-google-ads/references/reporting.md +105 -0
- package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
- package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
- package/skills/egbe-os-google-auth/SKILL.md +285 -0
- package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
- package/skills/egbe-os-images/SKILL.md +163 -0
- package/skills/egbe-os-images/references/stock-images.md +21 -0
- package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
- package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
- package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
- package/skills/egbe-os-llm/SKILL.md +160 -0
- package/skills/egbe-os-mail/SKILL.md +180 -0
- package/skills/egbe-os-me/SKILL.md +119 -0
- package/skills/egbe-os-meta-ads/SKILL.md +114 -0
- package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
- package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
- package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
- package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
- package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
- package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
- package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
- package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
- package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
- package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
- package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
- package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
- package/skills/egbe-os-notifications/SKILL.md +78 -0
- package/skills/egbe-os-openai-ads/SKILL.md +149 -0
- package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
- package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
- package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
- package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
- package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
- package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
- package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
- package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
- package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
- package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
- package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
- package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
- package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
- package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
- package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
- package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
- package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
- package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
- package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
- package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
- package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
- package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
- package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
- package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
- package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
- package/skills/egbe-os-payments/SKILL.md +215 -0
- package/skills/egbe-os-payments/agents/openai.yaml +4 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
- package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
- package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
- package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
- package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
- package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
- package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
- package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
- package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
- package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
- package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
- package/skills/egbe-os-playbook/SKILL.md +79 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
- package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
- package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
- package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
- package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
- package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
- package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
- package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
- package/skills/egbe-os-playbook/playbooks/product.md +67 -0
- package/skills/egbe-os-playbook/references/catalog.md +127 -0
- package/skills/egbe-os-playbook/references/changes.md +167 -0
- package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
- package/skills/egbe-os-product-auth/SKILL.md +151 -0
- package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
- package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
- package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
- package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
- package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
- package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
- package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
- package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
- package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
- package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
- package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
- package/skills/egbe-os-product-build/SKILL.md +111 -0
- package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-build/references/capabilities.md +25 -0
- package/skills/egbe-os-product-build/references/engineering.md +72 -0
- package/skills/egbe-os-product-build/references/execution.md +55 -0
- package/skills/egbe-os-product-build/references/foundations.md +93 -0
- package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
- package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
- package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
- package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
- package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
- package/skills/egbe-os-product-market-research/SKILL.md +159 -0
- package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
- package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
- package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
- package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
- package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
- package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
- package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
- package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
- package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
- package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
- package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
- package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
- package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
- package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
- package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
- package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
- package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
- package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
- package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
- package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
- package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
- package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
- package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
- package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
- package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
- package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
- package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
- package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
- package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
- package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
- package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
- package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
- package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
- package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
- package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
- package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
- package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
- package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
- package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
- package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
- package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
- package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
- package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
- package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
- package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
- package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
- package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
- package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
- package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
- package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
- package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
- package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
- package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
- package/skills/egbe-os-product-report/SKILL.md +150 -0
- package/skills/egbe-os-product-report/references/artifacts.md +84 -0
- package/skills/egbe-os-product-report/references/brief.md +34 -0
- package/skills/egbe-os-product-report/references/daily.md +76 -0
- package/skills/egbe-os-product-report/references/sources.md +68 -0
- package/skills/egbe-os-product-routines/SKILL.md +166 -0
- package/skills/egbe-os-product-routines/references/catalog.md +22 -0
- package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
- package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
- package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
- package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
- package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
- package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
- package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
- package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
- package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
- package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
- package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
- package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
- package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
- package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
- package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
- package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
- package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
- package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
- package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
- package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
- package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
- package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
- package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
- package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
- package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
- package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
- package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
- package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
- package/skills/egbe-os-report-ui/SKILL.md +133 -0
- package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
- package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
- package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
- package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
- package/skills/egbe-os-report-ui/references/components.md +209 -0
- package/skills/egbe-os-research/SKILL.md +125 -0
- package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
- package/skills/egbe-os-scrape/SKILL.md +160 -0
- package/skills/egbe-os-search/SKILL.md +182 -0
- package/skills/egbe-os-search/references/google-places.md +81 -0
- package/skills/egbe-os-support/SKILL.md +149 -0
- package/skills/egbe-os-telemetry/SKILL.md +344 -0
- package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
- package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
- package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
- package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
- package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
- package/skills/egbe-os-tools/SKILL.md +180 -0
- package/skills/egbe-os-tools/references/theona.md +26 -0
- package/skills/egbe-os-wallet/SKILL.md +181 -0
- package/skills/egbe-setup/EGBE.md +484 -0
- package/skills/egbe-setup/SKILL.md +116 -0
- package/skills/egbe-setup/references/project.md +80 -0
- package/state.js +37 -0
- package/update.js +361 -0
|
@@ -0,0 +1,1443 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Run the deterministic Keyword Economics section against OS DataForSEO tools.
|
|
3
|
+
|
|
4
|
+
Unlike the OS Research sections, this runner performs NO free-form web
|
|
5
|
+
research: every number comes from a metered DataForSEO call through the OS
|
|
6
|
+
tools passthrough (`POST {OS_URL}/tools/dataforseo/<op>`), and the section
|
|
7
|
+
markdown plus a machine-readable evidence file are rendered deterministically
|
|
8
|
+
from those responses. The executing agent supplies only candidate seed
|
|
9
|
+
keywords (validated by auction data — a hallucinated seed returns zero volume
|
|
10
|
+
and is dropped); it never writes section prose.
|
|
11
|
+
|
|
12
|
+
Pipeline: seeds -> expansion (keywords_for_keywords + search_volume) ->
|
|
13
|
+
commercial filter -> SERP on top heads (who ranks, who pays) -> competitor
|
|
14
|
+
confirmation -> ranked-keywords enrichment per competitor -> test-budget
|
|
15
|
+
simulation (ad_traffic_by_keywords) -> evidence.json + report section.
|
|
16
|
+
|
|
17
|
+
Every upstream response is cached under the runtime directory keyed by request
|
|
18
|
+
hash, so re-runs replay from cache instead of re-spending the shared deposit.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import sys as _artifact_sys
|
|
24
|
+
from pathlib import Path as _ArtifactPath
|
|
25
|
+
_artifact_sys.path.insert(0, str(_ArtifactPath(__file__).resolve().parent))
|
|
26
|
+
from egbe_artifacts import artifact_path as _artifact_path
|
|
27
|
+
|
|
28
|
+
import runpy as _egbe_runpy
|
|
29
|
+
from pathlib import Path as _EgbePath
|
|
30
|
+
_egbe_adapter = _EgbePath(__file__).with_name("connection_identity.py")
|
|
31
|
+
_connection_headers = _egbe_runpy.run_path(str(_egbe_adapter))["connection_headers"] if _egbe_adapter.is_file() else lambda _url: {}
|
|
32
|
+
|
|
33
|
+
import argparse
|
|
34
|
+
import hashlib
|
|
35
|
+
import importlib.util
|
|
36
|
+
import json
|
|
37
|
+
import math
|
|
38
|
+
import os
|
|
39
|
+
import re
|
|
40
|
+
import sys
|
|
41
|
+
import time
|
|
42
|
+
import urllib.error
|
|
43
|
+
import urllib.request
|
|
44
|
+
from dataclasses import dataclass, field
|
|
45
|
+
from datetime import date
|
|
46
|
+
from pathlib import Path
|
|
47
|
+
from types import ModuleType
|
|
48
|
+
from typing import Any, Callable
|
|
49
|
+
from urllib.parse import urlparse
|
|
50
|
+
|
|
51
|
+
SCHEMA_VERSION = 1
|
|
52
|
+
SECTION_ID = "keyword_economics"
|
|
53
|
+
SECTION_HEADING = "Keyword economics"
|
|
54
|
+
DISPLAY_NAME = "Keyword Economics"
|
|
55
|
+
|
|
56
|
+
DEFAULT_OS_URL = "https://os.hz.egbe.dev"
|
|
57
|
+
RUNTIME_REL = Path(_artifact_path("egbe-docs/research/market/work")) / SECTION_ID
|
|
58
|
+
EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/keyword-economics.json"))
|
|
59
|
+
REPORT_REL = Path(_artifact_path("egbe-docs/research/market/full-report.md"))
|
|
60
|
+
|
|
61
|
+
# Hard ceiling on metered upstream tasks per run (cache hits are free). The
|
|
62
|
+
# most expensive task is $0.09, so this bounds one run at ~$2.25 worst case;
|
|
63
|
+
# a normal run uses 8-16 tasks (~$0.50).
|
|
64
|
+
MAX_TASKS_PER_RUN = 25
|
|
65
|
+
|
|
66
|
+
# DataForSEO rate-limits google_ads live endpoints at 12 requests/minute and
|
|
67
|
+
# answers bursts with task status 50301 "Too many requests". Failed tasks are
|
|
68
|
+
# not billed, so a bounded backoff retry is safe; the window clears within a
|
|
69
|
+
# minute.
|
|
70
|
+
# Also used for structured internal errors explicitly confirmed as unpaid.
|
|
71
|
+
PROVIDER_MAX_RETRIES = 3
|
|
72
|
+
PROVIDER_BACKOFF_SECONDS = (20.0, 40.0, 60.0)
|
|
73
|
+
MAX_ERROR_RESPONSE_BYTES = 64 * 1024
|
|
74
|
+
|
|
75
|
+
MAX_SEED_KEYWORDS = 20
|
|
76
|
+
MAX_SEED_DOMAINS = 10
|
|
77
|
+
SERP_HEADS = 8
|
|
78
|
+
SERP_DEPTH = 20
|
|
79
|
+
CONFIRM_COMPETITORS = 4
|
|
80
|
+
RANKED_KEYWORDS_LIMIT = 200
|
|
81
|
+
BUDGET_SIM_KEYWORDS = 1000
|
|
82
|
+
TOP_TABLE_ROWS = 20
|
|
83
|
+
|
|
84
|
+
# Keywords containing one of these tokens carry commercial/search-to-buy
|
|
85
|
+
# intent and get priority in head selection and the report tables.
|
|
86
|
+
INTENT_TOKENS = {
|
|
87
|
+
"software", "tool", "tools", "app", "apps", "platform", "service",
|
|
88
|
+
"services", "pricing", "price", "cost", "best", "top", "alternative",
|
|
89
|
+
"alternatives", "vs", "generator", "builder", "maker", "for", "buy",
|
|
90
|
+
"online", "ai",
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
# Queries carrying these tokens signal free/education intent: real demand,
|
|
94
|
+
# but poor targets for PAID acquisition simulation, and their ad-free SERPs
|
|
95
|
+
# are partially self-explanatory.
|
|
96
|
+
FREE_EDU_TOKENS = {
|
|
97
|
+
"free", "teacher", "teachers", "teaching", "classroom", "student",
|
|
98
|
+
"students", "school", "kids",
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
# Lexical evidence, not a verdict that the query fits a particular offer.
|
|
102
|
+
# Google Ads compares these signals with the actual product and campaign goal.
|
|
103
|
+
VENDOR_TERMS = {"agency", "agencies", "consultant", "consultants", "consulting", "company", "companies", "services", "service", "hire"}
|
|
104
|
+
TOOL_TERMS = {"tool", "tools", "checker", "tracker", "software", "platform", "app", "apps"}
|
|
105
|
+
INFORMATION_TERMS = {"what", "how", "why", "meaning", "definition", "guide", "tutorial"}
|
|
106
|
+
JOB_TERMS = {"job", "jobs", "career", "careers", "salary", "salaries", "hiring"}
|
|
107
|
+
|
|
108
|
+
# SERP domains that are platforms/media, not product competitors.
|
|
109
|
+
NOISE_DOMAINS = {
|
|
110
|
+
"google.com", "youtube.com", "reddit.com", "facebook.com",
|
|
111
|
+
"instagram.com", "wikipedia.org", "amazon.com", "pinterest.com",
|
|
112
|
+
"linkedin.com", "twitter.com", "x.com", "tiktok.com", "quora.com",
|
|
113
|
+
"medium.com", "etsy.com", "ebay.com", "apple.com", "microsoft.com",
|
|
114
|
+
"yelp.com", "github.com", "canva.com", "adobe.com",
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
# Function words never anchor topicality: a competitor keyword sharing ONLY
|
|
118
|
+
# one of these with the seed market (e.g. "from", "list", "and") must not
|
|
119
|
+
# merge into the demand pool. Found live: "best flavor wings from wingstop"
|
|
120
|
+
# merged into a meal-planning market via "from".
|
|
121
|
+
FUNCTION_TOKENS = {
|
|
122
|
+
"the", "and", "from", "with", "your", "our", "you", "what", "when",
|
|
123
|
+
"where", "how", "why", "this", "that", "near", "name", "list", "get",
|
|
124
|
+
"make", "made", "does", "can", "will", "are", "was", "has", "have",
|
|
125
|
+
"free", "new", "without", "using", "use",
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
# Review/listing directories: strong demand evidence, but not competitors.
|
|
129
|
+
DIRECTORY_DOMAINS = {
|
|
130
|
+
"g2.com", "capterra.com", "getapp.com", "softwareadvice.com",
|
|
131
|
+
"trustpilot.com", "producthunt.com", "sourceforge.net", "slashdot.org",
|
|
132
|
+
"alternativeto.net", "trustradius.com",
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class KeywordEconomicsError(RuntimeError):
|
|
137
|
+
"""Terminal runner failure; runtime state is preserved for resume."""
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class DataForSeoError(KeywordEconomicsError):
|
|
141
|
+
"""Structured provider failure preserved through the OS tools envelope."""
|
|
142
|
+
|
|
143
|
+
def __init__(self, operation: str, http_status: int, detail: dict[str, Any]):
|
|
144
|
+
self.http_status = http_status
|
|
145
|
+
self.detail = detail
|
|
146
|
+
super().__init__(
|
|
147
|
+
f"OS tools/dataforseo/{operation} failed: HTTP {http_status} "
|
|
148
|
+
f"{detail.get('error')} status={detail.get('status_code')} "
|
|
149
|
+
f"task={detail.get('task_id') or 'unknown'}: "
|
|
150
|
+
f"{str(detail.get('message') or '')[:500]}"
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _is_rate_limited(exc: Exception) -> bool:
|
|
155
|
+
# 50301 also wraps invalid Google Ads input; retrying that cannot recover.
|
|
156
|
+
message = str(exc.detail.get("message") or "") if isinstance(exc, DataForSeoError) else str(exc)
|
|
157
|
+
if "invalid argument" in message.lower():
|
|
158
|
+
return False
|
|
159
|
+
if isinstance(exc, DataForSeoError):
|
|
160
|
+
return exc.detail.get("status_code") == 50301
|
|
161
|
+
text = str(exc)
|
|
162
|
+
return "50301" in text or "Too many requests" in text
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _is_unpaid_internal_error(exc: Exception) -> bool:
|
|
166
|
+
if not isinstance(exc, DataForSeoError):
|
|
167
|
+
return False
|
|
168
|
+
detail = exc.detail
|
|
169
|
+
return (
|
|
170
|
+
exc.http_status == 503
|
|
171
|
+
and detail.get("error") == "dataforseo_upstream_internal_error"
|
|
172
|
+
and detail.get("status_code") == 50000
|
|
173
|
+
and detail.get("retryable") is True
|
|
174
|
+
and all(type(detail.get(key)) in (int, float) and detail[key] == 0
|
|
175
|
+
for key in ("cost", "task_cost"))
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
# ── bundled-script loading (same pattern as the other section runners) ─────
|
|
180
|
+
|
|
181
|
+
def _load_script(filename: str, module_name: str) -> ModuleType:
|
|
182
|
+
path = Path(__file__).resolve().parent / filename
|
|
183
|
+
spec = importlib.util.spec_from_file_location(module_name, path)
|
|
184
|
+
if spec is None or spec.loader is None:
|
|
185
|
+
raise KeywordEconomicsError(f"cannot load bundled script {filename}")
|
|
186
|
+
module = importlib.util.module_from_spec(spec)
|
|
187
|
+
sys.modules[spec.name] = module
|
|
188
|
+
previous = sys.dont_write_bytecode
|
|
189
|
+
sys.dont_write_bytecode = True
|
|
190
|
+
try:
|
|
191
|
+
spec.loader.exec_module(module)
|
|
192
|
+
finally:
|
|
193
|
+
sys.dont_write_bytecode = previous
|
|
194
|
+
return module
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
# ── OS tools client with response cache and task budget ───────────────────
|
|
198
|
+
|
|
199
|
+
@dataclass
|
|
200
|
+
class ToolsClient:
|
|
201
|
+
os_url: str
|
|
202
|
+
api_key: str
|
|
203
|
+
cache_dir: Path
|
|
204
|
+
max_tasks: int = MAX_TASKS_PER_RUN
|
|
205
|
+
tasks_spent: int = 0
|
|
206
|
+
estimated_cost_usd: float = 0.0
|
|
207
|
+
cache_hits: int = 0
|
|
208
|
+
request: Callable[[str, dict[str, Any]], dict[str, Any]] | None = None
|
|
209
|
+
sleep: Callable[[float], None] | None = None
|
|
210
|
+
_ledger_entries: set[Path] = field(default_factory=set, init=False, repr=False)
|
|
211
|
+
|
|
212
|
+
def _http(self, operation: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|
213
|
+
url = f"{self.os_url.rstrip('/')}/tools/dataforseo/{operation}"
|
|
214
|
+
body = json.dumps(payload).encode("utf-8")
|
|
215
|
+
req = urllib.request.Request(
|
|
216
|
+
url,
|
|
217
|
+
data=body,
|
|
218
|
+
headers={
|
|
219
|
+
"Authorization": f"Bearer {self.api_key}",
|
|
220
|
+
"Content-Type": "application/json",
|
|
221
|
+
"Accept": "application/json",
|
|
222
|
+
},
|
|
223
|
+
method="POST",
|
|
224
|
+
)
|
|
225
|
+
try:
|
|
226
|
+
for _name, _value in _connection_headers(req.full_url).items():
|
|
227
|
+
req.add_unredirected_header(_name, _value)
|
|
228
|
+
with urllib.request.urlopen(req, timeout=180) as resp:
|
|
229
|
+
envelope = json.loads(resp.read().decode("utf-8"))
|
|
230
|
+
except urllib.error.HTTPError as exc:
|
|
231
|
+
raw = exc.read(MAX_ERROR_RESPONSE_BYTES + 1)
|
|
232
|
+
detail = raw.decode("utf-8", errors="replace")
|
|
233
|
+
if len(raw) <= MAX_ERROR_RESPONSE_BYTES:
|
|
234
|
+
try:
|
|
235
|
+
envelope = json.loads(detail)
|
|
236
|
+
except ValueError:
|
|
237
|
+
envelope = None
|
|
238
|
+
error = envelope.get("data") if isinstance(envelope, dict) else None
|
|
239
|
+
if isinstance(error, dict) and str(error.get("error", "")).startswith("dataforseo_"):
|
|
240
|
+
raise DataForSeoError(operation, exc.code, error) from exc
|
|
241
|
+
raise KeywordEconomicsError(
|
|
242
|
+
f"OS tools/dataforseo/{operation} failed: HTTP {exc.code} {detail[:500]}"
|
|
243
|
+
) from exc
|
|
244
|
+
except urllib.error.URLError as exc:
|
|
245
|
+
raise KeywordEconomicsError(
|
|
246
|
+
f"OS tools/dataforseo/{operation} unreachable: {exc.reason}"
|
|
247
|
+
) from exc
|
|
248
|
+
data = envelope.get("data")
|
|
249
|
+
if envelope.get("code") not in (None, 200) or not isinstance(data, dict):
|
|
250
|
+
raise KeywordEconomicsError(
|
|
251
|
+
f"OS tools/dataforseo/{operation} returned no data: "
|
|
252
|
+
f"{str(envelope)[:300]}"
|
|
253
|
+
)
|
|
254
|
+
return data
|
|
255
|
+
|
|
256
|
+
def call(self, operation: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|
257
|
+
key_raw = json.dumps({"op": operation, "payload": payload}, sort_keys=True)
|
|
258
|
+
digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
|
|
259
|
+
cache_file = self.cache_dir / f"{digest}.json"
|
|
260
|
+
if cache_file.is_file():
|
|
261
|
+
cached = json.loads(cache_file.read_text(encoding="utf-8"))
|
|
262
|
+
self.cache_hits += 1
|
|
263
|
+
self._ledger_entries.add(cache_file)
|
|
264
|
+
return cached["response"]
|
|
265
|
+
|
|
266
|
+
if self.tasks_spent >= self.max_tasks:
|
|
267
|
+
raise KeywordEconomicsError(
|
|
268
|
+
f"task budget exhausted ({self.max_tasks} metered calls); "
|
|
269
|
+
"narrow the seeds or raise --max-tasks"
|
|
270
|
+
)
|
|
271
|
+
transport = self.request or self._http
|
|
272
|
+
attempt = 0
|
|
273
|
+
while True:
|
|
274
|
+
try:
|
|
275
|
+
response = transport(operation, payload)
|
|
276
|
+
break
|
|
277
|
+
except KeywordEconomicsError as exc:
|
|
278
|
+
if not (_is_rate_limited(exc) or _is_unpaid_internal_error(exc)):
|
|
279
|
+
raise
|
|
280
|
+
if attempt >= PROVIDER_MAX_RETRIES:
|
|
281
|
+
raise KeywordEconomicsError(
|
|
282
|
+
f"{exc}; automatic retries exhausted ({attempt}). "
|
|
283
|
+
"Research is incomplete. Successful responses remain cached; "
|
|
284
|
+
"preserve the workspace and report the blocked section. "
|
|
285
|
+
"A later recovery attempt can rerun the same command from cache."
|
|
286
|
+
) from exc
|
|
287
|
+
delay = PROVIDER_BACKOFF_SECONDS[attempt]
|
|
288
|
+
print(
|
|
289
|
+
f"{DISPLAY_NAME}: {exc}; retry {attempt + 1}/{PROVIDER_MAX_RETRIES} "
|
|
290
|
+
f"in {delay:g}s with unchanged parameters.", file=sys.stderr,
|
|
291
|
+
)
|
|
292
|
+
(self.sleep or time.sleep)(delay)
|
|
293
|
+
attempt += 1
|
|
294
|
+
self.tasks_spent += 1
|
|
295
|
+
cost = response.get("cost")
|
|
296
|
+
if isinstance(cost, (int, float)):
|
|
297
|
+
self.estimated_cost_usd += float(cost)
|
|
298
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
299
|
+
_write_atomic(cache_file,
|
|
300
|
+
json.dumps(
|
|
301
|
+
{
|
|
302
|
+
"operation": operation,
|
|
303
|
+
"payload": payload,
|
|
304
|
+
"response": response,
|
|
305
|
+
"retrieved_on": date.today().isoformat(),
|
|
306
|
+
}
|
|
307
|
+
),
|
|
308
|
+
)
|
|
309
|
+
self._ledger_entries.add(cache_file)
|
|
310
|
+
return response
|
|
311
|
+
|
|
312
|
+
def ledger_totals(self) -> dict[str, Any]:
|
|
313
|
+
"""Durable spend provenance for calls used by this pipeline execution.
|
|
314
|
+
|
|
315
|
+
Every used cache entry is one metered upstream call that actually
|
|
316
|
+
happened, regardless of which attempt made it. A cache-only replay
|
|
317
|
+
therefore reports the ORIGINAL spend without pulling unrelated entries
|
|
318
|
+
from older runs with different inputs into the current section totals.
|
|
319
|
+
"""
|
|
320
|
+
tasks = 0
|
|
321
|
+
cost = 0.0
|
|
322
|
+
dates: list[str] = []
|
|
323
|
+
if self.cache_dir.is_dir():
|
|
324
|
+
for entry_path in sorted(self._ledger_entries):
|
|
325
|
+
try:
|
|
326
|
+
entry = json.loads(entry_path.read_text(encoding="utf-8"))
|
|
327
|
+
except (OSError, json.JSONDecodeError):
|
|
328
|
+
continue
|
|
329
|
+
tasks += 1
|
|
330
|
+
entry_cost = (entry.get("response") or {}).get("cost")
|
|
331
|
+
if isinstance(entry_cost, (int, float)):
|
|
332
|
+
cost += float(entry_cost)
|
|
333
|
+
if isinstance(entry.get("retrieved_on"), str):
|
|
334
|
+
dates.append(entry["retrieved_on"])
|
|
335
|
+
return {
|
|
336
|
+
"tasks_spent": tasks,
|
|
337
|
+
"estimated_cost_usd": round(cost, 4),
|
|
338
|
+
"first_retrieved_on": min(dates) if dates else date.today().isoformat(),
|
|
339
|
+
"last_retrieved_on": max(dates) if dates else date.today().isoformat(),
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
# ── deterministic pipeline steps ───────────────────────────────────────────
|
|
344
|
+
|
|
345
|
+
def parse_terms(raw: str, cap: int) -> list[str]:
|
|
346
|
+
"""Split a comma/semicolon/newline separated list, dedupe, cap."""
|
|
347
|
+
seen: dict[str, None] = {}
|
|
348
|
+
for part in re.split(r"[,;\n]", raw or ""):
|
|
349
|
+
term = " ".join(part.split()).strip().lower()
|
|
350
|
+
if term and term not in seen:
|
|
351
|
+
seen[term] = None
|
|
352
|
+
return list(seen)[:cap]
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
# Second-level labels that make a public suffix under a country code
|
|
356
|
+
# (example.co.uk, example.com.au, example.edu.in, example.or.kr): the
|
|
357
|
+
# registrable name is the label before them.
|
|
358
|
+
SECOND_LEVEL_SUFFIX_LABELS = frozenset({
|
|
359
|
+
"co", "com", "org", "net", "gov", "ac", "edu", "or", "ne", "go", "gob", "nic", "ltd", "plc", "sch", "res", "gen",
|
|
360
|
+
"ind", "firm", "nom", "mil", "int", "biz", "info",
|
|
361
|
+
})
|
|
362
|
+
# Course and creator hosts that give every seller a subdomain: the subdomain
|
|
363
|
+
# is the seller (dair-ai.thinkific.com is DAIR.AI's school, not Thinkific's),
|
|
364
|
+
# so it is a domain of its own. The host's service subdomains still mean the host.
|
|
365
|
+
HOSTED_SELLER_PLATFORMS = frozenset({
|
|
366
|
+
"thinkific.com", "teachable.com", "kajabi.com", "mykajabi.com", "podia.com", "learnworlds.com", "gumroad.com",
|
|
367
|
+
"substack.com", "systeme.io", "circle.so", "mn.co", "github.io", "notion.site", "webflow.io", "framer.website",
|
|
368
|
+
"carrd.co", "wixsite.com", "wordpress.com", "blogspot.com", "hashnode.dev",
|
|
369
|
+
})
|
|
370
|
+
HOST_SERVICE_LABELS = frozenset({"www", "app", "help", "support", "docs", "status", "api", "community", "blog"})
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def registrable_domain(value: str) -> str:
|
|
374
|
+
"""Normalize a URL or hostname to its registrable domain (best effort);
|
|
375
|
+
a seller's subdomain on a course or creator host is kept whole."""
|
|
376
|
+
host = (value or "").strip().lower()
|
|
377
|
+
if "//" in host:
|
|
378
|
+
host = urlparse(host).hostname or ""
|
|
379
|
+
host = host.split("/", 1)[0].removeprefix("www.")
|
|
380
|
+
parts = [p for p in host.split(".") if p]
|
|
381
|
+
if len(parts) <= 2:
|
|
382
|
+
return ".".join(parts)
|
|
383
|
+
# Collapse deep subdomains; keep two-level public suffixes intact.
|
|
384
|
+
keep = 3 if parts[-2] in SECOND_LEVEL_SUFFIX_LABELS and len(parts[-1]) == 2 else 2
|
|
385
|
+
if ".".join(parts[-keep:]) in HOSTED_SELLER_PLATFORMS and len(parts) > keep and parts[-keep - 1] not in HOST_SERVICE_LABELS:
|
|
386
|
+
keep += 1
|
|
387
|
+
return ".".join(parts[-keep:])
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def hosted_seller(domain: str) -> bool:
|
|
391
|
+
"""A seller's own subdomain on a course or creator host."""
|
|
392
|
+
return domain.count(".") >= 2 and domain.split(".", 1)[1] in HOSTED_SELLER_PLATFORMS
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def has_intent(keyword: str) -> bool:
|
|
396
|
+
return any(token in keyword.split() for token in INTENT_TOKENS)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def content_tokens(keywords: list[str]) -> set[str]:
|
|
400
|
+
"""Category-bearing tokens of the seed market: everything except generic
|
|
401
|
+
intent modifiers and short function words. These anchor the topical filter
|
|
402
|
+
that keeps competitor footprints from contaminating the demand model."""
|
|
403
|
+
tokens: set[str] = set()
|
|
404
|
+
for keyword in keywords:
|
|
405
|
+
for token in keyword.split():
|
|
406
|
+
if len(token) >= 3 and token not in INTENT_TOKENS and token not in FUNCTION_TOKENS:
|
|
407
|
+
tokens.add(token)
|
|
408
|
+
return tokens
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def brand_token(domain: str) -> str:
|
|
412
|
+
"""clickup.com -> clickup — the brand token used to keep competitor-brand
|
|
413
|
+
queries out of the budget simulation."""
|
|
414
|
+
stem = registrable_domain(domain).split(".")[0]
|
|
415
|
+
return re.sub(r"[^a-z0-9]", "", stem)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def has_free_edu_intent(keyword: str) -> bool:
|
|
419
|
+
"""Note/flag marker: free OR education audience terms."""
|
|
420
|
+
return any(token in FREE_EDU_TOKENS for token in keyword.split())
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def has_free_intent(keyword: str) -> bool:
|
|
424
|
+
"""Simulation gate: ONLY explicit 'free' intent — education/audience
|
|
425
|
+
terms mark verticals ('school management software' is commercial) and
|
|
426
|
+
must not exclude a query from paid simulation."""
|
|
427
|
+
return "free" in keyword.split()
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def query_intent_signal(keyword: str) -> str:
|
|
431
|
+
"""Return only a defensible lexical signal; unknown is a valid outcome."""
|
|
432
|
+
tokens = set(keyword.lower().split())
|
|
433
|
+
if tokens & JOB_TERMS:
|
|
434
|
+
return "job_seeking"
|
|
435
|
+
if "free" in tokens:
|
|
436
|
+
return "free_intent"
|
|
437
|
+
if tokens & INFORMATION_TERMS:
|
|
438
|
+
return "informational"
|
|
439
|
+
if tokens & VENDOR_TERMS:
|
|
440
|
+
return "vendor_seeking"
|
|
441
|
+
if tokens & TOOL_TERMS:
|
|
442
|
+
return "tool_seeking"
|
|
443
|
+
return "unknown"
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def _tokens_match(a: str, b: str) -> bool:
|
|
447
|
+
"""Exact match, or simple plural-suffix equality (+s/+es) for stems of
|
|
448
|
+
6+ characters: 'whiteboards' matches 'whiteboard', but 'canvas' does NOT
|
|
449
|
+
match 'canva' (short stems collide — the live canva/canvas adversarial
|
|
450
|
+
pair). Deliberately NOT a stemmer: irregular morphology ('categories')
|
|
451
|
+
is out of scope and documented."""
|
|
452
|
+
if a == b:
|
|
453
|
+
return True
|
|
454
|
+
shorter, longer = (a, b) if len(a) <= len(b) else (b, a)
|
|
455
|
+
if len(shorter) < 6:
|
|
456
|
+
return False
|
|
457
|
+
return longer == shorter + "s" or longer == shorter + "es"
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def is_topical(keyword: str, market_tokens: set[str]) -> bool:
|
|
461
|
+
"""A keyword joins the market pool only when it shares at least one
|
|
462
|
+
category-bearing token (plural-tolerant) with the seed anchors. 'ai
|
|
463
|
+
furniture staging' shares 'staging'; a broad competitor's 'best crm app'
|
|
464
|
+
shares nothing and stays footprint-only evidence."""
|
|
465
|
+
for token in keyword.split():
|
|
466
|
+
if len(token) < 3 or token in INTENT_TOKENS or token in FUNCTION_TOKENS:
|
|
467
|
+
continue
|
|
468
|
+
if any(_tokens_match(token, anchor) for anchor in market_tokens):
|
|
469
|
+
return True
|
|
470
|
+
return False
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def build_foreign_brand_veto(
|
|
474
|
+
rejected_keywords: list[str], seed_tokens: set[str]
|
|
475
|
+
) -> set[str]:
|
|
476
|
+
"""Bounded foreign-brand detector using POSITION as brand evidence, not
|
|
477
|
+
bare frequency: brands lead their phrases ('canva ai', 'canva ai
|
|
478
|
+
generator'), while recurring category nouns trail ('brainstorm
|
|
479
|
+
template', 'meeting template' must not veto 'template'). A non-seed
|
|
480
|
+
content token LEADING 2+ drift-rejected phrases is treated as a foreign
|
|
481
|
+
brand; keywords that slipped past the first pass on a weak shared token
|
|
482
|
+
('canva ai text' via seed token 'text') are removed only when the veto
|
|
483
|
+
token leads them too."""
|
|
484
|
+
counts: dict[str, int] = {}
|
|
485
|
+
for keyword in rejected_keywords:
|
|
486
|
+
tokens = keyword.split()
|
|
487
|
+
if not tokens:
|
|
488
|
+
continue
|
|
489
|
+
first = tokens[0]
|
|
490
|
+
if len(first) < 3 or first in INTENT_TOKENS or first in FUNCTION_TOKENS:
|
|
491
|
+
continue
|
|
492
|
+
if any(_tokens_match(first, anchor) for anchor in seed_tokens):
|
|
493
|
+
continue
|
|
494
|
+
counts[first] = counts.get(first, 0) + 1
|
|
495
|
+
return {token for token, count in counts.items() if count >= 2}
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def normalize_keyword_row(raw: dict[str, Any], source: str) -> dict[str, Any] | None:
|
|
499
|
+
keyword = " ".join(str(raw.get("keyword") or "").split()).lower()
|
|
500
|
+
if not keyword:
|
|
501
|
+
return None
|
|
502
|
+
return {
|
|
503
|
+
"type": "keyword",
|
|
504
|
+
"keyword": keyword,
|
|
505
|
+
"intent_signal": query_intent_signal(keyword),
|
|
506
|
+
"free_edu_intent": has_free_edu_intent(keyword),
|
|
507
|
+
"search_volume": raw.get("search_volume"),
|
|
508
|
+
"cpc": raw.get("cpc"),
|
|
509
|
+
"cpc_low": raw.get("low_top_of_page_bid"),
|
|
510
|
+
"cpc_high": raw.get("high_top_of_page_bid"),
|
|
511
|
+
"competition": raw.get("competition"),
|
|
512
|
+
"sources": [source],
|
|
513
|
+
"tier": "A",
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def normalize_ranked_row(item: dict[str, Any], domain: str) -> dict[str, Any] | None:
|
|
518
|
+
keyword_data = item.get("keyword_data") or {}
|
|
519
|
+
info = keyword_data.get("keyword_info") or {}
|
|
520
|
+
serp_item = (item.get("ranked_serp_element") or {}).get("serp_item") or {}
|
|
521
|
+
row = normalize_keyword_row(
|
|
522
|
+
{
|
|
523
|
+
"keyword": keyword_data.get("keyword"),
|
|
524
|
+
"search_volume": info.get("search_volume"),
|
|
525
|
+
"cpc": info.get("cpc"),
|
|
526
|
+
"low_top_of_page_bid": info.get("low_top_of_page_bid"),
|
|
527
|
+
"high_top_of_page_bid": info.get("high_top_of_page_bid"),
|
|
528
|
+
"competition": info.get("competition_level") or info.get("competition"),
|
|
529
|
+
},
|
|
530
|
+
source=f"ranked_keywords:{domain}",
|
|
531
|
+
)
|
|
532
|
+
if row is None:
|
|
533
|
+
return None
|
|
534
|
+
if serp_item.get("type") == "paid":
|
|
535
|
+
# The competitor BUYS this query — the strongest channel evidence here.
|
|
536
|
+
row["paid_by"] = [domain]
|
|
537
|
+
return row
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def merge_keyword_rows(*row_lists: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
541
|
+
merged: dict[str, dict[str, Any]] = {}
|
|
542
|
+
for rows in row_lists:
|
|
543
|
+
for row in rows:
|
|
544
|
+
if row is None:
|
|
545
|
+
continue
|
|
546
|
+
existing = merged.get(row["keyword"])
|
|
547
|
+
if existing is None:
|
|
548
|
+
merged[row["keyword"]] = {**row, "sources": list(row["sources"])}
|
|
549
|
+
continue
|
|
550
|
+
for source in row["sources"]:
|
|
551
|
+
if source not in existing["sources"]:
|
|
552
|
+
existing["sources"].append(source)
|
|
553
|
+
for paid_domain in row.get("paid_by", []):
|
|
554
|
+
existing.setdefault("paid_by", [])
|
|
555
|
+
if paid_domain not in existing["paid_by"]:
|
|
556
|
+
existing["paid_by"].append(paid_domain)
|
|
557
|
+
for key in ("search_volume", "cpc", "cpc_low", "cpc_high", "competition"):
|
|
558
|
+
if existing.get(key) is None and row.get(key) is not None:
|
|
559
|
+
existing[key] = row[key]
|
|
560
|
+
return list(merged.values())
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
def rank_keywords(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
564
|
+
def sort_key(row: dict[str, Any]):
|
|
565
|
+
volume = row.get("search_volume") or 0
|
|
566
|
+
return (1 if has_intent(row["keyword"]) else 0, volume)
|
|
567
|
+
|
|
568
|
+
return sorted(
|
|
569
|
+
(r for r in rows if (r.get("search_volume") or 0) > 0),
|
|
570
|
+
key=sort_key,
|
|
571
|
+
reverse=True,
|
|
572
|
+
)
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def pick_serp_heads(ranked: list[dict[str, Any]], count: int = SERP_HEADS) -> list[str]:
|
|
576
|
+
heads: list[str] = []
|
|
577
|
+
seen_first_tokens: set[str] = set()
|
|
578
|
+
# First pass prefers lexical diversity so eight heads don't all probe the
|
|
579
|
+
# same phrase family; second pass fills remaining slots by rank.
|
|
580
|
+
for row in ranked:
|
|
581
|
+
first = row["keyword"].split()[0]
|
|
582
|
+
if first not in seen_first_tokens:
|
|
583
|
+
heads.append(row["keyword"])
|
|
584
|
+
seen_first_tokens.add(first)
|
|
585
|
+
if len(heads) >= count:
|
|
586
|
+
return heads
|
|
587
|
+
for row in ranked:
|
|
588
|
+
if row["keyword"] not in heads:
|
|
589
|
+
heads.append(row["keyword"])
|
|
590
|
+
if len(heads) >= count:
|
|
591
|
+
break
|
|
592
|
+
return heads
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def extract_serp_observation(keyword: str, serp_result: list[dict[str, Any]]) -> dict[str, Any]:
|
|
596
|
+
entry = (serp_result or [{}])[0]
|
|
597
|
+
available = bool(serp_result) and isinstance(entry.get("items"), list)
|
|
598
|
+
items = entry.get("items") or []
|
|
599
|
+
paid: list[str] = []
|
|
600
|
+
organic: list[str] = []
|
|
601
|
+
for item in items:
|
|
602
|
+
domain = registrable_domain(item.get("domain") or item.get("url") or "")
|
|
603
|
+
if not domain:
|
|
604
|
+
continue
|
|
605
|
+
if item.get("type") == "paid" and domain not in paid:
|
|
606
|
+
paid.append(domain)
|
|
607
|
+
elif item.get("type") == "organic" and len(organic) < 10 and domain not in organic:
|
|
608
|
+
organic.append(domain)
|
|
609
|
+
return {
|
|
610
|
+
"type": "serp_observation",
|
|
611
|
+
"keyword": keyword,
|
|
612
|
+
"intent_signal": query_intent_signal(keyword),
|
|
613
|
+
"paid_advertisers": paid,
|
|
614
|
+
"ads_count": len(paid) if available else None,
|
|
615
|
+
"response_status": "available" if available else "unavailable",
|
|
616
|
+
"organic_top": organic,
|
|
617
|
+
"se_results_count": entry.get("se_results_count"),
|
|
618
|
+
"tier": "A",
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def classify_domain(domain: str) -> str:
|
|
623
|
+
if domain in NOISE_DOMAINS:
|
|
624
|
+
return "noise"
|
|
625
|
+
if domain in DIRECTORY_DOMAINS:
|
|
626
|
+
return "directory"
|
|
627
|
+
return "candidate"
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def tally_competitors(
|
|
631
|
+
observations: list[dict[str, Any]],
|
|
632
|
+
seed_domains: list[str],
|
|
633
|
+
keyword_volumes: dict[str, int] | None = None,
|
|
634
|
+
) -> list[dict[str, Any]]:
|
|
635
|
+
"""Score competitors by VOLUME-WEIGHTED SERP presence: one appearance on
|
|
636
|
+
a 165k/mo head outweighs several appearances on tiny queries (observed
|
|
637
|
+
live: education micro-sites outranked figma.com on raw appearance
|
|
638
|
+
counts). Weight = log10(head volume), ads-block presence x3."""
|
|
639
|
+
volumes = keyword_volumes or {}
|
|
640
|
+
tally: dict[str, dict[str, Any]] = {}
|
|
641
|
+
|
|
642
|
+
def bump(domain: str, via: str, weight: float) -> None:
|
|
643
|
+
kind = classify_domain(domain)
|
|
644
|
+
if kind == "noise":
|
|
645
|
+
return
|
|
646
|
+
row = tally.setdefault(
|
|
647
|
+
domain,
|
|
648
|
+
{
|
|
649
|
+
"type": "competitor_domain",
|
|
650
|
+
"domain": domain,
|
|
651
|
+
"kind": kind,
|
|
652
|
+
"found_via": [],
|
|
653
|
+
"serp_paid_appearances": 0,
|
|
654
|
+
"serp_organic_appearances": 0,
|
|
655
|
+
"serp_queries": [],
|
|
656
|
+
"score": 0,
|
|
657
|
+
"tier": "A",
|
|
658
|
+
},
|
|
659
|
+
)
|
|
660
|
+
if via not in row["found_via"]:
|
|
661
|
+
row["found_via"].append(via)
|
|
662
|
+
if via == "serp_ads":
|
|
663
|
+
row["serp_paid_appearances"] += 1
|
|
664
|
+
elif via == "serp_organic":
|
|
665
|
+
row["serp_organic_appearances"] += 1
|
|
666
|
+
row["score"] = round(row["score"] + weight, 2)
|
|
667
|
+
|
|
668
|
+
def head_weight(keyword: str) -> float:
|
|
669
|
+
return math.log10(max(volumes.get(keyword, 0), 10))
|
|
670
|
+
|
|
671
|
+
for domain in seed_domains:
|
|
672
|
+
normalized = registrable_domain(domain)
|
|
673
|
+
if normalized:
|
|
674
|
+
bump(normalized, "seed", 2.0)
|
|
675
|
+
for obs in observations:
|
|
676
|
+
weight = head_weight(obs["keyword"])
|
|
677
|
+
for domain in obs["paid_advertisers"]:
|
|
678
|
+
# A paid appearance is evidence, not yet category confirmation.
|
|
679
|
+
bump(domain, "serp_ads", 3.0 * weight)
|
|
680
|
+
row = tally.get(domain)
|
|
681
|
+
if row is not None:
|
|
682
|
+
row["serp_queries"].append({"keyword": obs["keyword"], "via": "serp_ads"})
|
|
683
|
+
for domain in obs["organic_top"]:
|
|
684
|
+
bump(domain, "serp_organic", weight)
|
|
685
|
+
row = tally.get(domain)
|
|
686
|
+
if row is not None:
|
|
687
|
+
row["serp_queries"].append({"keyword": obs["keyword"], "via": "serp_organic"})
|
|
688
|
+
|
|
689
|
+
return sorted(tally.values(), key=lambda r: r["score"], reverse=True)
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
def confirmed_competitors(
|
|
693
|
+
candidates: list[dict[str, Any]],
|
|
694
|
+
seed_domains: list[str] | None = None,
|
|
695
|
+
count: int = CONFIRM_COMPETITORS,
|
|
696
|
+
seed_keywords: list[str] | None = None,
|
|
697
|
+
) -> list[str]:
|
|
698
|
+
"""Founder-named seed domains are reserved FIRST — volume-weighted
|
|
699
|
+
discovery must not evict an explicitly supplied competitor from the
|
|
700
|
+
footprint pulls."""
|
|
701
|
+
ordered: list[str] = []
|
|
702
|
+
for domain in seed_domains or []:
|
|
703
|
+
normalized = registrable_domain(domain)
|
|
704
|
+
if normalized and normalized not in ordered and classify_domain(normalized) == "candidate":
|
|
705
|
+
ordered.append(normalized)
|
|
706
|
+
for row in candidates:
|
|
707
|
+
if len(ordered) >= count:
|
|
708
|
+
break
|
|
709
|
+
if (
|
|
710
|
+
row["kind"] == "candidate"
|
|
711
|
+
and row["domain"] not in ordered
|
|
712
|
+
and (seed_keywords is None or competitor_category_evidence(row, seed_keywords))
|
|
713
|
+
):
|
|
714
|
+
ordered.append(row["domain"])
|
|
715
|
+
return ordered[:count]
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
def competitor_category_evidence(row: dict[str, Any], seeds: list[str]) -> bool:
|
|
719
|
+
"""Require evidence beyond an ambiguous one-word category SERP.
|
|
720
|
+
|
|
721
|
+
One paid appearance on a category-qualified query, or two independent
|
|
722
|
+
organic appearances on such queries, is enough. Generic query modifiers
|
|
723
|
+
such as 'agency' are not category context. 'AI' can complete a two-token
|
|
724
|
+
context only alongside a real seed anchor (e.g. 'AI visibility').
|
|
725
|
+
"""
|
|
726
|
+
anchors = content_tokens(seeds) - VENDOR_TERMS
|
|
727
|
+
relevant: dict[str, set[str]] = {}
|
|
728
|
+
for match in row.get("serp_queries") or []:
|
|
729
|
+
keyword = str(match.get("keyword") or "")
|
|
730
|
+
tokens = set(keyword.split())
|
|
731
|
+
category = tokens & anchors
|
|
732
|
+
contextual = len(category) >= 2 or ("ai" in tokens and bool(category))
|
|
733
|
+
if contextual:
|
|
734
|
+
relevant.setdefault(keyword, set()).add(str(match.get("via") or ""))
|
|
735
|
+
return any("serp_ads" in via for via in relevant.values()) or len(relevant) >= 2
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def median(values: list[float]) -> float | None:
|
|
739
|
+
cleaned = sorted(v for v in values if isinstance(v, (int, float)) and v > 0)
|
|
740
|
+
if not cleaned:
|
|
741
|
+
return None
|
|
742
|
+
middle = len(cleaned) // 2
|
|
743
|
+
if len(cleaned) % 2:
|
|
744
|
+
return float(cleaned[middle])
|
|
745
|
+
return (cleaned[middle - 1] + cleaned[middle]) / 2.0
|
|
746
|
+
|
|
747
|
+
|
|
748
|
+
def build_budget_rows(
|
|
749
|
+
sim_keywords: list[str],
|
|
750
|
+
bid: float,
|
|
751
|
+
ad_traffic_result: list[dict[str, Any]],
|
|
752
|
+
test_budget_usd: float,
|
|
753
|
+
) -> list[dict[str, Any]]:
|
|
754
|
+
rows: list[dict[str, Any]] = []
|
|
755
|
+
for raw in ad_traffic_result or []:
|
|
756
|
+
clicks = raw.get("clicks")
|
|
757
|
+
cost = raw.get("cost")
|
|
758
|
+
keyword = raw.get("keyword")
|
|
759
|
+
scale = None
|
|
760
|
+
clicks_for_budget = None
|
|
761
|
+
if test_budget_usd is not None and isinstance(clicks, (int, float)) and isinstance(cost, (int, float)) and cost > 0:
|
|
762
|
+
scale = test_budget_usd / cost
|
|
763
|
+
clicks_for_budget = round(clicks * min(scale, 1.0), 1) if scale < 1 else round(clicks, 1)
|
|
764
|
+
elif test_budget_usd is not None and clicks == 0 and cost == 0:
|
|
765
|
+
scale = 1.0
|
|
766
|
+
clicks_for_budget = 0
|
|
767
|
+
rows.append(
|
|
768
|
+
{
|
|
769
|
+
"type": "budget_simulation",
|
|
770
|
+
"keyword": keyword or "(all simulated keywords)",
|
|
771
|
+
"simulated_keywords": sim_keywords if keyword is None else None,
|
|
772
|
+
"bid": bid,
|
|
773
|
+
"match": raw.get("match"),
|
|
774
|
+
"average_cpc": raw.get("average_cpc"),
|
|
775
|
+
"monthly_clicks_at_bid": clicks,
|
|
776
|
+
"monthly_cost_at_bid": cost,
|
|
777
|
+
"test_budget_usd": test_budget_usd,
|
|
778
|
+
"est_clicks_within_test_budget": clicks_for_budget,
|
|
779
|
+
"budget_covers_full_month": scale >= 1 if scale is not None else None,
|
|
780
|
+
"tier": "A",
|
|
781
|
+
}
|
|
782
|
+
)
|
|
783
|
+
return rows
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
# ── rendering ──────────────────────────────────────────────────────────────
|
|
787
|
+
|
|
788
|
+
def _fmt(value: Any, money: bool = False) -> str:
|
|
789
|
+
if value is None:
|
|
790
|
+
return "—"
|
|
791
|
+
if isinstance(value, float):
|
|
792
|
+
return f"${value:,.2f}" if money else f"{value:,.1f}".rstrip("0").rstrip(".")
|
|
793
|
+
if isinstance(value, int):
|
|
794
|
+
return f"${value:,}" if money else f"{value:,}"
|
|
795
|
+
return str(value)
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def render_section(
|
|
799
|
+
*,
|
|
800
|
+
locale: dict[str, Any],
|
|
801
|
+
seeds: list[str],
|
|
802
|
+
seed_rows_alive: int,
|
|
803
|
+
keyword_rows: list[dict[str, Any]],
|
|
804
|
+
footprint_rows: list[dict[str, Any]],
|
|
805
|
+
observations: list[dict[str, Any]],
|
|
806
|
+
competitor_rows: list[dict[str, Any]],
|
|
807
|
+
confirmed: list[str],
|
|
808
|
+
budget_rows: list[dict[str, Any]],
|
|
809
|
+
overlap_domains: list[str],
|
|
810
|
+
totals: dict[str, Any],
|
|
811
|
+
test_budget_usd: float,
|
|
812
|
+
retrieved_on: str,
|
|
813
|
+
expansion_rejected_count: int = 0,
|
|
814
|
+
expansion_veto_rejected_count: int = 0,
|
|
815
|
+
simulation_omitted_reason: str | None = None,
|
|
816
|
+
simulation_coverage: dict | None = None,
|
|
817
|
+
) -> str:
|
|
818
|
+
ranked = rank_keywords(keyword_rows)
|
|
819
|
+
top_rows = ranked[:TOP_TABLE_ROWS]
|
|
820
|
+
lines: list[str] = []
|
|
821
|
+
add = lines.append
|
|
822
|
+
|
|
823
|
+
add(
|
|
824
|
+
f"Primary auction data retrieved {retrieved_on} via DataForSEO "
|
|
825
|
+
f"(Google Ads / Labs / SERP endpoints) for location code "
|
|
826
|
+
f"{locale['location_code']}, language `{locale['language_code']}`. "
|
|
827
|
+
f"Seeds supplied by the workflow: {len(seeds)}; seeds surviving "
|
|
828
|
+
f"volume validation: {seed_rows_alive}. All numbers below are "
|
|
829
|
+
"measured or engine-estimated market data, not model knowledge."
|
|
830
|
+
)
|
|
831
|
+
add("")
|
|
832
|
+
add("### Demand and click cost by keyword")
|
|
833
|
+
add("")
|
|
834
|
+
add("| Keyword | Monthly searches | CPC (avg) | Top-of-page bid low–high | Competition | Paid by competitor | Sources |")
|
|
835
|
+
add("| --- | ---: | ---: | ---: | --- | --- | --- |")
|
|
836
|
+
for row in top_rows:
|
|
837
|
+
bid_range = (
|
|
838
|
+
f"{_fmt(row.get('cpc_low'), money=True)}–{_fmt(row.get('cpc_high'), money=True)}"
|
|
839
|
+
if row.get("cpc_low") is not None or row.get("cpc_high") is not None
|
|
840
|
+
else "—"
|
|
841
|
+
)
|
|
842
|
+
add(
|
|
843
|
+
f"| {row['keyword']} | {_fmt(row.get('search_volume'))} | "
|
|
844
|
+
f"{_fmt(row.get('cpc'), money=True)} | {bid_range} | "
|
|
845
|
+
f"{_fmt(row.get('competition'))} | "
|
|
846
|
+
f"{', '.join(row.get('paid_by', [])) or '—'} | "
|
|
847
|
+
f"{', '.join(sorted({s.split(':', 1)[0] for s in row['sources']}))} |"
|
|
848
|
+
)
|
|
849
|
+
add("")
|
|
850
|
+
add(f"{len(ranked)} keywords with non-zero volume collected in total; the table shows the top {len(top_rows)} by commercial intent and volume. The full set is in the machine-readable evidence file.")
|
|
851
|
+
add("")
|
|
852
|
+
|
|
853
|
+
add("### Who occupies the results page")
|
|
854
|
+
add("")
|
|
855
|
+
add("| Head keyword | Advertisers in ads block | Organic top domains |")
|
|
856
|
+
add("| --- | --- | --- |")
|
|
857
|
+
for obs in observations:
|
|
858
|
+
add(
|
|
859
|
+
f"| {obs['keyword']} | "
|
|
860
|
+
f"{', '.join(obs['paid_advertisers']) or ('unavailable' if obs.get('response_status') == 'unavailable' else 'none observed')} | "
|
|
861
|
+
f"{', '.join(obs['organic_top'][:5])} |"
|
|
862
|
+
)
|
|
863
|
+
add("")
|
|
864
|
+
zero_ads = [o["keyword"] for o in observations if o["ads_count"] == 0]
|
|
865
|
+
if zero_ads:
|
|
866
|
+
add(
|
|
867
|
+
"No ads block was observed on: "
|
|
868
|
+
+ ", ".join(f"`{k}`" for k in zero_ads)
|
|
869
|
+
+ ". This sampled observation does not establish that nobody advertises "
|
|
870
|
+
"or pays for these queries; competitor spending remains unproven here."
|
|
871
|
+
)
|
|
872
|
+
free_heads = [k for k in zero_ads if has_free_edu_intent(k)]
|
|
873
|
+
if free_heads:
|
|
874
|
+
add(
|
|
875
|
+
"Note: "
|
|
876
|
+
+ ", ".join(f"`{k}`" for k in free_heads)
|
|
877
|
+
+ " carry free/education intent markers — an empty ads block "
|
|
878
|
+
"there is partially self-explanatory."
|
|
879
|
+
)
|
|
880
|
+
add("")
|
|
881
|
+
|
|
882
|
+
add("### Competitors on the demand surface")
|
|
883
|
+
add("")
|
|
884
|
+
add("| Domain | Kind | Found via | SERP paid | SERP organic |")
|
|
885
|
+
add("| --- | --- | --- | ---: | ---: |")
|
|
886
|
+
for row in competitor_rows[:12]:
|
|
887
|
+
add(
|
|
888
|
+
f"| {row['domain']} | {row['kind']} | {', '.join(row['found_via'])} | "
|
|
889
|
+
f"{row['serp_paid_appearances']} | {row['serp_organic_appearances']} |"
|
|
890
|
+
)
|
|
891
|
+
add("")
|
|
892
|
+
if confirmed:
|
|
893
|
+
add(
|
|
894
|
+
"Confirmed product competitors (keyword footprints pulled): "
|
|
895
|
+
+ ", ".join(f"`{d}`" for d in confirmed)
|
|
896
|
+
+ "."
|
|
897
|
+
)
|
|
898
|
+
if overlap_domains:
|
|
899
|
+
add(
|
|
900
|
+
"Additional keyword-overlap neighbors (Labs `competitors_domain`, "
|
|
901
|
+
"not re-crawled): " + ", ".join(f"`{d}`" for d in overlap_domains[:10]) + "."
|
|
902
|
+
)
|
|
903
|
+
add("")
|
|
904
|
+
|
|
905
|
+
if footprint_rows:
|
|
906
|
+
add("### Competitor keyword footprints (separate evidence)")
|
|
907
|
+
add("")
|
|
908
|
+
add(
|
|
909
|
+
"Footprints are competitor-scoped evidence: only rows sharing a "
|
|
910
|
+
"category token with the seed market are merged into the demand "
|
|
911
|
+
"table above; everything else stays here and in the evidence "
|
|
912
|
+
"file, and none of it feeds the budget simulation."
|
|
913
|
+
)
|
|
914
|
+
add("")
|
|
915
|
+
add("| Domain | Keyword | Monthly searches | CPC (avg) | Competitor pays for it | In market pool |")
|
|
916
|
+
add("| --- | --- | ---: | ---: | --- | --- |")
|
|
917
|
+
for domain in confirmed:
|
|
918
|
+
domain_rows = sorted(
|
|
919
|
+
(r for r in footprint_rows if r.get("domain") == domain),
|
|
920
|
+
key=lambda r: r.get("search_volume") or 0,
|
|
921
|
+
reverse=True,
|
|
922
|
+
)[:5]
|
|
923
|
+
for row in domain_rows:
|
|
924
|
+
add(
|
|
925
|
+
f"| {domain} | {row['keyword']} | "
|
|
926
|
+
f"{_fmt(row.get('search_volume'))} | "
|
|
927
|
+
f"{_fmt(row.get('cpc'), money=True)} | "
|
|
928
|
+
f"{'yes' if row.get('paid_by') else 'no'} | "
|
|
929
|
+
f"{'yes' if row.get('topical') else 'no'} |"
|
|
930
|
+
)
|
|
931
|
+
add("")
|
|
932
|
+
|
|
933
|
+
add("### Candidate-set forecast" + (f" — ${test_budget_usd:,.0f} test-budget arithmetic" if test_budget_usd is not None else " — provider monthly estimates; no test budget supplied"))
|
|
934
|
+
add("")
|
|
935
|
+
if simulation_coverage:
|
|
936
|
+
c = simulation_coverage
|
|
937
|
+
add(f"Scenario: {c['kind']}; {c['submitted_count']}/{c['requested_count']} requested keywords, "
|
|
938
|
+
f"{len(c['excluded'])} excluded, {len(c['omitted_keywords'])} omitted by cap; "
|
|
939
|
+
f"match `{c['match']}`, bid assumption ${c['bid']:g}, period `{c['forecast_period']}`.")
|
|
940
|
+
add(c["limitations"])
|
|
941
|
+
add("")
|
|
942
|
+
if simulation_omitted_reason:
|
|
943
|
+
add(
|
|
944
|
+
f"Simulation omitted: {simulation_omitted_reason}. Rather than "
|
|
945
|
+
"simulating on excluded queries, this run reports the limitation "
|
|
946
|
+
"explicitly — revise seeds toward commercial category phrasing "
|
|
947
|
+
"and re-run if a paid-search estimate is needed."
|
|
948
|
+
)
|
|
949
|
+
add("")
|
|
950
|
+
if budget_rows:
|
|
951
|
+
add("| Simulated keyword | Bid | Est. avg CPC | Est. clicks/month at bid | Est. cost/month | Est. clicks within test budget |")
|
|
952
|
+
add("| --- | ---: | ---: | ---: | ---: | ---: |")
|
|
953
|
+
elif not simulation_omitted_reason:
|
|
954
|
+
add("Forecast unavailable: the provider returned no usable result for this keyword set. "
|
|
955
|
+
"Expected clicks and cost are unknown, not zero. No budget feasibility estimate is available.")
|
|
956
|
+
for row in budget_rows:
|
|
957
|
+
add(
|
|
958
|
+
f"| {row['keyword']} | {_fmt(row.get('bid'), money=True)} | "
|
|
959
|
+
f"{_fmt(row.get('average_cpc'), money=True)} | "
|
|
960
|
+
f"{_fmt(row.get('monthly_clicks_at_bid'))} | "
|
|
961
|
+
f"{_fmt(row.get('monthly_cost_at_bid'), money=True)} | "
|
|
962
|
+
f"{_fmt(row.get('est_clicks_within_test_budget'))} |"
|
|
963
|
+
)
|
|
964
|
+
add("")
|
|
965
|
+
if budget_rows:
|
|
966
|
+
add(
|
|
967
|
+
"Estimates come from Google's ad-traffic forecast at the stated bid "
|
|
968
|
+
"(`ad_traffic_by_keywords`); clicks within the test budget are a "
|
|
969
|
+
"linear pro-rata of the monthly forecast and ignore learning-phase "
|
|
970
|
+
"effects. They anchor feasibility math; they do not guarantee auction "
|
|
971
|
+
"outcomes."
|
|
972
|
+
)
|
|
973
|
+
if simulation_coverage and simulation_coverage["kind"] == "explicit_scenario":
|
|
974
|
+
add("This forecast uses the caller's explicit scenario, not an automatically selected seed pool.")
|
|
975
|
+
else:
|
|
976
|
+
add("Simulation keywords are drawn only from the eligible seed and expansion pool — "
|
|
977
|
+
"never from competitor footprints, competitor-brand queries, or free/education-intent queries.")
|
|
978
|
+
add("")
|
|
979
|
+
|
|
980
|
+
add("### Method and coverage")
|
|
981
|
+
add("")
|
|
982
|
+
add(
|
|
983
|
+
f"- Metered DataForSEO tasks recorded for this section: "
|
|
984
|
+
f"{totals['tasks_spent']} (≈ ${totals['estimated_cost_usd']:.2f}), "
|
|
985
|
+
"accumulated across attempts; cached replays add no new spend."
|
|
986
|
+
)
|
|
987
|
+
add(
|
|
988
|
+
f"- Coverage caps: {SERP_HEADS} SERP heads at depth {SERP_DEPTH}, "
|
|
989
|
+
f"{CONFIRM_COMPETITORS} competitor keyword footprints at "
|
|
990
|
+
f"{RANKED_KEYWORDS_LIMIT} keywords each, {BUDGET_SIM_KEYWORDS} "
|
|
991
|
+
"budget-simulation keywords."
|
|
992
|
+
)
|
|
993
|
+
if expansion_rejected_count or expansion_veto_rejected_count:
|
|
994
|
+
add(
|
|
995
|
+
f"- Expansion drift guard rejected {expansion_rejected_count} "
|
|
996
|
+
"Google-suggested keywords sharing no seed content token, plus "
|
|
997
|
+
f"{expansion_veto_rejected_count} more via the leading-brand "
|
|
998
|
+
"veto (both groups listed with their reasons in the evidence "
|
|
999
|
+
"file) — foreign-brand drift like 'ai canvas' -> 'canva ai' "
|
|
1000
|
+
"stays out of the demand model."
|
|
1001
|
+
)
|
|
1002
|
+
add(
|
|
1003
|
+
"- Not covered here: non-Google channels, organic difficulty, "
|
|
1004
|
+
"seasonality time series, and any interpretation of which "
|
|
1005
|
+
"segment/offer/channel combination to pursue (downstream concern)."
|
|
1006
|
+
)
|
|
1007
|
+
add("")
|
|
1008
|
+
add("### Sources")
|
|
1009
|
+
add("")
|
|
1010
|
+
add(
|
|
1011
|
+
f"- DataForSEO Google Ads endpoints (`search_volume`, "
|
|
1012
|
+
f"`keywords_for_keywords`, `ad_traffic_by_keywords`), Labs "
|
|
1013
|
+
f"(`ranked_keywords`, `competitors_domain`), and SERP advanced — "
|
|
1014
|
+
f"retrieved {retrieved_on} via the OS tools gateway; evidence rows in "
|
|
1015
|
+
f"`{EVIDENCE_REL.as_posix()}`."
|
|
1016
|
+
)
|
|
1017
|
+
return "\n".join(lines).strip() + "\n"
|
|
1018
|
+
|
|
1019
|
+
|
|
1020
|
+
# ── orchestration ──────────────────────────────────────────────────────────
|
|
1021
|
+
|
|
1022
|
+
@dataclass
|
|
1023
|
+
class RunInputs:
|
|
1024
|
+
seeds: list[str]
|
|
1025
|
+
seed_domains: list[str]
|
|
1026
|
+
location_code: int
|
|
1027
|
+
language_code: str
|
|
1028
|
+
test_budget_usd: float
|
|
1029
|
+
scenario: dict | None = None
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
def run_pipeline(client: ToolsClient, inputs: RunInputs) -> dict[str, Any]:
|
|
1033
|
+
locale = {"location_code": inputs.location_code, "language_code": inputs.language_code}
|
|
1034
|
+
forecast = _load_script("keyword-forecast.py", "pmr_keyword_forecast")
|
|
1035
|
+
scenario = forecast.validate_scenario(inputs.scenario, locale) if inputs.scenario is not None else None
|
|
1036
|
+
if type(inputs.location_code) is not int or inputs.location_code <= 0 or not re.fullmatch(r"[a-z]{2,3}(?:-[a-z]{2})?", inputs.language_code):
|
|
1037
|
+
raise KeywordEconomicsError("location-code and normalized language-code are invalid")
|
|
1038
|
+
if inputs.test_budget_usd is not None and (not math.isfinite(inputs.test_budget_usd) or inputs.test_budget_usd <= 0):
|
|
1039
|
+
raise KeywordEconomicsError("test-budget-usd must be finite and positive")
|
|
1040
|
+
|
|
1041
|
+
seed_volume = client.call("search-volume", {"keywords": inputs.seeds, **locale})
|
|
1042
|
+
expansion = client.call("keywords-for-keywords", {"keywords": inputs.seeds, **locale})
|
|
1043
|
+
|
|
1044
|
+
seed_rows = [normalize_keyword_row(r, "search_volume") for r in seed_volume.get("result") or []]
|
|
1045
|
+
expansion_rows = [normalize_keyword_row(r, "keywords_for_keywords") for r in expansion.get("result") or []]
|
|
1046
|
+
# Google's expansion drifts into foreign brands ("ai canvas" -> "canva
|
|
1047
|
+
# ai", observed live at 135k volume): an expansion keyword joins the
|
|
1048
|
+
# market pool only when it shares a seed content token. Seeds themselves
|
|
1049
|
+
# are the declared market and always stay.
|
|
1050
|
+
seed_anchor_tokens = content_tokens(inputs.seeds)
|
|
1051
|
+
kept_expansion: list[dict[str, Any]] = []
|
|
1052
|
+
rejected_expansion: list[dict[str, Any]] = []
|
|
1053
|
+
for row in expansion_rows:
|
|
1054
|
+
if row is None:
|
|
1055
|
+
continue
|
|
1056
|
+
if is_topical(row["keyword"], seed_anchor_tokens):
|
|
1057
|
+
kept_expansion.append(row)
|
|
1058
|
+
else:
|
|
1059
|
+
rejected_expansion.append(row)
|
|
1060
|
+
# Second pass: tokens dominating the rejected set are foreign brands;
|
|
1061
|
+
# keywords that slipped through on a single shared seed token ('canva ai
|
|
1062
|
+
# text' via 'text') get vetoed too.
|
|
1063
|
+
veto_tokens = build_foreign_brand_veto(
|
|
1064
|
+
[row["keyword"] for row in rejected_expansion], seed_anchor_tokens
|
|
1065
|
+
)
|
|
1066
|
+
veto_rejected: list[dict[str, Any]] = []
|
|
1067
|
+
if veto_tokens:
|
|
1068
|
+
still_kept: list[dict[str, Any]] = []
|
|
1069
|
+
for row in kept_expansion:
|
|
1070
|
+
tokens = row["keyword"].split()
|
|
1071
|
+
if tokens and tokens[0] in veto_tokens:
|
|
1072
|
+
veto_rejected.append(row)
|
|
1073
|
+
else:
|
|
1074
|
+
still_kept.append(row)
|
|
1075
|
+
kept_expansion = still_kept
|
|
1076
|
+
rejected_expansion.sort(key=lambda r: r.get("search_volume") or 0, reverse=True)
|
|
1077
|
+
veto_rejected.sort(key=lambda r: r.get("search_volume") or 0, reverse=True)
|
|
1078
|
+
expansion_rejected_rows = [
|
|
1079
|
+
{
|
|
1080
|
+
"type": "expansion_rejected",
|
|
1081
|
+
"keyword": row["keyword"],
|
|
1082
|
+
"intent_signal": row["intent_signal"],
|
|
1083
|
+
"search_volume": row.get("search_volume"),
|
|
1084
|
+
"reason": "no_seed_token_overlap",
|
|
1085
|
+
"tier": "C",
|
|
1086
|
+
}
|
|
1087
|
+
for row in rejected_expansion[:20]
|
|
1088
|
+
] + [
|
|
1089
|
+
{
|
|
1090
|
+
"type": "expansion_rejected",
|
|
1091
|
+
"keyword": row["keyword"],
|
|
1092
|
+
"intent_signal": row["intent_signal"],
|
|
1093
|
+
"search_volume": row.get("search_volume"),
|
|
1094
|
+
"reason": "foreign_brand_veto",
|
|
1095
|
+
"tier": "C",
|
|
1096
|
+
}
|
|
1097
|
+
for row in veto_rejected[:20]
|
|
1098
|
+
]
|
|
1099
|
+
# The MARKET pool: seeds + the anchored part of Google's expansion. This
|
|
1100
|
+
# pool alone defines the demand model; competitor footprints may only
|
|
1101
|
+
# annotate it (topical intersection) and never replace it.
|
|
1102
|
+
market_pool = merge_keyword_rows(
|
|
1103
|
+
[r for r in seed_rows if r],
|
|
1104
|
+
kept_expansion,
|
|
1105
|
+
)
|
|
1106
|
+
base_ranked = rank_keywords(market_pool)
|
|
1107
|
+
if not base_ranked:
|
|
1108
|
+
raise KeywordEconomicsError(
|
|
1109
|
+
"no seed or expanded keyword has non-zero search volume; the seed "
|
|
1110
|
+
"list does not map to measurable demand — revise seeds in the "
|
|
1111
|
+
"workflow and re-run"
|
|
1112
|
+
)
|
|
1113
|
+
seed_rows_alive = sum(
|
|
1114
|
+
1
|
|
1115
|
+
for r in seed_rows
|
|
1116
|
+
if r and (r.get("search_volume") or 0) > 0
|
|
1117
|
+
)
|
|
1118
|
+
# Topicality anchors come from the SEEDS ONLY (same set that gates the
|
|
1119
|
+
# expansion above).
|
|
1120
|
+
market_tokens = seed_anchor_tokens
|
|
1121
|
+
|
|
1122
|
+
heads = pick_serp_heads(base_ranked)
|
|
1123
|
+
observations = [
|
|
1124
|
+
extract_serp_observation(
|
|
1125
|
+
head,
|
|
1126
|
+
client.call("serp", {"keyword": head, "depth": SERP_DEPTH, **locale}).get("result") or [],
|
|
1127
|
+
)
|
|
1128
|
+
for head in heads
|
|
1129
|
+
]
|
|
1130
|
+
|
|
1131
|
+
keyword_volumes = {
|
|
1132
|
+
row["keyword"]: int(row.get("search_volume") or 0) for row in base_ranked
|
|
1133
|
+
}
|
|
1134
|
+
competitor_rows = tally_competitors(
|
|
1135
|
+
observations, inputs.seed_domains, keyword_volumes
|
|
1136
|
+
)
|
|
1137
|
+
confirmed = confirmed_competitors(
|
|
1138
|
+
competitor_rows, inputs.seed_domains, seed_keywords=inputs.seeds
|
|
1139
|
+
)
|
|
1140
|
+
named_domains = {registrable_domain(domain) for domain in inputs.seed_domains}
|
|
1141
|
+
for row in competitor_rows:
|
|
1142
|
+
row["confirmation_status"] = (
|
|
1143
|
+
"founder_seed" if row["domain"] in named_domains and row["domain"] in confirmed
|
|
1144
|
+
else "confirmed" if row["domain"] in confirmed
|
|
1145
|
+
else "unconfirmed"
|
|
1146
|
+
)
|
|
1147
|
+
if row["confirmation_status"] == "unconfirmed" and row["kind"] == "candidate":
|
|
1148
|
+
row["confirmation_reason"] = "insufficient_category_relevant_serp_overlap"
|
|
1149
|
+
|
|
1150
|
+
# Competitor footprints stay a SEPARATE evidence set; only rows sharing a
|
|
1151
|
+
# category-bearing token with the market join the pool (annotated), so a
|
|
1152
|
+
# broad competitor cannot inject an unrelated high-volume query into the
|
|
1153
|
+
# demand tables (review finding #2).
|
|
1154
|
+
footprint_rows: list[dict[str, Any]] = []
|
|
1155
|
+
topical_rows: list[dict[str, Any]] = []
|
|
1156
|
+
for domain in confirmed:
|
|
1157
|
+
response = client.call(
|
|
1158
|
+
"ranked-keywords",
|
|
1159
|
+
{"target": domain, "limit": RANKED_KEYWORDS_LIMIT, **locale},
|
|
1160
|
+
)
|
|
1161
|
+
entry = (response.get("result") or [{}])[0]
|
|
1162
|
+
for item in entry.get("items") or []:
|
|
1163
|
+
row = normalize_ranked_row(item, domain)
|
|
1164
|
+
if row is None:
|
|
1165
|
+
continue
|
|
1166
|
+
topical = is_topical(row["keyword"], market_tokens) and not (
|
|
1167
|
+
set(row["keyword"].split()) & veto_tokens
|
|
1168
|
+
)
|
|
1169
|
+
footprint_rows.append(
|
|
1170
|
+
{
|
|
1171
|
+
**row,
|
|
1172
|
+
"type": "competitor_keyword",
|
|
1173
|
+
"domain": domain,
|
|
1174
|
+
"topical": topical,
|
|
1175
|
+
}
|
|
1176
|
+
)
|
|
1177
|
+
if topical:
|
|
1178
|
+
topical_rows.append({**row, "from_competitor_footprint": True})
|
|
1179
|
+
|
|
1180
|
+
market_pool = merge_keyword_rows(market_pool, topical_rows)
|
|
1181
|
+
ranked_pool = rank_keywords(market_pool)
|
|
1182
|
+
|
|
1183
|
+
overlap_domains: list[str] = []
|
|
1184
|
+
if confirmed:
|
|
1185
|
+
overlap = client.call(
|
|
1186
|
+
"competitors-domain",
|
|
1187
|
+
{"target": confirmed[0], "limit": 20, **locale},
|
|
1188
|
+
)
|
|
1189
|
+
entry = (overlap.get("result") or [{}])[0]
|
|
1190
|
+
for item in entry.get("items") or []:
|
|
1191
|
+
domain = registrable_domain(item.get("domain") or "")
|
|
1192
|
+
if domain and classify_domain(domain) == "candidate" and domain not in confirmed:
|
|
1193
|
+
overlap_domains.append(domain)
|
|
1194
|
+
|
|
1195
|
+
# Budget-simulation keywords come STRICTLY from the seed+expansion pool —
|
|
1196
|
+
# never from competitor footprints, even topical ones. They also estimate
|
|
1197
|
+
# CATEGORY economics, so competitor-brand queries (bidding on 'miro …' is
|
|
1198
|
+
# a separate tactic) and explicit free-intent queries are excluded; when
|
|
1199
|
+
# nothing trustworthy remains the simulation is OMITTED rather than
|
|
1200
|
+
# silently bypassing the filters.
|
|
1201
|
+
competitor_brands = {
|
|
1202
|
+
brand_token(d)
|
|
1203
|
+
for d in confirmed + [registrable_domain(x) for x in inputs.seed_domains]
|
|
1204
|
+
}
|
|
1205
|
+
competitor_brands.discard("")
|
|
1206
|
+
# A brand token that is itself category vocabulary (whiteboard.chat ->
|
|
1207
|
+
# 'whiteboard') must not exclude core category queries.
|
|
1208
|
+
competitor_brands = {
|
|
1209
|
+
brand
|
|
1210
|
+
for brand in competitor_brands
|
|
1211
|
+
if not any(_tokens_match(brand, anchor) for anchor in seed_anchor_tokens)
|
|
1212
|
+
}
|
|
1213
|
+
_BRAND_SUFFIXES = ("com", "io", "app", "ai", "co", "net", "org")
|
|
1214
|
+
|
|
1215
|
+
def _token_is_brand(token: str) -> bool:
|
|
1216
|
+
normalized = re.sub(r"[^a-z0-9]", "", token)
|
|
1217
|
+
for brand in competitor_brands:
|
|
1218
|
+
if normalized == brand:
|
|
1219
|
+
return True
|
|
1220
|
+
# 'monday.com' -> 'mondaycom' must match brand 'monday'.
|
|
1221
|
+
if normalized.startswith(brand) and normalized[len(brand):] in _BRAND_SUFFIXES:
|
|
1222
|
+
return True
|
|
1223
|
+
return False
|
|
1224
|
+
|
|
1225
|
+
def sim_eligible(row: dict[str, Any]) -> bool:
|
|
1226
|
+
if any(_token_is_brand(token) for token in row["keyword"].split()):
|
|
1227
|
+
return False
|
|
1228
|
+
if has_free_intent(row["keyword"]):
|
|
1229
|
+
return False
|
|
1230
|
+
return True
|
|
1231
|
+
|
|
1232
|
+
eligible = [r for r in base_ranked if sim_eligible(r)]
|
|
1233
|
+
sim_rows = eligible[:BUDGET_SIM_KEYWORDS]
|
|
1234
|
+
requested_keywords = scenario["keywords"] if scenario else [r["keyword"] for r in eligible]
|
|
1235
|
+
sim_keywords = requested_keywords[:BUDGET_SIM_KEYWORDS]
|
|
1236
|
+
simulation_omitted_reason = None
|
|
1237
|
+
if not sim_keywords:
|
|
1238
|
+
simulation_omitted_reason = (
|
|
1239
|
+
"no trustworthy simulation keywords remained after excluding "
|
|
1240
|
+
"competitor-brand and free-intent queries"
|
|
1241
|
+
)
|
|
1242
|
+
bid = median([r.get("cpc_high") for r in sim_rows]) or median(
|
|
1243
|
+
[r.get("cpc") for r in sim_rows]
|
|
1244
|
+
) or 1.0
|
|
1245
|
+
# The forecast endpoint requires whole USD, unlike observed auction CPCs.
|
|
1246
|
+
# Choose a valid exploratory assumption before spending, never after failure.
|
|
1247
|
+
bid_normalized_to_int = not scenario and float(bid) != math.ceil(float(bid))
|
|
1248
|
+
# Preserve the numeric representation used by existing successful cache keys.
|
|
1249
|
+
bid = scenario["bid"] if scenario else float(max(1, min(math.ceil(float(bid)), 500)))
|
|
1250
|
+
match = scenario["match"] if scenario else "exact"
|
|
1251
|
+
budget_rows: list[dict[str, Any]] = []
|
|
1252
|
+
if sim_keywords:
|
|
1253
|
+
ad_traffic = client.call(
|
|
1254
|
+
"ad-traffic",
|
|
1255
|
+
{"keywords": sim_keywords, "bid": bid, "match": match, **locale},
|
|
1256
|
+
)
|
|
1257
|
+
budget_rows = build_budget_rows(
|
|
1258
|
+
sim_keywords, bid, ad_traffic.get("result") or [], inputs.test_budget_usd
|
|
1259
|
+
)
|
|
1260
|
+
simulation_coverage = forecast.coverage(
|
|
1261
|
+
requested=requested_keywords, submitted=sim_keywords, locale=locale, bid=bid, match=match, scenario=scenario,
|
|
1262
|
+
excluded=[] if scenario else [{"keyword": r["keyword"], "reason": "competitor_brand_or_free_education_intent"}
|
|
1263
|
+
for r in base_ranked if not sim_eligible(r)],
|
|
1264
|
+
)
|
|
1265
|
+
simulation_coverage["response_rows"] = len(budget_rows)
|
|
1266
|
+
simulation_coverage["response_status"] = "available" if budget_rows else "unavailable"
|
|
1267
|
+
|
|
1268
|
+
for domain in overlap_domains:
|
|
1269
|
+
competitor_rows.append(
|
|
1270
|
+
{
|
|
1271
|
+
"type": "competitor_domain",
|
|
1272
|
+
"domain": domain,
|
|
1273
|
+
"kind": "candidate",
|
|
1274
|
+
"found_via": ["keyword_overlap"],
|
|
1275
|
+
"serp_paid_appearances": 0,
|
|
1276
|
+
"serp_organic_appearances": 0,
|
|
1277
|
+
"score": 0,
|
|
1278
|
+
"tier": "B",
|
|
1279
|
+
}
|
|
1280
|
+
)
|
|
1281
|
+
|
|
1282
|
+
# Durable spend provenance comes from the response ledger, not this
|
|
1283
|
+
# attempt's counters — a cache-only replay reports the original metered
|
|
1284
|
+
# spend and renders byte-identical artifacts (review finding #1).
|
|
1285
|
+
totals = client.ledger_totals()
|
|
1286
|
+
retrieved_on = (
|
|
1287
|
+
totals["first_retrieved_on"]
|
|
1288
|
+
if totals["first_retrieved_on"] == totals["last_retrieved_on"]
|
|
1289
|
+
else f"{totals['first_retrieved_on']}..{totals['last_retrieved_on']}"
|
|
1290
|
+
)
|
|
1291
|
+
|
|
1292
|
+
evidence = {
|
|
1293
|
+
"schema_version": SCHEMA_VERSION,
|
|
1294
|
+
"section_id": SECTION_ID,
|
|
1295
|
+
"generated_by": Path(__file__).name,
|
|
1296
|
+
"retrieved_on": retrieved_on,
|
|
1297
|
+
"locale": locale,
|
|
1298
|
+
"simulation": simulation_coverage,
|
|
1299
|
+
"request_hashes": sorted(p.stem for p in client._ledger_entries),
|
|
1300
|
+
"inputs": {
|
|
1301
|
+
"seed_keywords": inputs.seeds,
|
|
1302
|
+
"seed_domains": inputs.seed_domains,
|
|
1303
|
+
"test_budget_usd": inputs.test_budget_usd,
|
|
1304
|
+
"bid_normalized_to_int": bid_normalized_to_int,
|
|
1305
|
+
"scenario_sha256": simulation_coverage["scenario_sha256"],
|
|
1306
|
+
"simulation_omitted_reason": simulation_omitted_reason,
|
|
1307
|
+
"expansion_rejected_count": len(rejected_expansion),
|
|
1308
|
+
"expansion_veto_rejected_count": len(veto_rejected),
|
|
1309
|
+
},
|
|
1310
|
+
"rows": (
|
|
1311
|
+
ranked_pool
|
|
1312
|
+
+ footprint_rows
|
|
1313
|
+
+ expansion_rejected_rows
|
|
1314
|
+
+ observations
|
|
1315
|
+
+ competitor_rows
|
|
1316
|
+
+ budget_rows
|
|
1317
|
+
),
|
|
1318
|
+
"totals": {
|
|
1319
|
+
"tasks_spent": totals["tasks_spent"],
|
|
1320
|
+
"estimated_cost_usd": totals["estimated_cost_usd"],
|
|
1321
|
+
},
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
section_markdown = render_section(
|
|
1325
|
+
locale=locale,
|
|
1326
|
+
seeds=inputs.seeds,
|
|
1327
|
+
seed_rows_alive=seed_rows_alive,
|
|
1328
|
+
keyword_rows=market_pool,
|
|
1329
|
+
footprint_rows=footprint_rows,
|
|
1330
|
+
observations=observations,
|
|
1331
|
+
competitor_rows=competitor_rows,
|
|
1332
|
+
confirmed=confirmed,
|
|
1333
|
+
budget_rows=budget_rows,
|
|
1334
|
+
overlap_domains=overlap_domains,
|
|
1335
|
+
totals=totals,
|
|
1336
|
+
test_budget_usd=inputs.test_budget_usd,
|
|
1337
|
+
retrieved_on=retrieved_on,
|
|
1338
|
+
expansion_rejected_count=len(rejected_expansion),
|
|
1339
|
+
expansion_veto_rejected_count=len(veto_rejected),
|
|
1340
|
+
simulation_omitted_reason=simulation_omitted_reason,
|
|
1341
|
+
simulation_coverage=simulation_coverage,
|
|
1342
|
+
)
|
|
1343
|
+
return {
|
|
1344
|
+
"evidence": evidence,
|
|
1345
|
+
"section_markdown": section_markdown,
|
|
1346
|
+
"totals": totals,
|
|
1347
|
+
"attempt": {
|
|
1348
|
+
"tasks_spent": client.tasks_spent,
|
|
1349
|
+
"cache_hits": client.cache_hits,
|
|
1350
|
+
},
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
|
|
1354
|
+
def _write_atomic(path: Path, content: str) -> None:
|
|
1355
|
+
_load_script("research-artifact-io.py", "pmr_artifact_io").write_atomic(path, content)
|
|
1356
|
+
|
|
1357
|
+
|
|
1358
|
+
def run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
|
|
1359
|
+
workspace = workspace.resolve()
|
|
1360
|
+
runtime = workspace / RUNTIME_REL
|
|
1361
|
+
cache_dir = runtime / "cache"
|
|
1362
|
+
runtime.mkdir(parents=True, exist_ok=True)
|
|
1363
|
+
|
|
1364
|
+
api_key = (os.environ.get("OS_API_KEY") or "").strip()
|
|
1365
|
+
if not api_key:
|
|
1366
|
+
raise KeywordEconomicsError("OS_API_KEY is required")
|
|
1367
|
+
os_url = (os.environ.get("OS_URL") or DEFAULT_OS_URL).strip() or DEFAULT_OS_URL
|
|
1368
|
+
|
|
1369
|
+
seeds = parse_terms(args.seed_keywords, MAX_SEED_KEYWORDS)
|
|
1370
|
+
if not seeds:
|
|
1371
|
+
raise KeywordEconomicsError(
|
|
1372
|
+
"at least one seed keyword is required for the assigned search scope"
|
|
1373
|
+
)
|
|
1374
|
+
seed_domains = parse_terms(args.seed_domains or "", MAX_SEED_DOMAINS)
|
|
1375
|
+
|
|
1376
|
+
inputs = RunInputs(
|
|
1377
|
+
seeds=seeds,
|
|
1378
|
+
seed_domains=seed_domains,
|
|
1379
|
+
location_code=args.location_code,
|
|
1380
|
+
language_code=args.language_code,
|
|
1381
|
+
test_budget_usd=args.test_budget_usd,
|
|
1382
|
+
scenario=json.loads(Path(args.scenario).read_text()) if getattr(args, "scenario", None) else None,
|
|
1383
|
+
)
|
|
1384
|
+
client = ToolsClient(
|
|
1385
|
+
os_url=os_url,
|
|
1386
|
+
api_key=api_key,
|
|
1387
|
+
cache_dir=cache_dir,
|
|
1388
|
+
max_tasks=args.max_tasks,
|
|
1389
|
+
)
|
|
1390
|
+
|
|
1391
|
+
io = _load_script("research-artifact-io.py", "pmr_artifact_io")
|
|
1392
|
+
publication = _load_script("keyword-locale-publication.py", "pmr_keyword_publication")
|
|
1393
|
+
upsert = _load_script("upsert-markdown-section.py", "pmr_upsert_markdown_section")
|
|
1394
|
+
persist = _load_script("persist-research-report.py", "pmr_persist_research_report")
|
|
1395
|
+
key = publication.locale_key(inputs.location_code, inputs.language_code)
|
|
1396
|
+
io.bind_context(workspace, "keyword_economics", context_file=getattr(args, "context_file", None))
|
|
1397
|
+
context_hash = io.context_binding(workspace, "keyword_economics")
|
|
1398
|
+
with io.file_lock(runtime / "locales" / key / ".run.lock", timeout=1):
|
|
1399
|
+
publication.begin(workspace, key, context_hash, io)
|
|
1400
|
+
try:
|
|
1401
|
+
outcome = run_pipeline(client, inputs)
|
|
1402
|
+
return publication.publish(workspace, key, context_hash, outcome, io, upsert, persist)
|
|
1403
|
+
except Exception:
|
|
1404
|
+
publication.failed(workspace, key, context_hash, io)
|
|
1405
|
+
raise
|
|
1406
|
+
|
|
1407
|
+
|
|
1408
|
+
def main() -> int:
|
|
1409
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
1410
|
+
parser.add_argument("--context-file", type=Path, help="Explicit additional context for this measurement.")
|
|
1411
|
+
parser.add_argument("--workspace", type=Path, required=True)
|
|
1412
|
+
parser.add_argument(
|
|
1413
|
+
"--seed-keywords",
|
|
1414
|
+
required=True,
|
|
1415
|
+
help="Comma/semicolon-separated candidate search phrases (3-20).",
|
|
1416
|
+
)
|
|
1417
|
+
parser.add_argument(
|
|
1418
|
+
"--seed-domains",
|
|
1419
|
+
default="",
|
|
1420
|
+
help="Comma-separated competitor domains supplied for this research.",
|
|
1421
|
+
)
|
|
1422
|
+
parser.add_argument("--location-code", type=int, default=2840)
|
|
1423
|
+
parser.add_argument("--language-code", default="en")
|
|
1424
|
+
parser.add_argument("--test-budget-usd", type=float, default=None, help="Optional task-specific budget for conditional arithmetic; no budget is chosen by this runner.")
|
|
1425
|
+
parser.add_argument("--max-tasks", type=int, default=MAX_TASKS_PER_RUN)
|
|
1426
|
+
parser.add_argument("--scenario", type=Path, help="Explicit homogeneous forecast scenario JSON; validated before paid calls")
|
|
1427
|
+
args = parser.parse_args()
|
|
1428
|
+
|
|
1429
|
+
try:
|
|
1430
|
+
state = run(args.workspace, args)
|
|
1431
|
+
except (KeywordEconomicsError, ValueError, OSError, RuntimeError) as exc:
|
|
1432
|
+
print(f"{DISPLAY_NAME}: {exc}", file=sys.stderr)
|
|
1433
|
+
return 1
|
|
1434
|
+
print(
|
|
1435
|
+
f"{DISPLAY_NAME}: {state['status']} — {state['tasks_spent']} metered tasks "
|
|
1436
|
+
f"(≈ ${state['estimated_cost_usd']:.2f}), evidence at {state['evidence_path']}, "
|
|
1437
|
+
f"publication {state['publication_mode']} {state['report_commit_sha'][:12]}"
|
|
1438
|
+
)
|
|
1439
|
+
return 0 if state["status"] == "completed" else 1
|
|
1440
|
+
|
|
1441
|
+
|
|
1442
|
+
if __name__ == "__main__":
|
|
1443
|
+
raise SystemExit(main())
|