@egbe-ai/opencode 0.0.0-stage → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +124 -2
- package/index.js +5 -0
- package/package.json +55 -4
- package/plugin.js +230 -0
- package/session-log.js +315 -0
- package/skills/.egbe-catalog/manifest.json +1 -0
- package/skills/egbe-os-analytics/SKILL.md +499 -0
- package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
- package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
- package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
- package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
- package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
- package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
- package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
- package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
- package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
- package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
- package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
- package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
- package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
- package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
- package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
- package/skills/egbe-os-db/SKILL.md +174 -0
- package/skills/egbe-os-deploy/SKILL.md +412 -0
- package/skills/egbe-os-design/SKILL.md +182 -0
- package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
- package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
- package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
- package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
- package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
- package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
- package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
- package/skills/egbe-os-destination-readiness/references/report.md +116 -0
- package/skills/egbe-os-domains/SKILL.md +189 -0
- package/skills/egbe-os-errors/SKILL.md +259 -0
- package/skills/egbe-os-git/SKILL.md +275 -0
- package/skills/egbe-os-git/references/issues.md +33 -0
- package/skills/egbe-os-google-ads/SKILL.md +163 -0
- package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
- package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
- package/skills/egbe-os-google-ads/references/conversions.md +153 -0
- package/skills/egbe-os-google-ads/references/delivery.md +128 -0
- package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
- package/skills/egbe-os-google-ads/references/reporting.md +105 -0
- package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
- package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
- package/skills/egbe-os-google-auth/SKILL.md +285 -0
- package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
- package/skills/egbe-os-images/SKILL.md +163 -0
- package/skills/egbe-os-images/references/stock-images.md +21 -0
- package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
- package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
- package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
- package/skills/egbe-os-llm/SKILL.md +160 -0
- package/skills/egbe-os-mail/SKILL.md +180 -0
- package/skills/egbe-os-me/SKILL.md +119 -0
- package/skills/egbe-os-meta-ads/SKILL.md +114 -0
- package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
- package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
- package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
- package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
- package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
- package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
- package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
- package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
- package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
- package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
- package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
- package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
- package/skills/egbe-os-notifications/SKILL.md +78 -0
- package/skills/egbe-os-openai-ads/SKILL.md +149 -0
- package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
- package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
- package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
- package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
- package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
- package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
- package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
- package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
- package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
- package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
- package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
- package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
- package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
- package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
- package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
- package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
- package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
- package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
- package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
- package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
- package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
- package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
- package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
- package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
- package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
- package/skills/egbe-os-payments/SKILL.md +215 -0
- package/skills/egbe-os-payments/agents/openai.yaml +4 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
- package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
- package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
- package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
- package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
- package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
- package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
- package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
- package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
- package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
- package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
- package/skills/egbe-os-playbook/SKILL.md +79 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
- package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
- package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
- package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
- package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
- package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
- package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
- package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
- package/skills/egbe-os-playbook/playbooks/product.md +67 -0
- package/skills/egbe-os-playbook/references/catalog.md +127 -0
- package/skills/egbe-os-playbook/references/changes.md +167 -0
- package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
- package/skills/egbe-os-product-auth/SKILL.md +151 -0
- package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
- package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
- package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
- package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
- package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
- package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
- package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
- package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
- package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
- package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
- package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
- package/skills/egbe-os-product-build/SKILL.md +111 -0
- package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-build/references/capabilities.md +25 -0
- package/skills/egbe-os-product-build/references/engineering.md +72 -0
- package/skills/egbe-os-product-build/references/execution.md +55 -0
- package/skills/egbe-os-product-build/references/foundations.md +93 -0
- package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
- package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
- package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
- package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
- package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
- package/skills/egbe-os-product-market-research/SKILL.md +159 -0
- package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
- package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
- package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
- package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
- package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
- package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
- package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
- package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
- package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
- package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
- package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
- package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
- package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
- package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
- package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
- package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
- package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
- package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
- package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
- package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
- package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
- package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
- package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
- package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
- package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
- package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
- package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
- package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
- package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
- package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
- package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
- package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
- package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
- package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
- package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
- package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
- package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
- package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
- package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
- package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
- package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
- package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
- package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
- package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
- package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
- package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
- package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
- package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
- package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
- package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
- package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
- package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
- package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
- package/skills/egbe-os-product-report/SKILL.md +150 -0
- package/skills/egbe-os-product-report/references/artifacts.md +84 -0
- package/skills/egbe-os-product-report/references/brief.md +34 -0
- package/skills/egbe-os-product-report/references/daily.md +76 -0
- package/skills/egbe-os-product-report/references/sources.md +68 -0
- package/skills/egbe-os-product-routines/SKILL.md +166 -0
- package/skills/egbe-os-product-routines/references/catalog.md +22 -0
- package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
- package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
- package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
- package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
- package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
- package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
- package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
- package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
- package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
- package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
- package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
- package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
- package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
- package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
- package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
- package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
- package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
- package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
- package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
- package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
- package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
- package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
- package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
- package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
- package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
- package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
- package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
- package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
- package/skills/egbe-os-report-ui/SKILL.md +133 -0
- package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
- package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
- package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
- package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
- package/skills/egbe-os-report-ui/references/components.md +209 -0
- package/skills/egbe-os-research/SKILL.md +125 -0
- package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
- package/skills/egbe-os-scrape/SKILL.md +160 -0
- package/skills/egbe-os-search/SKILL.md +182 -0
- package/skills/egbe-os-search/references/google-places.md +81 -0
- package/skills/egbe-os-support/SKILL.md +149 -0
- package/skills/egbe-os-telemetry/SKILL.md +344 -0
- package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
- package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
- package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
- package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
- package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
- package/skills/egbe-os-tools/SKILL.md +180 -0
- package/skills/egbe-os-tools/references/theona.md +26 -0
- package/skills/egbe-os-wallet/SKILL.md +181 -0
- package/skills/egbe-setup/EGBE.md +484 -0
- package/skills/egbe-setup/SKILL.md +116 -0
- package/skills/egbe-setup/references/project.md +80 -0
- package/state.js +37 -0
- package/update.js +361 -0
|
@@ -0,0 +1,1051 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Run the deterministic Competitor Reviews section against review platforms.
|
|
3
|
+
|
|
4
|
+
Reviews of paid competitor products are offer material: a 1-3 star review
|
|
5
|
+
is a reviewer of a paid product reporting an unmet need (an offer wedge),
|
|
6
|
+
and a 5-star review is the exact language of value its users speak. No
|
|
7
|
+
payment signal exists per reviewer, so the section never claims reviewers
|
|
8
|
+
themselves paid. This runner collects both sides for
|
|
9
|
+
the market's competitors:
|
|
10
|
+
|
|
11
|
+
- G2 via the Apify actor `automation-lab/g2-scraper` (B2B reviews with
|
|
12
|
+
company-segment/industry codes, love/hate themes, switched-from fields);
|
|
13
|
+
- Trustpilot via `memo23/trustpilot-scraper-ppe` (consumer/prosumer
|
|
14
|
+
reviews, DOMAIN-keyed — identity by construction; billing and
|
|
15
|
+
cancellation complaints live here).
|
|
16
|
+
|
|
17
|
+
Two pulls per competitor per platform: lowest-rated (wedges) and top-rated
|
|
18
|
+
(value language). G2 is slug-keyed, so its product identity is verified
|
|
19
|
+
against the brand and unverified slugs are EXCLUDED from analysis rather
|
|
20
|
+
than silently mixed in (the Competitor Ads namesake lesson).
|
|
21
|
+
|
|
22
|
+
No LLM anywhere: platform-provided theme fields (hateTheme/loveTheme,
|
|
23
|
+
switchedReason) and deterministic extraction carry the analysis; synthesis
|
|
24
|
+
into hypotheses is a downstream concern. Caching, spend provenance, budgets
|
|
25
|
+
and retries mirror the Competitor Ads runner.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import sys as _artifact_sys
|
|
31
|
+
from pathlib import Path as _ArtifactPath
|
|
32
|
+
_artifact_sys.path.insert(0, str(_ArtifactPath(__file__).resolve().parent))
|
|
33
|
+
from egbe_artifacts import artifact_path as _artifact_path
|
|
34
|
+
|
|
35
|
+
import runpy as _egbe_runpy
|
|
36
|
+
from pathlib import Path as _EgbePath
|
|
37
|
+
_egbe_adapter = _EgbePath(__file__).with_name("connection_identity.py")
|
|
38
|
+
_connection_headers = _egbe_runpy.run_path(str(_egbe_adapter))["connection_headers"] if _egbe_adapter.is_file() else lambda _url: {}
|
|
39
|
+
|
|
40
|
+
import argparse
|
|
41
|
+
import hashlib
|
|
42
|
+
import importlib.util
|
|
43
|
+
import json
|
|
44
|
+
import os
|
|
45
|
+
import re
|
|
46
|
+
import sys
|
|
47
|
+
import time
|
|
48
|
+
import urllib.error
|
|
49
|
+
import urllib.request
|
|
50
|
+
from collections import Counter
|
|
51
|
+
from dataclasses import dataclass
|
|
52
|
+
from datetime import date
|
|
53
|
+
from pathlib import Path
|
|
54
|
+
from types import ModuleType
|
|
55
|
+
from typing import Any, Callable
|
|
56
|
+
from urllib.parse import urlparse
|
|
57
|
+
|
|
58
|
+
SCHEMA_VERSION = 1
|
|
59
|
+
SECTION_ID = "review_mining"
|
|
60
|
+
SECTION_HEADING = "Competitor reviews"
|
|
61
|
+
DISPLAY_NAME = "Competitor Reviews"
|
|
62
|
+
|
|
63
|
+
DEFAULT_OS_URL = "https://os.hz.egbe.dev"
|
|
64
|
+
RUNTIME_REL = Path(_artifact_path("egbe-docs/research/market/work")) / SECTION_ID
|
|
65
|
+
EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/review-mining.json"))
|
|
66
|
+
REPORT_REL = Path(_artifact_path("egbe-docs/research/market/full-report.md"))
|
|
67
|
+
|
|
68
|
+
G2_ACTOR = "automation-lab/g2-scraper"
|
|
69
|
+
TRUSTPILOT_ACTOR = "memo23/trustpilot-scraper-ppe"
|
|
70
|
+
|
|
71
|
+
# Two platforms x two pulls (low stars, top stars) per competitor.
|
|
72
|
+
MAX_ACTOR_RUNS_PER_ATTEMPT = 12
|
|
73
|
+
MAX_COMPETITORS = 3
|
|
74
|
+
LOW_REVIEWS_CAP = 60
|
|
75
|
+
HIGH_REVIEWS_CAP = 40
|
|
76
|
+
STORED_LOW_PER_PLATFORM = 30
|
|
77
|
+
STORED_HIGH_PER_PLATFORM = 15
|
|
78
|
+
TEXT_TRUNCATE = 300
|
|
79
|
+
LOW_STARS = {1, 2, 3}
|
|
80
|
+
|
|
81
|
+
ACTOR_RETRY_DELAY_SECONDS = 20.0
|
|
82
|
+
|
|
83
|
+
# Billing/pricing pain markers inside low-star texts — recurring anti-
|
|
84
|
+
# positioning material for offers ("no credit card tricks", "cancel anytime").
|
|
85
|
+
_BILLING_RE = re.compile(
|
|
86
|
+
r"\b(refund(?:s|ed)?|cancel(?:led|ed|ling|ing)?|billing|billed|"
|
|
87
|
+
r"charge[ds]?|subscription[s]?|price[sd]?|pricing|expensive|cost[s]?|"
|
|
88
|
+
r"renewal[s]?|auto-?renew(?:al|s)?|trial[s]?)\b",
|
|
89
|
+
re.IGNORECASE,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class ReviewMiningError(RuntimeError):
|
|
94
|
+
"""Terminal runner failure; runtime state is preserved for resume."""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _load_script(filename: str, module_name: str) -> ModuleType:
|
|
98
|
+
path = Path(__file__).resolve().parent / filename
|
|
99
|
+
spec = importlib.util.spec_from_file_location(module_name, path)
|
|
100
|
+
if spec is None or spec.loader is None:
|
|
101
|
+
raise ReviewMiningError(f"cannot load bundled script {filename}")
|
|
102
|
+
module = importlib.util.module_from_spec(spec)
|
|
103
|
+
sys.modules[spec.name] = module
|
|
104
|
+
previous = sys.dont_write_bytecode
|
|
105
|
+
sys.dont_write_bytecode = True
|
|
106
|
+
try:
|
|
107
|
+
spec.loader.exec_module(module)
|
|
108
|
+
finally:
|
|
109
|
+
sys.dont_write_bytecode = previous
|
|
110
|
+
return module
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
# ── OS client (same contract as the Competitor Ads runner) ─────────────────
|
|
114
|
+
|
|
115
|
+
@dataclass
|
|
116
|
+
class OsClient:
|
|
117
|
+
os_url: str
|
|
118
|
+
api_key: str
|
|
119
|
+
cache_dir: Path
|
|
120
|
+
max_actor_runs: int = MAX_ACTOR_RUNS_PER_ATTEMPT
|
|
121
|
+
actor_runs: int = 0
|
|
122
|
+
cache_hits: int = 0
|
|
123
|
+
touched: set[str] = None # type: ignore[assignment] # set in __post_init__
|
|
124
|
+
request: Callable[..., dict[str, Any]] | None = None
|
|
125
|
+
sleep: Callable[[float], None] | None = None
|
|
126
|
+
|
|
127
|
+
def __post_init__(self) -> None:
|
|
128
|
+
if self.touched is None:
|
|
129
|
+
self.touched = set()
|
|
130
|
+
|
|
131
|
+
def _http(self, path: str, payload: dict[str, Any], timeout: float, *, idempotency_key: str | None = None) -> dict[str, Any]:
|
|
132
|
+
url = f"{self.os_url.rstrip('/')}{path}"
|
|
133
|
+
headers = {
|
|
134
|
+
"Authorization": f"Bearer {self.api_key}",
|
|
135
|
+
"Content-Type": "application/json",
|
|
136
|
+
"Accept": "application/json",
|
|
137
|
+
}
|
|
138
|
+
if idempotency_key:
|
|
139
|
+
headers["Idempotency-Key"] = idempotency_key
|
|
140
|
+
req = urllib.request.Request(
|
|
141
|
+
url,
|
|
142
|
+
data=json.dumps(payload).encode("utf-8"),
|
|
143
|
+
headers=headers,
|
|
144
|
+
method="POST",
|
|
145
|
+
)
|
|
146
|
+
try:
|
|
147
|
+
for _name, _value in _connection_headers(req.full_url).items():
|
|
148
|
+
req.add_unredirected_header(_name, _value)
|
|
149
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
150
|
+
envelope = json.loads(resp.read().decode("utf-8"))
|
|
151
|
+
except urllib.error.HTTPError as exc:
|
|
152
|
+
detail = exc.read().decode("utf-8", errors="replace")[:500]
|
|
153
|
+
raise ReviewMiningError(f"OS {path} failed: HTTP {exc.code} {detail}") from exc
|
|
154
|
+
except urllib.error.URLError as exc:
|
|
155
|
+
raise ReviewMiningError(f"OS {path} unreachable: {exc.reason}") from exc
|
|
156
|
+
data = envelope.get("data")
|
|
157
|
+
if envelope.get("code") not in (None, 200) or not isinstance(data, dict):
|
|
158
|
+
raise ReviewMiningError(f"OS {path} returned no data: {str(envelope)[:300]}")
|
|
159
|
+
return data
|
|
160
|
+
|
|
161
|
+
def run_actor(self, actor: str, actor_input: dict[str, Any], limit: int) -> tuple[list[dict[str, Any]], str]:
|
|
162
|
+
payload = {"actor": actor, "input": actor_input, "sync": True, "limit": limit}
|
|
163
|
+
key_raw = json.dumps({"kind": "actor", "payload": payload}, sort_keys=True)
|
|
164
|
+
digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
|
|
165
|
+
cache_file = self.cache_dir / f"{digest}.json"
|
|
166
|
+
self.touched.add(digest)
|
|
167
|
+
if cache_file.is_file():
|
|
168
|
+
try:
|
|
169
|
+
entry = json.loads(cache_file.read_text(encoding="utf-8"))
|
|
170
|
+
except (OSError, json.JSONDecodeError):
|
|
171
|
+
# A torn write must not poison replay forever: discard the
|
|
172
|
+
# corrupt entry and refetch (charged as a new paid start).
|
|
173
|
+
cache_file.unlink(missing_ok=True)
|
|
174
|
+
entry = None
|
|
175
|
+
if entry is not None:
|
|
176
|
+
self.cache_hits += 1
|
|
177
|
+
rows = entry["response"].get("rows")
|
|
178
|
+
return (
|
|
179
|
+
rows if isinstance(rows, list) else []
|
|
180
|
+
), entry.get("retrieved_on") or date.today().isoformat()
|
|
181
|
+
|
|
182
|
+
transport = self.request or self._http
|
|
183
|
+
attempts = 0
|
|
184
|
+
# Salted ONCE per call, before the retry loop: a within-call
|
|
185
|
+
# transient-5xx retry must reuse the SAME wire key (so the gateway
|
|
186
|
+
# can dedupe an ambiguous sync failure into one upstream run), but a
|
|
187
|
+
# fresh call — a prior attempt having already failed and journaled a
|
|
188
|
+
# start for this digest — must get a NEW key. The cache key (digest)
|
|
189
|
+
# stays the pure payload hash so replay lookups are unaffected.
|
|
190
|
+
wire_key = f"{digest}-{self._prior_start_count(digest)}"
|
|
191
|
+
while True:
|
|
192
|
+
# Paid starts count BEFORE the transport call (fail closed) and
|
|
193
|
+
# are journaled durably: a failed start may still have billed.
|
|
194
|
+
if self.actor_runs >= self.max_actor_runs:
|
|
195
|
+
raise ReviewMiningError(
|
|
196
|
+
f"actor-run budget exhausted ({self.max_actor_runs} per "
|
|
197
|
+
"attempt); narrow the competitor list or re-run to "
|
|
198
|
+
"continue from cache"
|
|
199
|
+
)
|
|
200
|
+
self.actor_runs += 1
|
|
201
|
+
self._journal_start(digest)
|
|
202
|
+
try:
|
|
203
|
+
response = transport(
|
|
204
|
+
"/scrape/apify/run", payload, 330.0, idempotency_key=wire_key
|
|
205
|
+
)
|
|
206
|
+
break
|
|
207
|
+
except ReviewMiningError as exc:
|
|
208
|
+
if attempts >= 1 or not re.search(r"HTTP 5\d\d", str(exc)):
|
|
209
|
+
raise
|
|
210
|
+
attempts += 1
|
|
211
|
+
(self.sleep or time.sleep)(ACTOR_RETRY_DELAY_SECONDS)
|
|
212
|
+
retrieved_on = date.today().isoformat()
|
|
213
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
214
|
+
tmp_file = cache_file.with_suffix(".json.tmp")
|
|
215
|
+
tmp_file.write_text(
|
|
216
|
+
json.dumps(
|
|
217
|
+
{
|
|
218
|
+
"kind": "actor",
|
|
219
|
+
"payload": payload,
|
|
220
|
+
"response": response,
|
|
221
|
+
"retrieved_on": retrieved_on,
|
|
222
|
+
}
|
|
223
|
+
),
|
|
224
|
+
encoding="utf-8",
|
|
225
|
+
)
|
|
226
|
+
tmp_file.replace(cache_file)
|
|
227
|
+
rows = response.get("rows")
|
|
228
|
+
return (rows if isinstance(rows, list) else []), retrieved_on
|
|
229
|
+
|
|
230
|
+
def _journal_start(self, digest: str) -> None:
|
|
231
|
+
"""Append-only durable record of every PAID start, successful or not
|
|
232
|
+
— the cache holds only successes, but failed sync starts may still
|
|
233
|
+
have billed upstream."""
|
|
234
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
235
|
+
with (self.cache_dir / "starts.log").open("a", encoding="utf-8") as handle:
|
|
236
|
+
handle.write(f"{date.today().isoformat()} {digest}\n")
|
|
237
|
+
|
|
238
|
+
def _prior_start_count(self, digest: str) -> int:
|
|
239
|
+
"""Paid starts already journaled for this digest by EARLIER
|
|
240
|
+
invocations — the salt for the wire Idempotency-Key. A failed
|
|
241
|
+
attempt's key must never be replayed verbatim: on the gateway (see
|
|
242
|
+
#3715), a client key can poison a wallet hold, and a byte-identical
|
|
243
|
+
retry of a poisoned key fails identically forever."""
|
|
244
|
+
starts_log = self.cache_dir / "starts.log"
|
|
245
|
+
try:
|
|
246
|
+
lines = starts_log.read_text(encoding="utf-8").splitlines()
|
|
247
|
+
except OSError:
|
|
248
|
+
return 0
|
|
249
|
+
return sum(1 for line in lines if line.endswith(f" {digest}"))
|
|
250
|
+
|
|
251
|
+
def ledger_totals(self) -> dict[str, Any]:
|
|
252
|
+
"""Durable provenance over entries THIS execution touched."""
|
|
253
|
+
actor_runs = 0
|
|
254
|
+
dates: list[str] = []
|
|
255
|
+
if self.cache_dir.is_dir():
|
|
256
|
+
for digest in sorted(self.touched):
|
|
257
|
+
entry_path = self.cache_dir / f"{digest}.json"
|
|
258
|
+
try:
|
|
259
|
+
entry = json.loads(entry_path.read_text(encoding="utf-8"))
|
|
260
|
+
except (OSError, json.JSONDecodeError):
|
|
261
|
+
continue
|
|
262
|
+
actor_runs += 1
|
|
263
|
+
if isinstance(entry.get("retrieved_on"), str):
|
|
264
|
+
dates.append(entry["retrieved_on"])
|
|
265
|
+
today = date.today().isoformat()
|
|
266
|
+
starts_log = self.cache_dir / "starts.log"
|
|
267
|
+
try:
|
|
268
|
+
starts_recorded = sum(
|
|
269
|
+
1 for line in starts_log.read_text(encoding="utf-8").splitlines() if line
|
|
270
|
+
)
|
|
271
|
+
except OSError:
|
|
272
|
+
starts_recorded = 0
|
|
273
|
+
return {
|
|
274
|
+
"actor_runs": actor_runs,
|
|
275
|
+
"provider_calls": actor_runs,
|
|
276
|
+
"actor_starts_recorded": starts_recorded,
|
|
277
|
+
"first_retrieved_on": min(dates) if dates else today,
|
|
278
|
+
"last_retrieved_on": max(dates) if dates else today,
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
# ── deterministic helpers ──────────────────────────────────────────────────
|
|
283
|
+
|
|
284
|
+
_KEYWORD_ECONOMICS: ModuleType | None = None
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def registrable_domain(value: str) -> str:
|
|
288
|
+
"""The shared normalization of the Keyword Economics runner: two-level
|
|
289
|
+
public suffixes, sellers' subdomains on course and creator hosts."""
|
|
290
|
+
global _KEYWORD_ECONOMICS
|
|
291
|
+
if _KEYWORD_ECONOMICS is None:
|
|
292
|
+
_KEYWORD_ECONOMICS = _load_script("run-keyword-economics.py", "pmr_keyword_economics_for_review_mining")
|
|
293
|
+
return _KEYWORD_ECONOMICS.registrable_domain(value)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def brand_token(domain: str) -> str:
|
|
297
|
+
stem = registrable_domain(domain).split(".")[0]
|
|
298
|
+
return re.sub(r"[^a-z0-9]", "", stem)
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _normalize_name(value: str) -> str:
|
|
302
|
+
return re.sub(r"[^a-z0-9]", "", (value or "").lower())
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _clean(value: Any) -> Any:
|
|
306
|
+
"""The G2 actor serializes empty fields as the literal string 'None' and
|
|
307
|
+
numbers as strings ('starRating': '1') — validated on a live probe."""
|
|
308
|
+
if value is None:
|
|
309
|
+
return None
|
|
310
|
+
if isinstance(value, str) and value.strip() in ("", "None", "null"):
|
|
311
|
+
return None
|
|
312
|
+
return value
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _to_int(value: Any) -> int | None:
|
|
316
|
+
value = _clean(value)
|
|
317
|
+
if value is None:
|
|
318
|
+
return None
|
|
319
|
+
try:
|
|
320
|
+
return int(float(value))
|
|
321
|
+
except (TypeError, ValueError):
|
|
322
|
+
return None
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _switched_from(value: Any) -> Any:
|
|
326
|
+
"""G2's switchedFromOtherProduct is a QUESTIONNAIRE answer observed live
|
|
327
|
+
as 'yes'/'no'/'unknown' (occasionally an actual product name). Only a
|
|
328
|
+
real product name is a switching source; 'yes' means the story lives in
|
|
329
|
+
switchedReason."""
|
|
330
|
+
value = _clean(value)
|
|
331
|
+
if value is None:
|
|
332
|
+
return None
|
|
333
|
+
lowered = str(value).strip().lower()
|
|
334
|
+
if lowered in ("no", "unknown", "none", "n/a", "na"):
|
|
335
|
+
return None
|
|
336
|
+
if lowered == "yes":
|
|
337
|
+
return "yes"
|
|
338
|
+
return str(value).strip()
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _to_bool(value: Any) -> bool | None:
|
|
342
|
+
"""Actors serialize booleans as strings ('False') — bool('False') is
|
|
343
|
+
True in Python, so parse explicitly (validated on a live probe)."""
|
|
344
|
+
value = _clean(value)
|
|
345
|
+
if value is None:
|
|
346
|
+
return None
|
|
347
|
+
if isinstance(value, bool):
|
|
348
|
+
return value
|
|
349
|
+
return str(value).strip().lower() in ("true", "1", "yes")
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _trustpilot_reviewed_domain(business_url: str, business_name: str) -> str:
|
|
353
|
+
"""businessUrl is the TRUSTPILOT page (trustpilot.com/review/<domain>),
|
|
354
|
+
not the company site — extract the reviewed domain from the path."""
|
|
355
|
+
path = urlparse(str(business_url or "")).path
|
|
356
|
+
if "/review/" in path:
|
|
357
|
+
return registrable_domain(path.rsplit("/review/", 1)[-1])
|
|
358
|
+
return registrable_domain(business_name or "")
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def parse_terms(raw: str, cap: int) -> list[str]:
|
|
362
|
+
seen: dict[str, None] = {}
|
|
363
|
+
for part in re.split(r"[,;\n]", raw or ""):
|
|
364
|
+
term = part.strip().lower()
|
|
365
|
+
if term and term not in seen:
|
|
366
|
+
seen[term] = None
|
|
367
|
+
return list(seen)[:cap]
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def parse_slug_overrides(raw: str) -> dict[str, str]:
|
|
371
|
+
"""Parse --g2-slugs 'domain=slug,other.com=other-slug' overrides."""
|
|
372
|
+
overrides: dict[str, str] = {}
|
|
373
|
+
for part in re.split(r"[,;\n]", raw or ""):
|
|
374
|
+
if "=" not in part:
|
|
375
|
+
continue
|
|
376
|
+
domain, slug = part.split("=", 1)
|
|
377
|
+
domain = registrable_domain(domain.strip())
|
|
378
|
+
slug = slug.strip().lower()
|
|
379
|
+
if domain and slug:
|
|
380
|
+
overrides[domain] = slug
|
|
381
|
+
return overrides
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def domains_from_keyword_evidence(workspace: Path, cap: int, exclude=frozenset()) -> list[str]:
|
|
386
|
+
evidence = _load_script("research-evidence.py", "pmr_evidence")
|
|
387
|
+
return evidence.keyword_inputs(workspace, consumer=SECTION_ID, row_type="competitor_domain", field="domain", cap=cap, exclude=exclude)
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def domains_from_competitor_set(workspace: Path, cap: int) -> list[str]:
|
|
391
|
+
"""Tiered competitors from a completed Competitor Set section, in rank order."""
|
|
392
|
+
evidence = _load_script("research-evidence.py", "pmr_evidence")
|
|
393
|
+
return evidence.competitor_set_inputs(workspace, consumer=SECTION_ID, cap=cap)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def domains_excluded_by_competitor_set(workspace: Path) -> set[str]:
|
|
397
|
+
"""Domains the completed Competitor Set rejected; never refilled automatically."""
|
|
398
|
+
evidence = _load_script("research-evidence.py", "pmr_evidence")
|
|
399
|
+
return evidence.competitor_set_excluded_domains(workspace)
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def normalize_g2_review(row: dict[str, Any], domain: str, slug: str) -> dict[str, Any] | None:
|
|
403
|
+
stars = _to_int(row.get("starRating"))
|
|
404
|
+
text = " ".join(str(_clean(row.get("reviewText")) or "").split())
|
|
405
|
+
if stars is None or not text:
|
|
406
|
+
return None
|
|
407
|
+
billing_flag = bool(_BILLING_RE.search(text)) # classify BEFORE truncation
|
|
408
|
+
switched_reason = " ".join(str(_clean(row.get("switchedReason")) or "").split())
|
|
409
|
+
return {
|
|
410
|
+
"type": "review",
|
|
411
|
+
"platform": "g2",
|
|
412
|
+
"domain": domain,
|
|
413
|
+
"product_slug": _clean(row.get("productSlug")) or slug,
|
|
414
|
+
"product_name": _clean(row.get("productName")),
|
|
415
|
+
"stars": stars,
|
|
416
|
+
"nps": _to_int(row.get("nps")),
|
|
417
|
+
"title": (str(_clean(row.get("title")) or "").strip() or None),
|
|
418
|
+
"text": text[:TEXT_TRUNCATE],
|
|
419
|
+
"date": str(_clean(row.get("publishedAt")) or "")[:10] or None,
|
|
420
|
+
"company_segment_code": _clean(row.get("companySegment")),
|
|
421
|
+
"industry_code": _clean(row.get("industry")),
|
|
422
|
+
"love_theme": _clean(row.get("loveTheme")),
|
|
423
|
+
"hate_theme": _clean(row.get("hateTheme")),
|
|
424
|
+
"switched_from": _switched_from(row.get("switchedFromOtherProduct")),
|
|
425
|
+
"switched_reason": switched_reason[:TEXT_TRUNCATE] or None,
|
|
426
|
+
"billing_pricing_complaint": billing_flag,
|
|
427
|
+
"helpful_votes": _to_int(row.get("helpfulVotes")),
|
|
428
|
+
"url": _clean(row.get("url")),
|
|
429
|
+
"tier": "A",
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def normalize_trustpilot_review(row: dict[str, Any], domain: str) -> dict[str, Any] | None:
|
|
434
|
+
stars = _to_int(row.get("rating"))
|
|
435
|
+
text = " ".join(str(_clean(row.get("text")) or "").split())
|
|
436
|
+
if stars is None or not text:
|
|
437
|
+
return None
|
|
438
|
+
billing_flag = bool(_BILLING_RE.search(text)) # classify BEFORE truncation
|
|
439
|
+
business = _trustpilot_reviewed_domain(
|
|
440
|
+
str(row.get("businessUrl") or ""), str(row.get("businessName") or "")
|
|
441
|
+
)
|
|
442
|
+
if business and business != registrable_domain(domain):
|
|
443
|
+
return None # regional sibling or wrong company — identity is by domain
|
|
444
|
+
return {
|
|
445
|
+
"type": "review",
|
|
446
|
+
"platform": "trustpilot",
|
|
447
|
+
"domain": domain,
|
|
448
|
+
"stars": stars,
|
|
449
|
+
"title": (str(row.get("title") or "").strip() or None),
|
|
450
|
+
"text": text[:TEXT_TRUNCATE],
|
|
451
|
+
"date": str(row.get("publishedDate") or "")[:10] or None,
|
|
452
|
+
"verified": _to_bool(row.get("isVerified")),
|
|
453
|
+
"reviewer_country": row.get("reviewerCountry"),
|
|
454
|
+
"has_company_reply": bool((row.get("reply") or {}).get("text")) if isinstance(row.get("reply"), dict) else False,
|
|
455
|
+
"billing_pricing_complaint": billing_flag,
|
|
456
|
+
"url": row.get("url"),
|
|
457
|
+
"tier": "A",
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def g2_expected_names(domain: str, slug: str) -> set[str]:
|
|
462
|
+
"""Accepted normalized product names for a competitor: the brand token,
|
|
463
|
+
brand+domain ('mondaycom' for monday.com), and the RESOLVED slug — so a
|
|
464
|
+
legitimate --g2-slugs override ('usemotion.com=motion' with productName
|
|
465
|
+
'Motion') validates instead of being rejected."""
|
|
466
|
+
return {
|
|
467
|
+
brand_token(domain),
|
|
468
|
+
_normalize_name(registrable_domain(domain)),
|
|
469
|
+
_normalize_name(slug),
|
|
470
|
+
} - {""}
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def g2_row_identity_ok(review: dict[str, Any], expected: set[str], slug: str) -> bool:
|
|
474
|
+
"""Per-ROW identity (one matching name must not admit a mixed batch):
|
|
475
|
+
the row's product name matches an expected form (or a '(formerly X)'
|
|
476
|
+
rename of one), or the row carries no name but echoes the requested
|
|
477
|
+
slug. Rows with neither are rejected individually."""
|
|
478
|
+
name = review.get("product_name")
|
|
479
|
+
if name:
|
|
480
|
+
normalized = _normalize_name(str(name))
|
|
481
|
+
if normalized in expected:
|
|
482
|
+
return True
|
|
483
|
+
if "formerly" in str(name).lower() and any(
|
|
484
|
+
normalized.startswith(candidate) for candidate in expected
|
|
485
|
+
):
|
|
486
|
+
return True
|
|
487
|
+
return False
|
|
488
|
+
return _normalize_name(str(review.get("product_slug") or "")) == _normalize_name(slug)
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def summarize_reviews(
|
|
492
|
+
domain: str, platform: str, low: list[dict[str, Any]], high: list[dict[str, Any]]
|
|
493
|
+
) -> dict[str, Any]:
|
|
494
|
+
hate = Counter(r["hate_theme"] for r in low if r.get("hate_theme"))
|
|
495
|
+
love = Counter(r["love_theme"] for r in high if r.get("love_theme"))
|
|
496
|
+
switched = [
|
|
497
|
+
{"from": r["switched_from"], "reason": r.get("switched_reason")}
|
|
498
|
+
for r in low + high
|
|
499
|
+
if r.get("switched_from")
|
|
500
|
+
and (r["switched_from"] != "yes" or r.get("switched_reason"))
|
|
501
|
+
]
|
|
502
|
+
billing_complaints = sum(1 for r in low if r.get("billing_pricing_complaint"))
|
|
503
|
+
segments = Counter(
|
|
504
|
+
r["company_segment_code"] for r in low + high if r.get("company_segment_code")
|
|
505
|
+
)
|
|
506
|
+
return {
|
|
507
|
+
"type": "review_summary",
|
|
508
|
+
"domain": domain,
|
|
509
|
+
"platform": platform,
|
|
510
|
+
"low_star_reviews": len(low),
|
|
511
|
+
"high_star_reviews": len(high),
|
|
512
|
+
"billing_pricing_complaints": billing_complaints,
|
|
513
|
+
"hate_themes": dict(hate.most_common(8)),
|
|
514
|
+
"love_themes": dict(love.most_common(8)),
|
|
515
|
+
"switched_from": switched[:10],
|
|
516
|
+
"company_segment_codes": dict(segments.most_common(6)),
|
|
517
|
+
"tier": "A",
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
# ── rendering ──────────────────────────────────────────────────────────────
|
|
522
|
+
|
|
523
|
+
def _excerpt_row(review: dict[str, Any]) -> str:
|
|
524
|
+
text = (review.get("text") or "").replace("|", "/")
|
|
525
|
+
cell = f"{text[:140]}"
|
|
526
|
+
if review.get("url"):
|
|
527
|
+
cell = f"[{cell}]({review['url']})"
|
|
528
|
+
date_str = review.get("date") or "—"
|
|
529
|
+
extra = review.get("hate_theme") or review.get("love_theme") or "—"
|
|
530
|
+
return f"| {review['domain']} | {review['platform']} | {review['stars']}★ | {date_str} | {extra} | {cell} |"
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def render_section(
|
|
534
|
+
*,
|
|
535
|
+
domains: list[str],
|
|
536
|
+
domain_sources: dict[str, str],
|
|
537
|
+
summaries: list[dict[str, Any]],
|
|
538
|
+
low_reviews: list[dict[str, Any]],
|
|
539
|
+
high_reviews: list[dict[str, Any]],
|
|
540
|
+
excluded_platforms: list[dict[str, Any]],
|
|
541
|
+
totals: dict[str, Any],
|
|
542
|
+
retrieved_on: str,
|
|
543
|
+
) -> str:
|
|
544
|
+
lines: list[str] = []
|
|
545
|
+
add = lines.append
|
|
546
|
+
|
|
547
|
+
add(
|
|
548
|
+
f"Customer reviews of paid competitor products retrieved {retrieved_on} "
|
|
549
|
+
"from G2 (B2B reviews with company-segment and switching data; "
|
|
550
|
+
"slug-keyed, per-row identity-checked against the brand) and "
|
|
551
|
+
"Trustpilot (consumer/prosumer reviews; DOMAIN-keyed). Two pulls per "
|
|
552
|
+
"competitor and platform: lowest-rated reviews come from reviewers "
|
|
553
|
+
"of a paid product reporting unmet needs (offer wedges), top-rated "
|
|
554
|
+
"reviews carry the exact language of value its users speak. No "
|
|
555
|
+
"per-reviewer payment signal exists — Trustpilot rows carry a "
|
|
556
|
+
"verified flag, G2 rows do not — so no claim is made that reviewers "
|
|
557
|
+
"themselves paid. Analysis uses only platform-recorded fields over "
|
|
558
|
+
"exactly the rows persisted in the evidence file; hypothesis "
|
|
559
|
+
"synthesis happens downstream."
|
|
560
|
+
)
|
|
561
|
+
add("")
|
|
562
|
+
add(
|
|
563
|
+
"Competitors examined: "
|
|
564
|
+
+ ", ".join(f"`{d}` ({domain_sources.get(d, 'explicit')})" for d in domains)
|
|
565
|
+
+ "."
|
|
566
|
+
)
|
|
567
|
+
add("")
|
|
568
|
+
|
|
569
|
+
add("### Review coverage")
|
|
570
|
+
add("")
|
|
571
|
+
add("| Domain | Platform | 1–3★ collected | 5★ collected | Billing/pricing complaints (1–3★) |")
|
|
572
|
+
add("| --- | --- | ---: | ---: | ---: |")
|
|
573
|
+
for summary in summaries:
|
|
574
|
+
add(
|
|
575
|
+
f"| {summary['domain']} | {summary['platform']} | "
|
|
576
|
+
f"{summary['low_star_reviews']} | {summary['high_star_reviews']} | "
|
|
577
|
+
f"{summary['billing_pricing_complaints']} |"
|
|
578
|
+
)
|
|
579
|
+
add("")
|
|
580
|
+
if excluded_platforms:
|
|
581
|
+
add(
|
|
582
|
+
"Excluded from analysis (identity not verified): "
|
|
583
|
+
+ "; ".join(
|
|
584
|
+
f"{row['domain']} on {row['platform']} ({row['reason']})"
|
|
585
|
+
for row in excluded_platforms
|
|
586
|
+
)
|
|
587
|
+
+ "."
|
|
588
|
+
)
|
|
589
|
+
add("")
|
|
590
|
+
|
|
591
|
+
themed = [s for s in summaries if s["hate_themes"] or s["switched_from"]]
|
|
592
|
+
if themed:
|
|
593
|
+
add("### Complaint themes and switching (as recorded by the platform)")
|
|
594
|
+
add("")
|
|
595
|
+
for summary in themed:
|
|
596
|
+
if summary["hate_themes"]:
|
|
597
|
+
themes = ", ".join(
|
|
598
|
+
f"{theme} × {count}" for theme, count in summary["hate_themes"].items()
|
|
599
|
+
)
|
|
600
|
+
add(f"- `{summary['domain']}` ({summary['platform']}) hate themes: {themes}.")
|
|
601
|
+
for switch in summary["switched_from"][:5]:
|
|
602
|
+
reason = f" — “{switch['reason']}”" if switch.get("reason") else ""
|
|
603
|
+
source = (
|
|
604
|
+
"another tool"
|
|
605
|
+
if switch["from"] == "yes"
|
|
606
|
+
else switch["from"]
|
|
607
|
+
)
|
|
608
|
+
add(
|
|
609
|
+
f"- `{summary['domain']}`: reviewer switched from "
|
|
610
|
+
f"{source}{reason}."
|
|
611
|
+
)
|
|
612
|
+
add("")
|
|
613
|
+
|
|
614
|
+
if low_reviews:
|
|
615
|
+
add("### Lowest-rated excerpts (unmet needs)")
|
|
616
|
+
add("")
|
|
617
|
+
add("| Domain | Platform | Stars | Date | Platform theme | Excerpt |")
|
|
618
|
+
add("| --- | --- | --- | --- | --- | --- |")
|
|
619
|
+
shown: dict[str, int] = {}
|
|
620
|
+
for review in low_reviews:
|
|
621
|
+
key = f"{review['domain']}|{review['platform']}"
|
|
622
|
+
if shown.get(key, 0) >= 4:
|
|
623
|
+
continue
|
|
624
|
+
shown[key] = shown.get(key, 0) + 1
|
|
625
|
+
add(_excerpt_row(review))
|
|
626
|
+
add("")
|
|
627
|
+
|
|
628
|
+
if high_reviews:
|
|
629
|
+
add("### Value language (top-rated excerpts)")
|
|
630
|
+
add("")
|
|
631
|
+
add("| Domain | Platform | Stars | Date | Platform theme | Excerpt |")
|
|
632
|
+
add("| --- | --- | --- | --- | --- | --- |")
|
|
633
|
+
shown = {}
|
|
634
|
+
for review in high_reviews:
|
|
635
|
+
key = f"{review['domain']}|{review['platform']}"
|
|
636
|
+
if shown.get(key, 0) >= 3:
|
|
637
|
+
continue
|
|
638
|
+
shown[key] = shown.get(key, 0) + 1
|
|
639
|
+
add(_excerpt_row(review))
|
|
640
|
+
add("")
|
|
641
|
+
|
|
642
|
+
add("### Method and coverage")
|
|
643
|
+
add("")
|
|
644
|
+
add(
|
|
645
|
+
f"- Provider calls recorded: {totals['actor_runs']} review-platform "
|
|
646
|
+
"actor run(s), accumulated across attempts; cached replays add no "
|
|
647
|
+
"new spend."
|
|
648
|
+
)
|
|
649
|
+
add(
|
|
650
|
+
f"- Caps: {MAX_COMPETITORS} competitors, {LOW_REVIEWS_CAP} lowest-rated "
|
|
651
|
+
f"and {HIGH_REVIEWS_CAP} top-rated reviews per competitor-platform, "
|
|
652
|
+
f"{MAX_ACTOR_RUNS_PER_ATTEMPT} actor runs per attempt."
|
|
653
|
+
)
|
|
654
|
+
add(
|
|
655
|
+
"- G2 company-segment and industry values are G2's integer codes "
|
|
656
|
+
"(kept raw in evidence). Review volumes reflect what the platforms "
|
|
657
|
+
"serve, are biased toward products that solicit reviews, and an "
|
|
658
|
+
"empty platform is weak evidence for small competitors. Not covered: "
|
|
659
|
+
"Capterra/GetApp, app stores, Shopify/Chrome/WordPress directories."
|
|
660
|
+
)
|
|
661
|
+
add("")
|
|
662
|
+
add("### Sources")
|
|
663
|
+
add("")
|
|
664
|
+
add(
|
|
665
|
+
f"- G2 via Apify `{G2_ACTOR}`; Trustpilot via Apify "
|
|
666
|
+
f"`{TRUSTPILOT_ACTOR}` — retrieved {retrieved_on}; the stored review "
|
|
667
|
+
f"rows in `{EVIDENCE_REL.as_posix()}` are exactly the set the "
|
|
668
|
+
"aggregates above are computed from (top rows per competitor-"
|
|
669
|
+
"platform by helpfulness/recency; texts truncated for storage, "
|
|
670
|
+
"billing flags classified on full text before truncation)."
|
|
671
|
+
)
|
|
672
|
+
return "\n".join(lines).strip() + "\n"
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
# ── orchestration ──────────────────────────────────────────────────────────
|
|
676
|
+
|
|
677
|
+
@dataclass
|
|
678
|
+
class RunInputs:
|
|
679
|
+
domains: list[str]
|
|
680
|
+
domain_sources: dict[str, str]
|
|
681
|
+
g2_slugs: dict[str, str]
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
def _call_with_context(client: OsClient, domain: str, label: str, fn: Callable[[], Any]) -> Any:
|
|
685
|
+
"""Re-raise a provider-call failure with WHICH competitor/call failed and
|
|
686
|
+
how much of the attempt's paid budget is already spent — the raw gateway
|
|
687
|
+
envelope alone doesn't say either, so a human can't tell whether a retry
|
|
688
|
+
is even possible without this."""
|
|
689
|
+
try:
|
|
690
|
+
return fn()
|
|
691
|
+
except ReviewMiningError as exc:
|
|
692
|
+
raise ReviewMiningError(
|
|
693
|
+
f"{label} for `{domain}` failed: {exc} — this attempt had "
|
|
694
|
+
f"already started {client.actor_runs}/{client.max_actor_runs} "
|
|
695
|
+
"actor run(s); see GET /me/limits?resource=apify:runs for "
|
|
696
|
+
"remaining daily quota."
|
|
697
|
+
) from exc
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
def run_pipeline(client: OsClient, inputs: RunInputs) -> dict[str, Any]:
|
|
701
|
+
if not inputs.domains:
|
|
702
|
+
raise ReviewMiningError(
|
|
703
|
+
"no competitor domains available: run the Keyword Economics "
|
|
704
|
+
"section first (its evidence feeds this section) or pass "
|
|
705
|
+
"--domains explicitly"
|
|
706
|
+
)
|
|
707
|
+
|
|
708
|
+
summaries: list[dict[str, Any]] = []
|
|
709
|
+
stored_low: list[dict[str, Any]] = []
|
|
710
|
+
stored_high: list[dict[str, Any]] = []
|
|
711
|
+
excluded_platforms: list[dict[str, Any]] = []
|
|
712
|
+
|
|
713
|
+
for domain in inputs.domains:
|
|
714
|
+
brand = brand_token(domain)
|
|
715
|
+
slug = inputs.g2_slugs.get(registrable_domain(domain), brand)
|
|
716
|
+
|
|
717
|
+
g2_low_rows, _ = _call_with_context(
|
|
718
|
+
client,
|
|
719
|
+
domain,
|
|
720
|
+
"G2 (lowest-rated)",
|
|
721
|
+
lambda: client.run_actor(
|
|
722
|
+
G2_ACTOR,
|
|
723
|
+
{
|
|
724
|
+
"mode": "product_reviews",
|
|
725
|
+
"productUrls": [slug],
|
|
726
|
+
"maxReviews": LOW_REVIEWS_CAP,
|
|
727
|
+
"sortReviews": "rating_low",
|
|
728
|
+
},
|
|
729
|
+
limit=LOW_REVIEWS_CAP,
|
|
730
|
+
),
|
|
731
|
+
)
|
|
732
|
+
g2_high_rows, _ = _call_with_context(
|
|
733
|
+
client,
|
|
734
|
+
domain,
|
|
735
|
+
"G2 (top-rated)",
|
|
736
|
+
lambda: client.run_actor(
|
|
737
|
+
G2_ACTOR,
|
|
738
|
+
{
|
|
739
|
+
"mode": "product_reviews",
|
|
740
|
+
"productUrls": [slug],
|
|
741
|
+
"maxReviews": HIGH_REVIEWS_CAP,
|
|
742
|
+
"sortReviews": "rating_high",
|
|
743
|
+
},
|
|
744
|
+
limit=HIGH_REVIEWS_CAP,
|
|
745
|
+
),
|
|
746
|
+
)
|
|
747
|
+
g2_low_all = [
|
|
748
|
+
r for r in (normalize_g2_review(x, domain, slug) for x in g2_low_rows) if r
|
|
749
|
+
]
|
|
750
|
+
g2_high_all = [
|
|
751
|
+
r for r in (normalize_g2_review(x, domain, slug) for x in g2_high_rows) if r
|
|
752
|
+
]
|
|
753
|
+
expected_names = g2_expected_names(domain, slug)
|
|
754
|
+
g2_low = [r for r in g2_low_all if g2_row_identity_ok(r, expected_names, slug)]
|
|
755
|
+
g2_high = [r for r in g2_high_all if g2_row_identity_ok(r, expected_names, slug)]
|
|
756
|
+
g2_identity_rejected = (len(g2_low_all) - len(g2_low)) + (
|
|
757
|
+
len(g2_high_all) - len(g2_high)
|
|
758
|
+
)
|
|
759
|
+
if (g2_low_all or g2_high_all) and not (g2_low or g2_high):
|
|
760
|
+
excluded_platforms.append(
|
|
761
|
+
{
|
|
762
|
+
"type": "excluded_platform",
|
|
763
|
+
"domain": domain,
|
|
764
|
+
"platform": "g2",
|
|
765
|
+
"slug": slug,
|
|
766
|
+
"reason": "slug_resolves_to_different_product",
|
|
767
|
+
"tier": "B",
|
|
768
|
+
}
|
|
769
|
+
)
|
|
770
|
+
# Sort filters are best-effort: keep only genuinely low rows and
|
|
771
|
+
# EXACTLY five-star rows (the report labels the column 5-star).
|
|
772
|
+
g2_low = [r for r in g2_low if r["stars"] in LOW_STARS]
|
|
773
|
+
g2_high = [r for r in g2_high if r["stars"] == 5]
|
|
774
|
+
|
|
775
|
+
tp_low_rows, _ = _call_with_context(
|
|
776
|
+
client,
|
|
777
|
+
domain,
|
|
778
|
+
"Trustpilot (lowest-rated)",
|
|
779
|
+
lambda: client.run_actor(
|
|
780
|
+
TRUSTPILOT_ACTOR,
|
|
781
|
+
{
|
|
782
|
+
"startUrls": [registrable_domain(domain)],
|
|
783
|
+
"filterStars": ["1", "2", "3"],
|
|
784
|
+
"maxItems": LOW_REVIEWS_CAP,
|
|
785
|
+
"sortBy": "recent",
|
|
786
|
+
"filterLanguages": ["en"],
|
|
787
|
+
},
|
|
788
|
+
limit=LOW_REVIEWS_CAP,
|
|
789
|
+
),
|
|
790
|
+
)
|
|
791
|
+
tp_high_rows, _ = _call_with_context(
|
|
792
|
+
client,
|
|
793
|
+
domain,
|
|
794
|
+
"Trustpilot (top-rated)",
|
|
795
|
+
lambda: client.run_actor(
|
|
796
|
+
TRUSTPILOT_ACTOR,
|
|
797
|
+
{
|
|
798
|
+
"startUrls": [registrable_domain(domain)],
|
|
799
|
+
"filterStars": ["5"],
|
|
800
|
+
"maxItems": HIGH_REVIEWS_CAP,
|
|
801
|
+
"sortBy": "recent",
|
|
802
|
+
"filterLanguages": ["en"],
|
|
803
|
+
},
|
|
804
|
+
limit=HIGH_REVIEWS_CAP,
|
|
805
|
+
),
|
|
806
|
+
)
|
|
807
|
+
tp_low = [
|
|
808
|
+
r
|
|
809
|
+
for r in (normalize_trustpilot_review(x, domain) for x in tp_low_rows)
|
|
810
|
+
if r and r["stars"] in LOW_STARS
|
|
811
|
+
]
|
|
812
|
+
tp_high = [
|
|
813
|
+
r
|
|
814
|
+
for r in (normalize_trustpilot_review(x, domain) for x in tp_high_rows)
|
|
815
|
+
if r and r["stars"] == 5
|
|
816
|
+
]
|
|
817
|
+
|
|
818
|
+
def _stored(reviews: list[dict[str, Any]], cap: int) -> list[dict[str, Any]]:
|
|
819
|
+
ranked = sorted(
|
|
820
|
+
reviews,
|
|
821
|
+
key=lambda r: (r.get("helpful_votes") or 0, r.get("date") or ""),
|
|
822
|
+
reverse=True,
|
|
823
|
+
)
|
|
824
|
+
return ranked[:cap]
|
|
825
|
+
|
|
826
|
+
# Aggregates are computed over EXACTLY the rows persisted in the
|
|
827
|
+
# evidence file, so the published numbers are reproducible from it.
|
|
828
|
+
g2_low_stored = _stored(g2_low, STORED_LOW_PER_PLATFORM)
|
|
829
|
+
g2_high_stored = _stored(g2_high, STORED_HIGH_PER_PLATFORM)
|
|
830
|
+
tp_low_stored = _stored(tp_low, STORED_LOW_PER_PLATFORM)
|
|
831
|
+
tp_high_stored = _stored(tp_high, STORED_HIGH_PER_PLATFORM)
|
|
832
|
+
|
|
833
|
+
g2_summary = summarize_reviews(domain, "g2", g2_low_stored, g2_high_stored)
|
|
834
|
+
g2_summary["identity_rejected_rows"] = g2_identity_rejected
|
|
835
|
+
summaries.append(g2_summary)
|
|
836
|
+
summaries.append(
|
|
837
|
+
summarize_reviews(domain, "trustpilot", tp_low_stored, tp_high_stored)
|
|
838
|
+
)
|
|
839
|
+
|
|
840
|
+
stored_low.extend(g2_low_stored)
|
|
841
|
+
stored_low.extend(tp_low_stored)
|
|
842
|
+
stored_high.extend(g2_high_stored)
|
|
843
|
+
stored_high.extend(tp_high_stored)
|
|
844
|
+
|
|
845
|
+
totals = client.ledger_totals()
|
|
846
|
+
retrieved_on = (
|
|
847
|
+
totals["first_retrieved_on"]
|
|
848
|
+
if totals["first_retrieved_on"] == totals["last_retrieved_on"]
|
|
849
|
+
else f"{totals['first_retrieved_on']}..{totals['last_retrieved_on']}"
|
|
850
|
+
)
|
|
851
|
+
|
|
852
|
+
domain_rows = [
|
|
853
|
+
{
|
|
854
|
+
"type": "competitor_domain",
|
|
855
|
+
"domain": domain,
|
|
856
|
+
"source": inputs.domain_sources.get(domain, "explicit"),
|
|
857
|
+
"tier": "A",
|
|
858
|
+
}
|
|
859
|
+
for domain in inputs.domains
|
|
860
|
+
]
|
|
861
|
+
|
|
862
|
+
evidence = {
|
|
863
|
+
"schema_version": SCHEMA_VERSION,
|
|
864
|
+
"section_id": SECTION_ID,
|
|
865
|
+
"generated_by": Path(__file__).name,
|
|
866
|
+
"retrieved_on": retrieved_on,
|
|
867
|
+
"inputs": {
|
|
868
|
+
"domains": inputs.domains,
|
|
869
|
+
"domain_sources": inputs.domain_sources,
|
|
870
|
+
"g2_slugs": inputs.g2_slugs,
|
|
871
|
+
},
|
|
872
|
+
"rows": (
|
|
873
|
+
domain_rows
|
|
874
|
+
+ summaries
|
|
875
|
+
+ stored_low
|
|
876
|
+
+ stored_high
|
|
877
|
+
+ excluded_platforms
|
|
878
|
+
),
|
|
879
|
+
"totals": {
|
|
880
|
+
"actor_runs": totals["actor_runs"],
|
|
881
|
+
"provider_calls": totals["provider_calls"],
|
|
882
|
+
},
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
section_markdown = render_section(
|
|
886
|
+
domains=inputs.domains,
|
|
887
|
+
domain_sources=inputs.domain_sources,
|
|
888
|
+
summaries=summaries,
|
|
889
|
+
low_reviews=stored_low,
|
|
890
|
+
high_reviews=stored_high,
|
|
891
|
+
excluded_platforms=excluded_platforms,
|
|
892
|
+
totals=totals,
|
|
893
|
+
retrieved_on=retrieved_on,
|
|
894
|
+
)
|
|
895
|
+
return {
|
|
896
|
+
"evidence": evidence,
|
|
897
|
+
"section_markdown": section_markdown,
|
|
898
|
+
"totals": totals,
|
|
899
|
+
"attempt": {
|
|
900
|
+
"actor_runs": client.actor_runs,
|
|
901
|
+
"cache_hits": client.cache_hits,
|
|
902
|
+
},
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
|
|
906
|
+
def _write_atomic(path: Path, content: str) -> None:
|
|
907
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
908
|
+
tmp = path.with_suffix(path.suffix + ".tmp")
|
|
909
|
+
tmp.write_text(content, encoding="utf-8")
|
|
910
|
+
tmp.replace(path)
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
def run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
|
|
914
|
+
workspace = workspace.resolve()
|
|
915
|
+
io = _load_script("research-artifact-io.py", "pmr_artifact_io")
|
|
916
|
+
with io.section_attempt(workspace, SECTION_ID, context_file=getattr(args, "context_file", None)):
|
|
917
|
+
return _run(workspace, args)
|
|
918
|
+
|
|
919
|
+
|
|
920
|
+
def _run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
|
|
921
|
+
workspace = workspace.resolve()
|
|
922
|
+
runtime = workspace / RUNTIME_REL
|
|
923
|
+
cache_dir = runtime / "cache"
|
|
924
|
+
runtime.mkdir(parents=True, exist_ok=True)
|
|
925
|
+
|
|
926
|
+
api_key = (os.environ.get("OS_API_KEY") or "").strip()
|
|
927
|
+
if not api_key:
|
|
928
|
+
raise ReviewMiningError("OS_API_KEY is required")
|
|
929
|
+
os_url = (os.environ.get("OS_URL") or DEFAULT_OS_URL).strip() or DEFAULT_OS_URL
|
|
930
|
+
|
|
931
|
+
explicit = parse_terms(args.domains or "", MAX_COMPETITORS)
|
|
932
|
+
explicit = [registrable_domain(d) for d in explicit if registrable_domain(d)]
|
|
933
|
+
# Explicit domains first, then the tiered Competitor Set, then the
|
|
934
|
+
# Keyword Economics demand surface; each source is optional.
|
|
935
|
+
tiered = domains_from_competitor_set(workspace, MAX_COMPETITORS)
|
|
936
|
+
# The tiering decisions constrain automatic fallbacks: a domain the
|
|
937
|
+
# Competitor Set rejected is not pulled back from the demand surface.
|
|
938
|
+
# Explicit --domains remain the founder's override.
|
|
939
|
+
rejected = domains_excluded_by_competitor_set(workspace) - set(explicit)
|
|
940
|
+
discovered = domains_from_keyword_evidence(workspace, MAX_COMPETITORS, exclude=rejected)
|
|
941
|
+
|
|
942
|
+
domains: list[str] = []
|
|
943
|
+
domain_sources: dict[str, str] = {}
|
|
944
|
+
for domain in explicit:
|
|
945
|
+
if domain not in domains:
|
|
946
|
+
domains.append(domain)
|
|
947
|
+
domain_sources[domain] = "explicit"
|
|
948
|
+
for source, candidates in (("competitor_set", tiered), ("keyword_economics", discovered)):
|
|
949
|
+
for domain in candidates:
|
|
950
|
+
if len(domains) >= args.max_competitors:
|
|
951
|
+
break
|
|
952
|
+
if domain not in domains:
|
|
953
|
+
domains.append(domain)
|
|
954
|
+
domain_sources[domain] = source
|
|
955
|
+
domains = domains[: args.max_competitors]
|
|
956
|
+
# Four actor runs per competitor: never admit more domains than the
|
|
957
|
+
# budget can finish in one attempt.
|
|
958
|
+
domains = domains[: max(1, args.max_actor_runs // 4)]
|
|
959
|
+
|
|
960
|
+
inputs = RunInputs(
|
|
961
|
+
domains=domains,
|
|
962
|
+
domain_sources=domain_sources,
|
|
963
|
+
g2_slugs=parse_slug_overrides(args.g2_slugs or ""),
|
|
964
|
+
)
|
|
965
|
+
client = OsClient(
|
|
966
|
+
os_url=os_url,
|
|
967
|
+
api_key=api_key,
|
|
968
|
+
cache_dir=cache_dir,
|
|
969
|
+
max_actor_runs=args.max_actor_runs,
|
|
970
|
+
)
|
|
971
|
+
|
|
972
|
+
outcome = run_pipeline(client, inputs)
|
|
973
|
+
|
|
974
|
+
evidence_path = workspace / EVIDENCE_REL
|
|
975
|
+
evidence_json = json.dumps(outcome["evidence"], indent=1, ensure_ascii=False)
|
|
976
|
+
_write_atomic(evidence_path, evidence_json)
|
|
977
|
+
|
|
978
|
+
section_file = runtime / "result.md"
|
|
979
|
+
_write_atomic(section_file, outcome["section_markdown"])
|
|
980
|
+
|
|
981
|
+
upsert = _load_script("upsert-markdown-section.py", "pmr_upsert_markdown_section")
|
|
982
|
+
report_path = workspace / REPORT_REL
|
|
983
|
+
report_path.parent.mkdir(parents=True, exist_ok=True)
|
|
984
|
+
upsert.upsert_markdown_section(report_path, SECTION_HEADING, outcome["section_markdown"])
|
|
985
|
+
|
|
986
|
+
persist = _load_script("persist-research-report.py", "pmr_persist_research_report")
|
|
987
|
+
persistence = persist.persist_research_report(
|
|
988
|
+
workspace,
|
|
989
|
+
commit_message=f"research: update {SECTION_HEADING.lower()} section",
|
|
990
|
+
extra_paths=(Path(_artifact_path("egbe-docs/research/market/evidence/review-mining.json")),),
|
|
991
|
+
)
|
|
992
|
+
|
|
993
|
+
state = {
|
|
994
|
+
"schema_version": SCHEMA_VERSION,
|
|
995
|
+
"section_id": SECTION_ID,
|
|
996
|
+
"status": "completed",
|
|
997
|
+
"domains": inputs.domains,
|
|
998
|
+
"domain_sources": inputs.domain_sources,
|
|
999
|
+
"evidence_path": EVIDENCE_REL.as_posix(),
|
|
1000
|
+
"evidence_sha256": hashlib.sha256(evidence_json.encode("utf-8")).hexdigest(),
|
|
1001
|
+
"result_path": (RUNTIME_REL / "result.md").as_posix(),
|
|
1002
|
+
"result_sha256": hashlib.sha256(outcome["section_markdown"].encode("utf-8")).hexdigest(),
|
|
1003
|
+
"provider_calls": outcome["totals"]["provider_calls"],
|
|
1004
|
+
"actor_runs": outcome["totals"]["actor_runs"],
|
|
1005
|
+
"actor_starts_recorded": outcome["totals"].get("actor_starts_recorded", 0),
|
|
1006
|
+
"attempt_actor_runs": outcome["attempt"]["actor_runs"],
|
|
1007
|
+
"attempt_cache_hits": outcome["attempt"]["cache_hits"],
|
|
1008
|
+
"report_commit_sha": persistence.commit_sha,
|
|
1009
|
+
"publication_mode": persistence.publication_mode,
|
|
1010
|
+
"research_context": _load_script("research-artifact-io.py", "pmr_input_io").context_binding(workspace, SECTION_ID),
|
|
1011
|
+
"error": None,
|
|
1012
|
+
}
|
|
1013
|
+
_write_atomic(runtime / "request.json", json.dumps(state, indent=1))
|
|
1014
|
+
return state
|
|
1015
|
+
|
|
1016
|
+
|
|
1017
|
+
def main() -> int:
|
|
1018
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
1019
|
+
parser.add_argument("--context-file", type=Path, help="Explicit context for this study; no project files are discovered implicitly.")
|
|
1020
|
+
parser.add_argument("--workspace", type=Path, required=True)
|
|
1021
|
+
parser.add_argument(
|
|
1022
|
+
"--domains",
|
|
1023
|
+
default="",
|
|
1024
|
+
help="Comma-separated competitor domains; merged ahead of the "
|
|
1025
|
+
"domains discovered by Keyword Economics.",
|
|
1026
|
+
)
|
|
1027
|
+
parser.add_argument(
|
|
1028
|
+
"--g2-slugs",
|
|
1029
|
+
default="",
|
|
1030
|
+
help="Optional G2 slug overrides as 'domain=slug,other.com=slug2' "
|
|
1031
|
+
"when a competitor's G2 slug differs from its brand token.",
|
|
1032
|
+
)
|
|
1033
|
+
parser.add_argument("--max-competitors", type=int, default=MAX_COMPETITORS)
|
|
1034
|
+
parser.add_argument("--max-actor-runs", type=int, default=MAX_ACTOR_RUNS_PER_ATTEMPT)
|
|
1035
|
+
args = parser.parse_args()
|
|
1036
|
+
|
|
1037
|
+
try:
|
|
1038
|
+
state = run(args.workspace, args)
|
|
1039
|
+
except (ReviewMiningError, ValueError) as exc:
|
|
1040
|
+
print(f"{DISPLAY_NAME}: {exc}", file=sys.stderr)
|
|
1041
|
+
return 1
|
|
1042
|
+
print(
|
|
1043
|
+
f"{DISPLAY_NAME}: completed — {state['actor_runs']} actor run(s), "
|
|
1044
|
+
f"evidence at {state['evidence_path']}, report commit "
|
|
1045
|
+
f"{state['report_commit_sha'][:12]}"
|
|
1046
|
+
)
|
|
1047
|
+
return 0
|
|
1048
|
+
|
|
1049
|
+
|
|
1050
|
+
if __name__ == "__main__":
|
|
1051
|
+
raise SystemExit(main())
|