@egbe-ai/opencode 0.0.0-stage → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +124 -2
- package/index.js +5 -0
- package/package.json +55 -4
- package/plugin.js +230 -0
- package/session-log.js +315 -0
- package/skills/.egbe-catalog/manifest.json +1 -0
- package/skills/egbe-os-analytics/SKILL.md +499 -0
- package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
- package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
- package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
- package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
- package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
- package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
- package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
- package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
- package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
- package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
- package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
- package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
- package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
- package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
- package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
- package/skills/egbe-os-db/SKILL.md +174 -0
- package/skills/egbe-os-deploy/SKILL.md +412 -0
- package/skills/egbe-os-design/SKILL.md +182 -0
- package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
- package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
- package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
- package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
- package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
- package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
- package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
- package/skills/egbe-os-destination-readiness/references/report.md +116 -0
- package/skills/egbe-os-domains/SKILL.md +189 -0
- package/skills/egbe-os-errors/SKILL.md +259 -0
- package/skills/egbe-os-git/SKILL.md +275 -0
- package/skills/egbe-os-git/references/issues.md +33 -0
- package/skills/egbe-os-google-ads/SKILL.md +163 -0
- package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
- package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
- package/skills/egbe-os-google-ads/references/conversions.md +153 -0
- package/skills/egbe-os-google-ads/references/delivery.md +128 -0
- package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
- package/skills/egbe-os-google-ads/references/reporting.md +105 -0
- package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
- package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
- package/skills/egbe-os-google-auth/SKILL.md +285 -0
- package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
- package/skills/egbe-os-images/SKILL.md +163 -0
- package/skills/egbe-os-images/references/stock-images.md +21 -0
- package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
- package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
- package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
- package/skills/egbe-os-llm/SKILL.md +160 -0
- package/skills/egbe-os-mail/SKILL.md +180 -0
- package/skills/egbe-os-me/SKILL.md +119 -0
- package/skills/egbe-os-meta-ads/SKILL.md +114 -0
- package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
- package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
- package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
- package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
- package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
- package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
- package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
- package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
- package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
- package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
- package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
- package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
- package/skills/egbe-os-notifications/SKILL.md +78 -0
- package/skills/egbe-os-openai-ads/SKILL.md +149 -0
- package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
- package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
- package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
- package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
- package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
- package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
- package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
- package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
- package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
- package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
- package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
- package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
- package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
- package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
- package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
- package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
- package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
- package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
- package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
- package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
- package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
- package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
- package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
- package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
- package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
- package/skills/egbe-os-payments/SKILL.md +215 -0
- package/skills/egbe-os-payments/agents/openai.yaml +4 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
- package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
- package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
- package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
- package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
- package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
- package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
- package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
- package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
- package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
- package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
- package/skills/egbe-os-playbook/SKILL.md +79 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
- package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
- package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
- package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
- package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
- package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
- package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
- package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
- package/skills/egbe-os-playbook/playbooks/product.md +67 -0
- package/skills/egbe-os-playbook/references/catalog.md +127 -0
- package/skills/egbe-os-playbook/references/changes.md +167 -0
- package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
- package/skills/egbe-os-product-auth/SKILL.md +151 -0
- package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
- package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
- package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
- package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
- package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
- package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
- package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
- package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
- package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
- package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
- package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
- package/skills/egbe-os-product-build/SKILL.md +111 -0
- package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-build/references/capabilities.md +25 -0
- package/skills/egbe-os-product-build/references/engineering.md +72 -0
- package/skills/egbe-os-product-build/references/execution.md +55 -0
- package/skills/egbe-os-product-build/references/foundations.md +93 -0
- package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
- package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
- package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
- package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
- package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
- package/skills/egbe-os-product-market-research/SKILL.md +159 -0
- package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
- package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
- package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
- package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
- package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
- package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
- package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
- package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
- package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
- package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
- package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
- package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
- package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
- package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
- package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
- package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
- package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
- package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
- package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
- package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
- package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
- package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
- package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
- package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
- package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
- package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
- package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
- package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
- package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
- package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
- package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
- package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
- package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
- package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
- package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
- package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
- package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
- package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
- package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
- package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
- package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
- package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
- package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
- package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
- package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
- package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
- package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
- package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
- package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
- package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
- package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
- package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
- package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
- package/skills/egbe-os-product-report/SKILL.md +150 -0
- package/skills/egbe-os-product-report/references/artifacts.md +84 -0
- package/skills/egbe-os-product-report/references/brief.md +34 -0
- package/skills/egbe-os-product-report/references/daily.md +76 -0
- package/skills/egbe-os-product-report/references/sources.md +68 -0
- package/skills/egbe-os-product-routines/SKILL.md +166 -0
- package/skills/egbe-os-product-routines/references/catalog.md +22 -0
- package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
- package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
- package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
- package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
- package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
- package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
- package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
- package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
- package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
- package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
- package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
- package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
- package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
- package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
- package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
- package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
- package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
- package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
- package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
- package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
- package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
- package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
- package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
- package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
- package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
- package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
- package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
- package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
- package/skills/egbe-os-report-ui/SKILL.md +133 -0
- package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
- package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
- package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
- package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
- package/skills/egbe-os-report-ui/references/components.md +209 -0
- package/skills/egbe-os-research/SKILL.md +125 -0
- package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
- package/skills/egbe-os-scrape/SKILL.md +160 -0
- package/skills/egbe-os-search/SKILL.md +182 -0
- package/skills/egbe-os-search/references/google-places.md +81 -0
- package/skills/egbe-os-support/SKILL.md +149 -0
- package/skills/egbe-os-telemetry/SKILL.md +344 -0
- package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
- package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
- package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
- package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
- package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
- package/skills/egbe-os-tools/SKILL.md +180 -0
- package/skills/egbe-os-tools/references/theona.md +26 -0
- package/skills/egbe-os-wallet/SKILL.md +181 -0
- package/skills/egbe-setup/EGBE.md +484 -0
- package/skills/egbe-setup/SKILL.md +116 -0
- package/skills/egbe-setup/references/project.md +80 -0
- package/state.js +37 -0
- package/update.js +361 -0
|
@@ -0,0 +1,1603 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Run the deterministic Community Demand section against community sources.
|
|
3
|
+
|
|
4
|
+
People asking for tools, describing workarounds, or naming prices in public
|
|
5
|
+
communities are demand evidence that precedes any paid channel: the exact
|
|
6
|
+
pain language feeds offers, and the communities themselves are organic and
|
|
7
|
+
paid placements (subreddit ads target subreddits). This runner collects
|
|
8
|
+
both from two sources:
|
|
9
|
+
|
|
10
|
+
- Hacker News via the keyless Algolia search API (stories + comments;
|
|
11
|
+
free, longitudinal);
|
|
12
|
+
- fresh Reddit (last 12 months) via the Apify actor
|
|
13
|
+
`trudax/reddit-scraper-lite` (posts + comments across all of Reddit —
|
|
14
|
+
the official Reddit API is commercially unviable and is never used);
|
|
15
|
+
- YouTube comments under the top videos per query (OS `/tools/youtube`,
|
|
16
|
+
metered per claw against a fleet-shared Google quota) — best-effort:
|
|
17
|
+
a failure is recorded as an excluded source, never a dead section.
|
|
18
|
+
|
|
19
|
+
Quotes are selected by a fixed, versioned marker-regex list (payment
|
|
20
|
+
language, pain language, tool-seeking language) — deterministic heuristics,
|
|
21
|
+
not comprehension. The community observation table is computed over EXACTLY the
|
|
22
|
+
quote rows persisted in the evidence file, so every published number is
|
|
23
|
+
reproducible from it. No LLM anywhere; synthesis into segment/offer/channel
|
|
24
|
+
hypotheses is a downstream concern. Caching, spend provenance, budgets and
|
|
25
|
+
retries mirror the Competitor Reviews runner.
|
|
26
|
+
|
|
27
|
+
Deliberately excluded (recorded as excluded_source rows): Product Hunt
|
|
28
|
+
(its GraphQL API cannot keyword-search launches — live-probed), Arctic
|
|
29
|
+
Shift (its search API cannot keyword-search across all of Reddit), and
|
|
30
|
+
Stack Exchange (v2 scope).
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import sys as _artifact_sys
|
|
36
|
+
from pathlib import Path as _ArtifactPath
|
|
37
|
+
_artifact_sys.path.insert(0, str(_ArtifactPath(__file__).resolve().parent))
|
|
38
|
+
from egbe_artifacts import artifact_path as _artifact_path
|
|
39
|
+
|
|
40
|
+
import runpy as _egbe_runpy
|
|
41
|
+
from pathlib import Path as _EgbePath
|
|
42
|
+
_egbe_adapter = _EgbePath(__file__).with_name("connection_identity.py")
|
|
43
|
+
_connection_headers = _egbe_runpy.run_path(str(_egbe_adapter))["connection_headers"] if _egbe_adapter.is_file() else lambda _url: {}
|
|
44
|
+
|
|
45
|
+
import argparse
|
|
46
|
+
import hashlib
|
|
47
|
+
import html
|
|
48
|
+
import importlib.util
|
|
49
|
+
import json
|
|
50
|
+
import os
|
|
51
|
+
import re
|
|
52
|
+
import sys
|
|
53
|
+
import time
|
|
54
|
+
import urllib.error
|
|
55
|
+
import urllib.parse
|
|
56
|
+
import urllib.request
|
|
57
|
+
from dataclasses import dataclass
|
|
58
|
+
from datetime import date, datetime, timedelta, timezone
|
|
59
|
+
from pathlib import Path
|
|
60
|
+
from types import ModuleType
|
|
61
|
+
from typing import Any, Callable
|
|
62
|
+
|
|
63
|
+
SCHEMA_VERSION = 1
|
|
64
|
+
SECTION_ID = "community_demand"
|
|
65
|
+
SECTION_HEADING = "Communities analysis"
|
|
66
|
+
# H2s written by bundles that predate the founder-facing rename. The runner
|
|
67
|
+
# folds them into the canonical heading before materializing, so the report
|
|
68
|
+
# never carries this section under two names.
|
|
69
|
+
LEGACY_SECTION_HEADINGS = ("Community demand",)
|
|
70
|
+
DISPLAY_NAME = "Communities Analysis"
|
|
71
|
+
|
|
72
|
+
DEFAULT_OS_URL = "https://os.hz.egbe.dev"
|
|
73
|
+
HN_API_URL = "https://hn.algolia.com/api/v1"
|
|
74
|
+
RUNTIME_REL = Path(_artifact_path("egbe-docs/research/market/work")) / SECTION_ID
|
|
75
|
+
EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/community-demand.json"))
|
|
76
|
+
REPORT_REL = Path(_artifact_path("egbe-docs/research/market/full-report.md"))
|
|
77
|
+
|
|
78
|
+
REDDIT_ACTOR = "trudax/reddit-scraper-lite"
|
|
79
|
+
|
|
80
|
+
# One Reddit actor run, two HN calls, and up to three best-effort YouTube
|
|
81
|
+
# calls (one search + comments of the top videos) per query.
|
|
82
|
+
MAX_QUERIES = 5
|
|
83
|
+
MAX_ACTOR_RUNS_PER_ATTEMPT = 6
|
|
84
|
+
MAX_HN_CALLS_PER_ATTEMPT = 16
|
|
85
|
+
MAX_YT_CALLS_PER_ATTEMPT = 18
|
|
86
|
+
# 60 items fits Apify's 300s sync-endpoint ceiling; 100 exceeded it on a
|
|
87
|
+
# live run (HTTP 408 run-timeout-exceeded). Recall scales via MORE QUERIES,
|
|
88
|
+
# not deeper single runs.
|
|
89
|
+
REDDIT_ITEMS_PER_QUERY = 60
|
|
90
|
+
HN_STORY_HITS = 30
|
|
91
|
+
HN_COMMENT_HITS = 50
|
|
92
|
+
HN_RECENCY_DAYS = 3 * 365
|
|
93
|
+
YT_RECENCY_DAYS = 2 * 365
|
|
94
|
+
YT_SEARCH_RESULTS = 5
|
|
95
|
+
YT_VIDEOS_PER_QUERY = 2
|
|
96
|
+
YT_COMMENTS_PER_VIDEO = 50
|
|
97
|
+
MIN_QUOTE_CHARS = 30
|
|
98
|
+
TEXT_TRUNCATE = 400
|
|
99
|
+
STORED_QUOTES_CAP = 120
|
|
100
|
+
STORED_PER_COMMUNITY_CAP = 15
|
|
101
|
+
|
|
102
|
+
ACTOR_RETRY_DELAY_SECONDS = 20.0
|
|
103
|
+
HN_RETRY_DELAY_SECONDS = 5.0
|
|
104
|
+
|
|
105
|
+
# Versioned deterministic marker list (v1). Quotes are MARKER MATCHES, not
|
|
106
|
+
# classified meaning: the categories name which list matched, nothing more.
|
|
107
|
+
# Payment language must be precise, not broad: a bare dollar amount matches
|
|
108
|
+
# marketing scale-talk ("$100M SaaS") and job listings ("$30-40/hr"), so a
|
|
109
|
+
# money mention counts only NEAR a payment word, and explicit first-person
|
|
110
|
+
# pay phrases count on their own (validated on a live HN pull).
|
|
111
|
+
MARKER_LIST_VERSION = 1
|
|
112
|
+
_PAY_PHRASE_RE = re.compile(
|
|
113
|
+
r"\b(?:would|i'?d|happy to|willing to|gladly|happily)\s+pay\b"
|
|
114
|
+
r"|\bpay(?:ing)?\s+for\b|\bworth paying\b"
|
|
115
|
+
r"|\bpaid\s+(?:plan|version|tier|tool|app)\b"
|
|
116
|
+
r"|\btake my money\b",
|
|
117
|
+
re.IGNORECASE,
|
|
118
|
+
)
|
|
119
|
+
# $100M/$2B are scale brags, not prices — the money marker rejects M/B/k
|
|
120
|
+
# magnitude suffixes.
|
|
121
|
+
_MONEY_RE = re.compile(
|
|
122
|
+
r"\$\s?\d[\d,]*(?:\.\d{1,2})?(?![\dkKmMbB])"
|
|
123
|
+
r"|\b\d+\s?(?:usd|dollars|bucks)\b",
|
|
124
|
+
re.IGNORECASE,
|
|
125
|
+
)
|
|
126
|
+
_PAY_CONTEXT_RE = re.compile(
|
|
127
|
+
r"\bpay(?:s|ing|ment)?\b|\bpaid\b|\bcosts?\b|\bprices?\b|\bpriced\b"
|
|
128
|
+
r"|\bpricing\b|\bcharge[sd]?\b|\bsubscription\b|\bsubscribe[d]?\b"
|
|
129
|
+
r"|\bbill(?:ed|ing)?\b|/mo\b|\bper\s+(?:month|year|seat|user)\b"
|
|
130
|
+
r"|\ba\s+month\b|\ba\s+year\b",
|
|
131
|
+
re.IGNORECASE,
|
|
132
|
+
)
|
|
133
|
+
_MONEY_CONTEXT_WINDOW = 60
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _wtp_marker_positions(text: str) -> list[int]:
|
|
137
|
+
positions = [match.start() for match in _PAY_PHRASE_RE.finditer(text)]
|
|
138
|
+
for money in _MONEY_RE.finditer(text):
|
|
139
|
+
start = max(0, money.start() - _MONEY_CONTEXT_WINDOW)
|
|
140
|
+
end = min(len(text), money.end() + _MONEY_CONTEXT_WINDOW)
|
|
141
|
+
if _PAY_CONTEXT_RE.search(text[start:end]):
|
|
142
|
+
positions.append(money.start())
|
|
143
|
+
return positions
|
|
144
|
+
_PAIN_RE = re.compile(
|
|
145
|
+
r"\bfrustrat\w*|\bannoy\w*|\bhate[sd]?\b|\bpainful\b|\btedious\b"
|
|
146
|
+
r"|\bclunky\b|\bnightmare\b|\bdrives me (?:crazy|nuts|insane)\b"
|
|
147
|
+
r"|\bwaste[sd]? (?:of )?(?:so much )?time\b|\bhours every (?:day|week)\b"
|
|
148
|
+
r"|\bmanually\b|\bstruggl\w*|\bcan'?t stand\b|\btoo (?:slow|complicated)\b"
|
|
149
|
+
r"|\bbuggy\b|\bunusable\b|\bgave up\b",
|
|
150
|
+
re.IGNORECASE,
|
|
151
|
+
)
|
|
152
|
+
_SEEKING_RE = re.compile(
|
|
153
|
+
r"\blooking for (?:a|an|some|any)\b|\bis there (?:a|an|any)\b"
|
|
154
|
+
r"|\bany (?:good )?(?:recommendations?|alternatives?|tools?)\b"
|
|
155
|
+
r"|\brecommend (?:a|an|any|me)\b|\balternative to\b"
|
|
156
|
+
r"|\bwish there (?:was|were)\b|\bi wish (?:it|there|someone)\b"
|
|
157
|
+
r"|\bdoes anyone (?:know|use)\b|\bwhat (?:do you|does everyone) use\b"
|
|
158
|
+
r"|\bhow do you (?:all )?(?:manage|track|handle)\b",
|
|
159
|
+
re.IGNORECASE,
|
|
160
|
+
)
|
|
161
|
+
# Price-sensitivity is its own diagnostic category, not a WTP signal: a
|
|
162
|
+
# segment repeatedly asking for "free/cheap" tools is a monetization
|
|
163
|
+
# warning that must be visible in the tables, not buried inside a seeking
|
|
164
|
+
# quote (the tutors' "ideally free" lesson from the first live run).
|
|
165
|
+
_PRICE_SENS_RE = re.compile(
|
|
166
|
+
r"\bfree\b|\bfreemium\b|\bfree tier\b|\bcheap(?:er|est)?\b"
|
|
167
|
+
r"|\baffordable\b|\bbudget\b|\blow[- ]cost\b|\binexpensive\b"
|
|
168
|
+
r"|\btoo expensive\b|\boverpriced\b|\bpricey\b|\bcan(?:'|no)t afford\b",
|
|
169
|
+
re.IGNORECASE,
|
|
170
|
+
)
|
|
171
|
+
_BOT_USERNAMES = {"automoderator", "sneakpeekbot", "remindmebot", "savevideo"}
|
|
172
|
+
_TAG_RE = re.compile(r"<[^>]+>")
|
|
173
|
+
# Sellers pitching their own product are not demand; separator-formatted
|
|
174
|
+
# classifieds (HN "Who is hiring"/"Who wants to be hired" comments, job
|
|
175
|
+
# posts) are listings, not demand language. All observed dominating a live
|
|
176
|
+
# relevance-sorted pull.
|
|
177
|
+
_PROMO_TITLE_RE = re.compile(r"^\s*(?:show|launch)\s+hn\b", re.IGNORECASE)
|
|
178
|
+
_JOBS_THREAD_RE = re.compile(
|
|
179
|
+
r"who is hiring|who wants to be hired|seeking freelancer", re.IGNORECASE
|
|
180
|
+
)
|
|
181
|
+
# Reddit self-promo posts are sellers, not askers ("wanted to share a
|
|
182
|
+
# little tool me and my SO have been building" — slipped through the first
|
|
183
|
+
# live run as a seeking quote).
|
|
184
|
+
_SELF_PROMO_RE = re.compile(
|
|
185
|
+
r"\b(?:i|we)(?:'ve| have)? (?:built|made|created|launched)\b"
|
|
186
|
+
r"|\bwanted to share\b|\bsharing (?:my|our)\b|\bcheck out (?:my|our)\b"
|
|
187
|
+
r"|\bmy (?:side |latest )?project\b",
|
|
188
|
+
re.IGNORECASE,
|
|
189
|
+
)
|
|
190
|
+
_LISTING_MIN_SEPARATORS = 3
|
|
191
|
+
_LISTING_LABELS = (
|
|
192
|
+
"location:",
|
|
193
|
+
"remote:",
|
|
194
|
+
"willing to relocate:",
|
|
195
|
+
"technologies:",
|
|
196
|
+
"compensation:",
|
|
197
|
+
"salary:",
|
|
198
|
+
"resume:",
|
|
199
|
+
"résumé:",
|
|
200
|
+
)
|
|
201
|
+
# Query tokens too generic to anchor topical relevance on their own.
|
|
202
|
+
_GENERIC_QUERY_TOKENS = {
|
|
203
|
+
"ai",
|
|
204
|
+
"app",
|
|
205
|
+
"apps",
|
|
206
|
+
"best",
|
|
207
|
+
"for",
|
|
208
|
+
"free",
|
|
209
|
+
"online",
|
|
210
|
+
"platform",
|
|
211
|
+
"service",
|
|
212
|
+
"software",
|
|
213
|
+
"the",
|
|
214
|
+
"tool",
|
|
215
|
+
"tools",
|
|
216
|
+
"top",
|
|
217
|
+
"web",
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
HN_COMMUNITY = "news.ycombinator.com"
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class CommunityDemandError(RuntimeError):
|
|
224
|
+
"""Terminal runner failure; runtime state is preserved for resume."""
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _load_script(filename: str, module_name: str) -> ModuleType:
|
|
228
|
+
path = Path(__file__).resolve().parent / filename
|
|
229
|
+
spec = importlib.util.spec_from_file_location(module_name, path)
|
|
230
|
+
if spec is None or spec.loader is None:
|
|
231
|
+
raise CommunityDemandError(f"cannot load bundled script {filename}")
|
|
232
|
+
module = importlib.util.module_from_spec(spec)
|
|
233
|
+
sys.modules[spec.name] = module
|
|
234
|
+
previous = sys.dont_write_bytecode
|
|
235
|
+
sys.dont_write_bytecode = True
|
|
236
|
+
try:
|
|
237
|
+
spec.loader.exec_module(module)
|
|
238
|
+
finally:
|
|
239
|
+
sys.dont_write_bytecode = previous
|
|
240
|
+
return module
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
# ── mixed client: paid OS actor runs + free keyless HN calls ───────────────
|
|
244
|
+
|
|
245
|
+
@dataclass
|
|
246
|
+
class OsClient:
|
|
247
|
+
os_url: str
|
|
248
|
+
api_key: str
|
|
249
|
+
cache_dir: Path
|
|
250
|
+
max_actor_runs: int = MAX_ACTOR_RUNS_PER_ATTEMPT
|
|
251
|
+
max_hn_calls: int = MAX_HN_CALLS_PER_ATTEMPT
|
|
252
|
+
max_yt_calls: int = MAX_YT_CALLS_PER_ATTEMPT
|
|
253
|
+
hn_url: str = HN_API_URL
|
|
254
|
+
actor_runs: int = 0
|
|
255
|
+
hn_calls: int = 0
|
|
256
|
+
yt_calls: int = 0
|
|
257
|
+
cache_hits: int = 0
|
|
258
|
+
touched: set[str] = None # type: ignore[assignment] # set in __post_init__
|
|
259
|
+
request: Callable[..., dict[str, Any]] | None = None
|
|
260
|
+
hn_request: Callable[[str], dict[str, Any]] | None = None
|
|
261
|
+
sleep: Callable[[float], None] | None = None
|
|
262
|
+
|
|
263
|
+
def __post_init__(self) -> None:
|
|
264
|
+
if self.touched is None:
|
|
265
|
+
self.touched = set()
|
|
266
|
+
|
|
267
|
+
def _http(self, path: str, payload: dict[str, Any], timeout: float, *, idempotency_key: str | None = None) -> dict[str, Any]:
|
|
268
|
+
url = f"{self.os_url.rstrip('/')}{path}"
|
|
269
|
+
headers = {
|
|
270
|
+
"Authorization": f"Bearer {self.api_key}",
|
|
271
|
+
"Content-Type": "application/json",
|
|
272
|
+
"Accept": "application/json",
|
|
273
|
+
}
|
|
274
|
+
if idempotency_key:
|
|
275
|
+
headers["Idempotency-Key"] = idempotency_key
|
|
276
|
+
req = urllib.request.Request(
|
|
277
|
+
url,
|
|
278
|
+
data=json.dumps(payload).encode("utf-8"),
|
|
279
|
+
headers=headers,
|
|
280
|
+
method="POST",
|
|
281
|
+
)
|
|
282
|
+
try:
|
|
283
|
+
for _name, _value in _connection_headers(req.full_url).items():
|
|
284
|
+
req.add_unredirected_header(_name, _value)
|
|
285
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
286
|
+
envelope = json.loads(resp.read().decode("utf-8"))
|
|
287
|
+
except urllib.error.HTTPError as exc:
|
|
288
|
+
detail = exc.read().decode("utf-8", errors="replace")[:500]
|
|
289
|
+
raise CommunityDemandError(f"OS {path} failed: HTTP {exc.code} {detail}") from exc
|
|
290
|
+
except urllib.error.URLError as exc:
|
|
291
|
+
raise CommunityDemandError(f"OS {path} unreachable: {exc.reason}") from exc
|
|
292
|
+
data = envelope.get("data")
|
|
293
|
+
if envelope.get("code") not in (None, 200) or not isinstance(data, dict):
|
|
294
|
+
raise CommunityDemandError(f"OS {path} returned no data: {str(envelope)[:300]}")
|
|
295
|
+
return data
|
|
296
|
+
|
|
297
|
+
def _hn_http(self, url: str) -> dict[str, Any]:
|
|
298
|
+
req = urllib.request.Request(url, headers={"Accept": "application/json"})
|
|
299
|
+
try:
|
|
300
|
+
for _name, _value in _connection_headers(req.full_url).items():
|
|
301
|
+
req.add_unredirected_header(_name, _value)
|
|
302
|
+
with urllib.request.urlopen(req, timeout=30.0) as resp:
|
|
303
|
+
value = json.loads(resp.read().decode("utf-8"))
|
|
304
|
+
except urllib.error.HTTPError as exc:
|
|
305
|
+
raise CommunityDemandError(f"HN search failed: HTTP {exc.code}") from exc
|
|
306
|
+
except urllib.error.URLError as exc:
|
|
307
|
+
raise CommunityDemandError(f"HN search unreachable: {exc.reason}") from exc
|
|
308
|
+
if not isinstance(value, dict):
|
|
309
|
+
raise CommunityDemandError("HN search returned a non-object response")
|
|
310
|
+
return value
|
|
311
|
+
|
|
312
|
+
def _cache_read(self, cache_file: Path) -> dict[str, Any] | None:
|
|
313
|
+
if not cache_file.is_file():
|
|
314
|
+
return None
|
|
315
|
+
try:
|
|
316
|
+
return json.loads(cache_file.read_text(encoding="utf-8"))
|
|
317
|
+
except (OSError, json.JSONDecodeError):
|
|
318
|
+
# A torn write must not poison replay forever: discard the
|
|
319
|
+
# corrupt entry and refetch.
|
|
320
|
+
cache_file.unlink(missing_ok=True)
|
|
321
|
+
return None
|
|
322
|
+
|
|
323
|
+
def _cache_write(self, cache_file: Path, entry: dict[str, Any]) -> None:
|
|
324
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
325
|
+
tmp_file = cache_file.with_suffix(".json.tmp")
|
|
326
|
+
tmp_file.write_text(json.dumps(entry), encoding="utf-8")
|
|
327
|
+
tmp_file.replace(cache_file)
|
|
328
|
+
|
|
329
|
+
def run_actor(self, actor: str, actor_input: dict[str, Any], limit: int) -> tuple[list[dict[str, Any]], str]:
|
|
330
|
+
payload = {"actor": actor, "input": actor_input, "sync": True, "limit": limit}
|
|
331
|
+
key_raw = json.dumps({"kind": "actor", "payload": payload}, sort_keys=True)
|
|
332
|
+
digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
|
|
333
|
+
cache_file = self.cache_dir / f"{digest}.json"
|
|
334
|
+
self.touched.add(digest)
|
|
335
|
+
entry = self._cache_read(cache_file)
|
|
336
|
+
if entry is not None:
|
|
337
|
+
self.cache_hits += 1
|
|
338
|
+
rows = entry["response"].get("rows")
|
|
339
|
+
return (
|
|
340
|
+
rows if isinstance(rows, list) else []
|
|
341
|
+
), entry.get("retrieved_on") or date.today().isoformat()
|
|
342
|
+
|
|
343
|
+
transport = self.request or self._http
|
|
344
|
+
attempts = 0
|
|
345
|
+
# Salted ONCE per call, before the retry loop: a within-call
|
|
346
|
+
# transient-5xx retry must reuse the SAME wire key (so the gateway
|
|
347
|
+
# can dedupe an ambiguous sync failure into one upstream run), but a
|
|
348
|
+
# fresh call — a prior attempt having already failed and journaled a
|
|
349
|
+
# start for this digest — must get a NEW key. The cache key (digest)
|
|
350
|
+
# stays the pure payload hash so replay lookups are unaffected.
|
|
351
|
+
wire_key = f"{digest}-{self._prior_start_count(digest)}"
|
|
352
|
+
while True:
|
|
353
|
+
# Paid starts count BEFORE the transport call (fail closed) and
|
|
354
|
+
# are journaled durably: a failed start may still have billed.
|
|
355
|
+
if self.actor_runs >= self.max_actor_runs:
|
|
356
|
+
raise CommunityDemandError(
|
|
357
|
+
f"actor-run budget exhausted ({self.max_actor_runs} per "
|
|
358
|
+
"attempt); narrow the query list or re-run to continue "
|
|
359
|
+
"from cache"
|
|
360
|
+
)
|
|
361
|
+
self.actor_runs += 1
|
|
362
|
+
self._journal_start(digest)
|
|
363
|
+
try:
|
|
364
|
+
response = transport(
|
|
365
|
+
"/scrape/apify/run", payload, 330.0, idempotency_key=wire_key
|
|
366
|
+
)
|
|
367
|
+
break
|
|
368
|
+
except CommunityDemandError as exc:
|
|
369
|
+
if attempts >= 1 or not re.search(r"HTTP 5\d\d", str(exc)):
|
|
370
|
+
raise
|
|
371
|
+
attempts += 1
|
|
372
|
+
(self.sleep or time.sleep)(ACTOR_RETRY_DELAY_SECONDS)
|
|
373
|
+
retrieved_on = date.today().isoformat()
|
|
374
|
+
self._cache_write(
|
|
375
|
+
cache_file,
|
|
376
|
+
{
|
|
377
|
+
"kind": "actor",
|
|
378
|
+
"payload": payload,
|
|
379
|
+
"response": response,
|
|
380
|
+
"retrieved_on": retrieved_on,
|
|
381
|
+
},
|
|
382
|
+
)
|
|
383
|
+
rows = response.get("rows")
|
|
384
|
+
return (rows if isinstance(rows, list) else []), retrieved_on
|
|
385
|
+
|
|
386
|
+
def hn_search(
|
|
387
|
+
self, query: str, tags: str, hits_per_page: int, created_after_epoch: int
|
|
388
|
+
) -> tuple[list[dict[str, Any]], str]:
|
|
389
|
+
# The recency window lives in the QUERY (and therefore the cache
|
|
390
|
+
# key): an all-time relevance page can fill up with old hits and
|
|
391
|
+
# silently hide newer demand.
|
|
392
|
+
params = urllib.parse.urlencode(
|
|
393
|
+
{
|
|
394
|
+
"query": query,
|
|
395
|
+
"tags": tags,
|
|
396
|
+
"hitsPerPage": hits_per_page,
|
|
397
|
+
"numericFilters": f"created_at_i>{created_after_epoch}",
|
|
398
|
+
}
|
|
399
|
+
)
|
|
400
|
+
url = f"{self.hn_url.rstrip('/')}/search?{params}"
|
|
401
|
+
key_raw = json.dumps({"kind": "hn", "url": url}, sort_keys=True)
|
|
402
|
+
digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
|
|
403
|
+
cache_file = self.cache_dir / f"{digest}.json"
|
|
404
|
+
self.touched.add(digest)
|
|
405
|
+
entry = self._cache_read(cache_file)
|
|
406
|
+
if entry is not None:
|
|
407
|
+
self.cache_hits += 1
|
|
408
|
+
hits = entry["response"].get("hits")
|
|
409
|
+
return (
|
|
410
|
+
hits if isinstance(hits, list) else []
|
|
411
|
+
), entry.get("retrieved_on") or date.today().isoformat()
|
|
412
|
+
|
|
413
|
+
transport = self.hn_request or self._hn_http
|
|
414
|
+
attempts = 0
|
|
415
|
+
while True:
|
|
416
|
+
# Free calls are still bounded: a broken loop must not hammer a
|
|
417
|
+
# keyless public API.
|
|
418
|
+
if self.hn_calls >= self.max_hn_calls:
|
|
419
|
+
raise CommunityDemandError(
|
|
420
|
+
f"HN call budget exhausted ({self.max_hn_calls} per attempt)"
|
|
421
|
+
)
|
|
422
|
+
self.hn_calls += 1
|
|
423
|
+
try:
|
|
424
|
+
response = transport(url)
|
|
425
|
+
break
|
|
426
|
+
except CommunityDemandError as exc:
|
|
427
|
+
if attempts >= 1 or not re.search(r"HTTP 5\d\d", str(exc)):
|
|
428
|
+
raise
|
|
429
|
+
attempts += 1
|
|
430
|
+
(self.sleep or time.sleep)(HN_RETRY_DELAY_SECONDS)
|
|
431
|
+
retrieved_on = date.today().isoformat()
|
|
432
|
+
self._cache_write(
|
|
433
|
+
cache_file,
|
|
434
|
+
{
|
|
435
|
+
"kind": "hn",
|
|
436
|
+
"url": url,
|
|
437
|
+
"response": response,
|
|
438
|
+
"retrieved_on": retrieved_on,
|
|
439
|
+
},
|
|
440
|
+
)
|
|
441
|
+
hits = response.get("hits")
|
|
442
|
+
return (hits if isinstance(hits, list) else []), retrieved_on
|
|
443
|
+
|
|
444
|
+
def yt_call(self, operation: str, payload: dict[str, Any]) -> tuple[dict[str, Any], str]:
|
|
445
|
+
"""One OS `/tools/youtube/<operation>` call. Metered per claw but
|
|
446
|
+
free in money terms; bounded so a broken loop cannot drain the
|
|
447
|
+
fleet-shared Google quota."""
|
|
448
|
+
key_raw = json.dumps(
|
|
449
|
+
{"kind": "yt", "operation": operation, "payload": payload}, sort_keys=True
|
|
450
|
+
)
|
|
451
|
+
digest = hashlib.sha256(key_raw.encode("utf-8")).hexdigest()
|
|
452
|
+
cache_file = self.cache_dir / f"{digest}.json"
|
|
453
|
+
self.touched.add(digest)
|
|
454
|
+
entry = self._cache_read(cache_file)
|
|
455
|
+
if entry is not None:
|
|
456
|
+
self.cache_hits += 1
|
|
457
|
+
response = entry["response"]
|
|
458
|
+
return (
|
|
459
|
+
response if isinstance(response, dict) else {}
|
|
460
|
+
), entry.get("retrieved_on") or date.today().isoformat()
|
|
461
|
+
|
|
462
|
+
transport = self.request or self._http
|
|
463
|
+
attempts = 0
|
|
464
|
+
while True:
|
|
465
|
+
if self.yt_calls >= self.max_yt_calls:
|
|
466
|
+
raise CommunityDemandError(
|
|
467
|
+
f"YouTube call budget exhausted ({self.max_yt_calls} per attempt)"
|
|
468
|
+
)
|
|
469
|
+
self.yt_calls += 1
|
|
470
|
+
try:
|
|
471
|
+
response = transport(f"/tools/youtube/{operation}", payload, 60.0)
|
|
472
|
+
break
|
|
473
|
+
except CommunityDemandError as exc:
|
|
474
|
+
if attempts >= 1 or not re.search(r"HTTP 5\d\d", str(exc)):
|
|
475
|
+
raise
|
|
476
|
+
attempts += 1
|
|
477
|
+
(self.sleep or time.sleep)(HN_RETRY_DELAY_SECONDS)
|
|
478
|
+
retrieved_on = date.today().isoformat()
|
|
479
|
+
self._cache_write(
|
|
480
|
+
cache_file,
|
|
481
|
+
{
|
|
482
|
+
"kind": "yt",
|
|
483
|
+
"operation": operation,
|
|
484
|
+
"payload": payload,
|
|
485
|
+
"response": response,
|
|
486
|
+
"retrieved_on": retrieved_on,
|
|
487
|
+
},
|
|
488
|
+
)
|
|
489
|
+
return (response if isinstance(response, dict) else {}), retrieved_on
|
|
490
|
+
|
|
491
|
+
def _journal_start(self, digest: str) -> None:
|
|
492
|
+
"""Append-only durable record of every PAID start, successful or not
|
|
493
|
+
— the cache holds only successes, but failed sync starts may still
|
|
494
|
+
have billed upstream."""
|
|
495
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
496
|
+
with (self.cache_dir / "starts.log").open("a", encoding="utf-8") as handle:
|
|
497
|
+
handle.write(f"{date.today().isoformat()} {digest}\n")
|
|
498
|
+
|
|
499
|
+
def _prior_start_count(self, digest: str) -> int:
|
|
500
|
+
"""Paid starts already journaled for this digest by EARLIER
|
|
501
|
+
invocations — the salt for the wire Idempotency-Key. A failed
|
|
502
|
+
attempt's key must never be replayed verbatim: on the gateway (see
|
|
503
|
+
#3715), a client key can poison a wallet hold, and a byte-identical
|
|
504
|
+
retry of a poisoned key fails identically forever."""
|
|
505
|
+
starts_log = self.cache_dir / "starts.log"
|
|
506
|
+
try:
|
|
507
|
+
lines = starts_log.read_text(encoding="utf-8").splitlines()
|
|
508
|
+
except OSError:
|
|
509
|
+
return 0
|
|
510
|
+
return sum(1 for line in lines if line.endswith(f" {digest}"))
|
|
511
|
+
|
|
512
|
+
def ledger_totals(self) -> dict[str, Any]:
|
|
513
|
+
"""Durable provenance over entries THIS execution touched."""
|
|
514
|
+
actor_entries = 0
|
|
515
|
+
hn_entries = 0
|
|
516
|
+
yt_entries = 0
|
|
517
|
+
dates: list[str] = []
|
|
518
|
+
if self.cache_dir.is_dir():
|
|
519
|
+
for digest in sorted(self.touched):
|
|
520
|
+
entry_path = self.cache_dir / f"{digest}.json"
|
|
521
|
+
try:
|
|
522
|
+
entry = json.loads(entry_path.read_text(encoding="utf-8"))
|
|
523
|
+
except (OSError, json.JSONDecodeError):
|
|
524
|
+
continue
|
|
525
|
+
kind = entry.get("kind")
|
|
526
|
+
if kind == "hn":
|
|
527
|
+
hn_entries += 1
|
|
528
|
+
elif kind == "yt":
|
|
529
|
+
yt_entries += 1
|
|
530
|
+
else:
|
|
531
|
+
actor_entries += 1
|
|
532
|
+
if isinstance(entry.get("retrieved_on"), str):
|
|
533
|
+
dates.append(entry["retrieved_on"])
|
|
534
|
+
today = date.today().isoformat()
|
|
535
|
+
starts_log = self.cache_dir / "starts.log"
|
|
536
|
+
try:
|
|
537
|
+
starts_recorded = sum(
|
|
538
|
+
1 for line in starts_log.read_text(encoding="utf-8").splitlines() if line
|
|
539
|
+
)
|
|
540
|
+
except OSError:
|
|
541
|
+
starts_recorded = 0
|
|
542
|
+
return {
|
|
543
|
+
"actor_runs": actor_entries,
|
|
544
|
+
"provider_calls": actor_entries,
|
|
545
|
+
"hn_calls": hn_entries,
|
|
546
|
+
"youtube_calls": yt_entries,
|
|
547
|
+
"actor_starts_recorded": starts_recorded,
|
|
548
|
+
"first_retrieved_on": min(dates) if dates else today,
|
|
549
|
+
"last_retrieved_on": max(dates) if dates else today,
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
# ── deterministic helpers ──────────────────────────────────────────────────
|
|
554
|
+
|
|
555
|
+
def _clean(value: Any) -> Any:
|
|
556
|
+
"""Apify actors serialize empty fields as the literal string 'None'."""
|
|
557
|
+
if value is None:
|
|
558
|
+
return None
|
|
559
|
+
if isinstance(value, str) and value.strip() in ("", "None", "null"):
|
|
560
|
+
return None
|
|
561
|
+
return value
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def _to_int(value: Any) -> int | None:
|
|
565
|
+
value = _clean(value)
|
|
566
|
+
if value is None:
|
|
567
|
+
return None
|
|
568
|
+
try:
|
|
569
|
+
return int(float(value))
|
|
570
|
+
except (TypeError, ValueError):
|
|
571
|
+
return None
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
def _plain_text(value: Any) -> str:
|
|
575
|
+
"""HTML-decode, strip tags (HN comment_text carries markup), collapse
|
|
576
|
+
whitespace."""
|
|
577
|
+
text = html.unescape(str(_clean(value) or ""))
|
|
578
|
+
text = _TAG_RE.sub(" ", text)
|
|
579
|
+
return " ".join(text.split())
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def _truncate(text: str, cap: int) -> str:
|
|
583
|
+
if len(text) <= cap:
|
|
584
|
+
return text
|
|
585
|
+
cut = text[:cap].rsplit(" ", 1)[0].rstrip()
|
|
586
|
+
return (cut or text[:cap].rstrip()) + "…"
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
# A long comment can mention the category once and rant about something
|
|
590
|
+
# else: a marker is topical only NEAR a query anchor token (validated on a
|
|
591
|
+
# live pull where AI-industry rants outnumbered category demand).
|
|
592
|
+
_ANCHOR_PROXIMITY_WINDOW = 120
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def _anchor_positions(text: str, query: str) -> list[int] | None:
|
|
596
|
+
"""None means the query is fully generic and imposes no anchor."""
|
|
597
|
+
anchors = query_anchor_tokens(query)
|
|
598
|
+
if not anchors:
|
|
599
|
+
return None
|
|
600
|
+
lowered = text.lower()
|
|
601
|
+
return [
|
|
602
|
+
match.start()
|
|
603
|
+
for anchor in anchors
|
|
604
|
+
for match in re.finditer(rf"\b{re.escape(anchor)}", lowered)
|
|
605
|
+
]
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def _near_any(positions: list[int] | None, marker_positions: list[int]) -> bool:
|
|
609
|
+
if not marker_positions:
|
|
610
|
+
return False
|
|
611
|
+
if positions is None:
|
|
612
|
+
return True
|
|
613
|
+
return any(
|
|
614
|
+
abs(marker - anchor) <= _ANCHOR_PROXIMITY_WINDOW
|
|
615
|
+
for marker in marker_positions
|
|
616
|
+
for anchor in positions
|
|
617
|
+
)
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def matched_categories(text: str, query: str = "") -> list[str]:
|
|
621
|
+
anchor_positions = _anchor_positions(text, query) if query else None
|
|
622
|
+
if anchor_positions is not None and not anchor_positions:
|
|
623
|
+
return [] # anchored query, text never mentions the category
|
|
624
|
+
categories: list[str] = []
|
|
625
|
+
if _near_any(anchor_positions, _wtp_marker_positions(text)):
|
|
626
|
+
categories.append("wtp")
|
|
627
|
+
if _near_any(
|
|
628
|
+
anchor_positions, [m.start() for m in _SEEKING_RE.finditer(text)]
|
|
629
|
+
):
|
|
630
|
+
categories.append("seeking")
|
|
631
|
+
if _near_any(
|
|
632
|
+
anchor_positions, [m.start() for m in _PAIN_RE.finditer(text)]
|
|
633
|
+
):
|
|
634
|
+
categories.append("pain")
|
|
635
|
+
if _near_any(
|
|
636
|
+
anchor_positions, [m.start() for m in _PRICE_SENS_RE.finditer(text)]
|
|
637
|
+
):
|
|
638
|
+
categories.append("price_sensitivity")
|
|
639
|
+
return categories
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
def parse_context_tokens(raw: str) -> list[str]:
|
|
643
|
+
tokens: list[str] = []
|
|
644
|
+
for part in re.split(r"[,;\n]", raw or ""):
|
|
645
|
+
token = re.sub(r"[^a-z0-9]", "", part.strip().lower())
|
|
646
|
+
if len(token) >= 2 and token not in tokens:
|
|
647
|
+
tokens.append(token)
|
|
648
|
+
return tokens[:8]
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def mentions_product_context(text: str, context_tokens: list[str]) -> bool:
|
|
652
|
+
"""Does the text speak to THIS product's mechanism (e.g. 'ai', 'agent')
|
|
653
|
+
rather than the bare category? Short tokens match exact words ('ai'
|
|
654
|
+
must not match 'aim'); longer ones are prefix-tolerant."""
|
|
655
|
+
lowered = text.lower()
|
|
656
|
+
for token in context_tokens:
|
|
657
|
+
pattern = (
|
|
658
|
+
rf"\b{re.escape(token)}\b" if len(token) <= 3 else rf"\b{re.escape(token)}"
|
|
659
|
+
)
|
|
660
|
+
if re.search(pattern, lowered):
|
|
661
|
+
return True
|
|
662
|
+
return False
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
def is_listing(text: str) -> bool:
|
|
666
|
+
"""Separator-formatted classifieds (job posts, 'Who is hiring' entries)
|
|
667
|
+
and labeled hire-me templates."""
|
|
668
|
+
if text.count(" | ") + text.count(" / ") >= _LISTING_MIN_SEPARATORS:
|
|
669
|
+
return True
|
|
670
|
+
lowered = text.lower()
|
|
671
|
+
return sum(1 for label in _LISTING_LABELS if label in lowered) >= 2
|
|
672
|
+
|
|
673
|
+
|
|
674
|
+
def query_anchor_tokens(query: str) -> list[str]:
|
|
675
|
+
return [
|
|
676
|
+
token
|
|
677
|
+
for token in re.findall(r"[a-z0-9]+", query.lower())
|
|
678
|
+
if len(token) >= 3 and token not in _GENERIC_QUERY_TOKENS
|
|
679
|
+
]
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
def parse_terms(raw: str, cap: int) -> list[str]:
|
|
685
|
+
seen: dict[str, None] = {}
|
|
686
|
+
for part in re.split(r"[,;\n]", raw or ""):
|
|
687
|
+
term = " ".join(part.strip().lower().split())
|
|
688
|
+
if term and term not in seen:
|
|
689
|
+
seen[term] = None
|
|
690
|
+
return list(seen)[:cap]
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
def select_queries(
|
|
695
|
+
explicit: list[str], discovered: list[str], cap: int
|
|
696
|
+
) -> tuple[list[str], dict[str, str]]:
|
|
697
|
+
"""Merge explicit seeds ahead of KWE-discovered keywords, dropping
|
|
698
|
+
token-set duplicates ('online whiteboard online' is the same query as
|
|
699
|
+
'online whiteboard' — a KWE keyword-variant artifact)."""
|
|
700
|
+
queries: list[str] = []
|
|
701
|
+
query_sources: dict[str, str] = {}
|
|
702
|
+
seen_token_sets: set[frozenset[str]] = set()
|
|
703
|
+
for query, source in [(q, "explicit") for q in explicit] + [
|
|
704
|
+
(q, "keyword_economics") for q in discovered
|
|
705
|
+
]:
|
|
706
|
+
if len(queries) >= cap:
|
|
707
|
+
break
|
|
708
|
+
token_set = frozenset(re.findall(r"[a-z0-9]+", query))
|
|
709
|
+
if not token_set or token_set in seen_token_sets:
|
|
710
|
+
continue
|
|
711
|
+
seen_token_sets.add(token_set)
|
|
712
|
+
queries.append(query)
|
|
713
|
+
query_sources[query] = source
|
|
714
|
+
return queries, query_sources
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
def queries_from_keyword_evidence(workspace: Path, cap: int) -> list[str]:
|
|
718
|
+
evidence = _load_script("research-evidence.py", "pmr_evidence")
|
|
719
|
+
return evidence.keyword_inputs(workspace, consumer=SECTION_ID, row_type="keyword", field="keyword", cap=cap)
|
|
720
|
+
|
|
721
|
+
|
|
722
|
+
# ── normalization: one raw item -> one candidate quote ─────────────────────
|
|
723
|
+
|
|
724
|
+
def normalize_hn_story(hit: dict[str, Any], query: str) -> dict[str, Any] | None:
|
|
725
|
+
object_id = str(_clean(hit.get("objectID")) or "")
|
|
726
|
+
title = _plain_text(hit.get("title"))
|
|
727
|
+
body = _plain_text(hit.get("story_text"))
|
|
728
|
+
text = " ".join(part for part in (title, body) if part)
|
|
729
|
+
if not object_id or not text:
|
|
730
|
+
return None
|
|
731
|
+
if _PROMO_TITLE_RE.search(title):
|
|
732
|
+
return None # the author is selling, not asking
|
|
733
|
+
return {
|
|
734
|
+
"id": f"hn_{object_id}",
|
|
735
|
+
"source": "hackernews",
|
|
736
|
+
"community": HN_COMMUNITY,
|
|
737
|
+
"kind": "story",
|
|
738
|
+
"query": query,
|
|
739
|
+
"text": text,
|
|
740
|
+
"url": f"https://news.ycombinator.com/item?id={object_id}",
|
|
741
|
+
"votes": _to_int(hit.get("points")) or 0,
|
|
742
|
+
"replies": _to_int(hit.get("num_comments")),
|
|
743
|
+
"created": str(_clean(hit.get("created_at")) or "")[:10] or None,
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
|
|
747
|
+
def normalize_hn_comment(hit: dict[str, Any], query: str) -> dict[str, Any] | None:
|
|
748
|
+
object_id = str(_clean(hit.get("objectID")) or "")
|
|
749
|
+
text = _plain_text(hit.get("comment_text"))
|
|
750
|
+
if not object_id or not text:
|
|
751
|
+
return None
|
|
752
|
+
if _JOBS_THREAD_RE.search(str(_clean(hit.get("story_title")) or "")):
|
|
753
|
+
return None # monthly jobs threads are classifieds, not demand
|
|
754
|
+
return {
|
|
755
|
+
"id": f"hn_{object_id}",
|
|
756
|
+
"source": "hackernews",
|
|
757
|
+
"community": HN_COMMUNITY,
|
|
758
|
+
"kind": "comment",
|
|
759
|
+
"query": query,
|
|
760
|
+
"text": text,
|
|
761
|
+
"url": f"https://news.ycombinator.com/item?id={object_id}",
|
|
762
|
+
"votes": _to_int(hit.get("points")) or 0,
|
|
763
|
+
"replies": None,
|
|
764
|
+
"created": str(_clean(hit.get("created_at")) or "")[:10] or None,
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
|
|
768
|
+
def normalize_reddit_row(row: dict[str, Any], query: str) -> dict[str, Any] | None:
|
|
769
|
+
data_type = str(_clean(row.get("dataType")) or "")
|
|
770
|
+
if data_type not in ("post", "comment"):
|
|
771
|
+
return None
|
|
772
|
+
row_id = str(_clean(row.get("id")) or "")
|
|
773
|
+
community = str(
|
|
774
|
+
_clean(row.get("parsedCommunityName"))
|
|
775
|
+
or str(_clean(row.get("communityName")) or "").removeprefix("r/")
|
|
776
|
+
or ""
|
|
777
|
+
).strip()
|
|
778
|
+
username = str(_clean(row.get("username")) or "").strip().lower()
|
|
779
|
+
title = _plain_text(row.get("title")) if data_type == "post" else ""
|
|
780
|
+
body = _plain_text(row.get("body"))
|
|
781
|
+
text = " ".join(part for part in (title, body) if part)
|
|
782
|
+
if not row_id or not community or not text:
|
|
783
|
+
return None
|
|
784
|
+
if username in _BOT_USERNAMES or "i am a bot" in text.lower():
|
|
785
|
+
return None
|
|
786
|
+
if data_type == "post" and _SELF_PROMO_RE.search(text):
|
|
787
|
+
return None # the author is selling, not asking
|
|
788
|
+
url = str(_clean(row.get("url")) or _clean(row.get("link")) or "")
|
|
789
|
+
if not url:
|
|
790
|
+
permalink = str(_clean(row.get("permalink")) or "")
|
|
791
|
+
url = f"https://www.reddit.com{permalink}" if permalink.startswith("/") else ""
|
|
792
|
+
return {
|
|
793
|
+
"id": f"reddit_{row_id}",
|
|
794
|
+
"source": "reddit",
|
|
795
|
+
"community": f"r/{community}",
|
|
796
|
+
"kind": data_type,
|
|
797
|
+
"query": query,
|
|
798
|
+
"text": text,
|
|
799
|
+
"url": url or None,
|
|
800
|
+
"votes": _to_int(row.get("upVotes")) or 0,
|
|
801
|
+
"replies": _to_int(row.get("numberOfComments") if data_type == "post" else row.get("numberOfReplies")),
|
|
802
|
+
"created": str(_clean(row.get("createdAt")) or "")[:10] or None,
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def normalize_youtube_comment(
|
|
807
|
+
row: dict[str, Any], query: str, video: dict[str, Any]
|
|
808
|
+
) -> dict[str, Any] | None:
|
|
809
|
+
comment_id = str(_clean(row.get("comment_id")) or "")
|
|
810
|
+
video_id = str(_clean(row.get("video_id")) or _clean(video.get("video_id")) or "")
|
|
811
|
+
text = _plain_text(row.get("text"))
|
|
812
|
+
if not comment_id or not video_id or not text:
|
|
813
|
+
return None
|
|
814
|
+
return {
|
|
815
|
+
"id": f"yt_{comment_id}",
|
|
816
|
+
"source": "youtube",
|
|
817
|
+
"community": "youtube.com",
|
|
818
|
+
"kind": "comment",
|
|
819
|
+
"query": query,
|
|
820
|
+
"text": text,
|
|
821
|
+
"url": f"https://www.youtube.com/watch?v={video_id}&lc={comment_id}",
|
|
822
|
+
"votes": _to_int(row.get("like_count")) or 0,
|
|
823
|
+
"replies": _to_int(row.get("reply_count")),
|
|
824
|
+
"created": str(_clean(row.get("published_at")) or "")[:10] or None,
|
|
825
|
+
"video_title": _truncate(_plain_text(video.get("title")), 120) or None,
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
def to_quote_row(
|
|
830
|
+
candidate: dict[str, Any], hn_cutoff: str, context_tokens: list[str]
|
|
831
|
+
) -> dict[str, Any] | None:
|
|
832
|
+
"""Marker-match one normalized candidate into an evidence quote row."""
|
|
833
|
+
text = candidate["text"]
|
|
834
|
+
if len(text) < MIN_QUOTE_CHARS:
|
|
835
|
+
return None
|
|
836
|
+
if is_listing(text):
|
|
837
|
+
return None
|
|
838
|
+
if candidate["source"] == "hackernews":
|
|
839
|
+
created = candidate.get("created")
|
|
840
|
+
if created is not None and created < hn_cutoff:
|
|
841
|
+
return None
|
|
842
|
+
categories = matched_categories(text, candidate["query"])
|
|
843
|
+
if not categories:
|
|
844
|
+
return None
|
|
845
|
+
quote = {
|
|
846
|
+
"type": "community_quote",
|
|
847
|
+
"id": candidate["id"],
|
|
848
|
+
"source": candidate["source"],
|
|
849
|
+
"community": candidate["community"],
|
|
850
|
+
"kind": candidate["kind"],
|
|
851
|
+
"query": candidate["query"],
|
|
852
|
+
"categories": categories,
|
|
853
|
+
"has_dollar_amount": bool(_MONEY_RE.search(text)),
|
|
854
|
+
# Category demand vs THIS product's mechanism: a quote about
|
|
855
|
+
# human-to-human realtime collaboration is not evidence for an
|
|
856
|
+
# AI-agent whiteboard even when both say "whiteboard".
|
|
857
|
+
"product_context": mentions_product_context(text, context_tokens)
|
|
858
|
+
if context_tokens
|
|
859
|
+
else None,
|
|
860
|
+
"text": _truncate(text, TEXT_TRUNCATE),
|
|
861
|
+
"url": candidate.get("url"),
|
|
862
|
+
"votes": candidate.get("votes") or 0,
|
|
863
|
+
"replies": candidate.get("replies"),
|
|
864
|
+
"created": candidate.get("created"),
|
|
865
|
+
"tier": "A",
|
|
866
|
+
}
|
|
867
|
+
if candidate.get("video_title"):
|
|
868
|
+
quote["video_title"] = candidate["video_title"]
|
|
869
|
+
return quote
|
|
870
|
+
|
|
871
|
+
|
|
872
|
+
def _created_recency_key(quote: dict[str, Any]) -> int:
|
|
873
|
+
"""Newest first, missing/unparsable dates last (0 sorts after every
|
|
874
|
+
negated real date)."""
|
|
875
|
+
digits = re.sub(r"[^0-9]", "", str(quote.get("created") or ""))[:8]
|
|
876
|
+
return -int(digits) if len(digits) == 8 else 0
|
|
877
|
+
|
|
878
|
+
|
|
879
|
+
def select_stored_quotes(quotes: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
880
|
+
"""Bounded observation sample: newest first, stable ID ties.
|
|
881
|
+
Per-community caps prevent one thread population consuming the sample;
|
|
882
|
+
payment markers and votes do not grant evidence priority."""
|
|
883
|
+
ranked = sorted(quotes, key=lambda q: (_created_recency_key(q), q["id"]))
|
|
884
|
+
stored: list[dict[str, Any]] = []
|
|
885
|
+
per_community: dict[str, int] = {}
|
|
886
|
+
for quote in ranked:
|
|
887
|
+
if len(stored) >= STORED_QUOTES_CAP:
|
|
888
|
+
break
|
|
889
|
+
count = per_community.get(quote["community"], 0)
|
|
890
|
+
if count >= STORED_PER_COMMUNITY_CAP:
|
|
891
|
+
continue
|
|
892
|
+
per_community[quote["community"]] = count + 1
|
|
893
|
+
stored.append(quote)
|
|
894
|
+
return stored
|
|
895
|
+
|
|
896
|
+
|
|
897
|
+
def summarize_communities(stored: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
898
|
+
by_community: dict[str, list[dict[str, Any]]] = {}
|
|
899
|
+
for quote in stored:
|
|
900
|
+
by_community.setdefault(quote["community"], []).append(quote)
|
|
901
|
+
rows: list[dict[str, Any]] = []
|
|
902
|
+
for community, quotes in by_community.items():
|
|
903
|
+
wtp = sum(1 for q in quotes if "wtp" in q["categories"])
|
|
904
|
+
seeking = sum(1 for q in quotes if "seeking" in q["categories"])
|
|
905
|
+
pain = sum(1 for q in quotes if "pain" in q["categories"])
|
|
906
|
+
price_sens = sum(
|
|
907
|
+
1 for q in quotes if "price_sensitivity" in q["categories"]
|
|
908
|
+
)
|
|
909
|
+
product_context = sum(1 for q in quotes if q.get("product_context"))
|
|
910
|
+
votes = sum(q["votes"] for q in quotes)
|
|
911
|
+
sample = min(quotes, key=lambda q: (_created_recency_key(q), q["id"]))
|
|
912
|
+
rows.append(
|
|
913
|
+
{
|
|
914
|
+
"type": "community",
|
|
915
|
+
"community": community,
|
|
916
|
+
"source": quotes[0]["source"],
|
|
917
|
+
"quotes": len(quotes),
|
|
918
|
+
"wtp_quotes": wtp,
|
|
919
|
+
"seeking_quotes": seeking,
|
|
920
|
+
"pain_quotes": pain,
|
|
921
|
+
"price_sensitivity_quotes": price_sens,
|
|
922
|
+
"product_context_quotes": product_context,
|
|
923
|
+
"votes_on_quotes": votes,
|
|
924
|
+
"latest_quote_date": max((q.get("created") or "" for q in quotes), default="") or None,
|
|
925
|
+
"sample_url": sample.get("url"),
|
|
926
|
+
"tier": "A",
|
|
927
|
+
}
|
|
928
|
+
)
|
|
929
|
+
rows.sort(key=lambda r: r["community"])
|
|
930
|
+
return rows
|
|
931
|
+
|
|
932
|
+
|
|
933
|
+
def excluded_source_rows(youtube_failure: str | None) -> list[dict[str, Any]]:
|
|
934
|
+
rows = [
|
|
935
|
+
{
|
|
936
|
+
"type": "excluded_source",
|
|
937
|
+
"source": "product_hunt",
|
|
938
|
+
"reason": "GraphQL API v2 cannot keyword-search launches (no "
|
|
939
|
+
"post search, coarse topics, url filter does not match "
|
|
940
|
+
"domains); excluded after live probes 2026-08-19",
|
|
941
|
+
"tier": "C",
|
|
942
|
+
},
|
|
943
|
+
{
|
|
944
|
+
"type": "excluded_source",
|
|
945
|
+
"source": "arctic_shift",
|
|
946
|
+
"reason": "search API cannot keyword-search across all of Reddit "
|
|
947
|
+
"(per-subreddit only); deferred to v2",
|
|
948
|
+
"tier": "C",
|
|
949
|
+
},
|
|
950
|
+
{
|
|
951
|
+
"type": "excluded_source",
|
|
952
|
+
"source": "stack_exchange",
|
|
953
|
+
"reason": "v2 scope",
|
|
954
|
+
"tier": "C",
|
|
955
|
+
},
|
|
956
|
+
]
|
|
957
|
+
if youtube_failure:
|
|
958
|
+
# YouTube is best-effort: the fleet-shared Google quota or an
|
|
959
|
+
# undeployed gateway must not fail the whole section, but the gap
|
|
960
|
+
# is recorded, never silent.
|
|
961
|
+
rows.append(
|
|
962
|
+
{
|
|
963
|
+
"type": "excluded_source",
|
|
964
|
+
"source": "youtube",
|
|
965
|
+
"reason": youtube_failure,
|
|
966
|
+
"tier": "C",
|
|
967
|
+
}
|
|
968
|
+
)
|
|
969
|
+
return rows
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
# ── rendering ──────────────────────────────────────────────────────────────
|
|
973
|
+
|
|
974
|
+
def _quote_cell(quote: dict[str, Any], cap: int = 200) -> str:
|
|
975
|
+
text = _truncate(quote["text"], cap).replace("|", "/")
|
|
976
|
+
if quote.get("url"):
|
|
977
|
+
return f"[{text}]({quote['url']})"
|
|
978
|
+
return text
|
|
979
|
+
|
|
980
|
+
|
|
981
|
+
def render_section(
|
|
982
|
+
*,
|
|
983
|
+
queries: list[str],
|
|
984
|
+
query_sources: dict[str, str],
|
|
985
|
+
product_context: list[str],
|
|
986
|
+
search_summaries: list[dict[str, Any]],
|
|
987
|
+
communities: list[dict[str, Any]],
|
|
988
|
+
stored_quotes: list[dict[str, Any]],
|
|
989
|
+
excluded_sources: list[dict[str, Any]],
|
|
990
|
+
totals: dict[str, Any],
|
|
991
|
+
retrieved_on: str,
|
|
992
|
+
) -> str:
|
|
993
|
+
lines: list[str] = []
|
|
994
|
+
add = lines.append
|
|
995
|
+
|
|
996
|
+
add(
|
|
997
|
+
f"Public community posts and comments retrieved {retrieved_on} from "
|
|
998
|
+
"Hacker News (keyless Algolia search; stories and comments, last "
|
|
999
|
+
f"{HN_RECENCY_DAYS // 365} years), Reddit (Apify actor search "
|
|
1000
|
+
"across all subreddits, posts and comments from the last 12 "
|
|
1001
|
+
"months), and YouTube (comments under the top videos for each "
|
|
1002
|
+
f"query, videos from the last {YT_RECENCY_DAYS // 365} years; "
|
|
1003
|
+
"best-effort). Quotes are selected by a fixed marker-regex list "
|
|
1004
|
+
f"(v{MARKER_LIST_VERSION}: payment language, tool-seeking language, "
|
|
1005
|
+
"pain language, price-sensitivity language) — deterministic keyword "
|
|
1006
|
+
"heuristics, not comprehension, so category counts measure marker "
|
|
1007
|
+
"density, not verified intent. All aggregates below are computed "
|
|
1008
|
+
"over exactly the quote rows persisted in the evidence file; "
|
|
1009
|
+
"hypothesis synthesis happens downstream."
|
|
1010
|
+
)
|
|
1011
|
+
add("")
|
|
1012
|
+
add(
|
|
1013
|
+
"Queries searched: "
|
|
1014
|
+
+ ", ".join(f"`{q}` ({query_sources.get(q, 'explicit')})" for q in queries)
|
|
1015
|
+
+ "."
|
|
1016
|
+
)
|
|
1017
|
+
add("")
|
|
1018
|
+
if product_context:
|
|
1019
|
+
add(
|
|
1020
|
+
"Product-context tokens: "
|
|
1021
|
+
+ ", ".join(f"`{t}`" for t in product_context)
|
|
1022
|
+
+ ". A quote marked product-context mentions this product's "
|
|
1023
|
+
"MECHANISM; all other quotes are bare category demand that may "
|
|
1024
|
+
"belong to different mechanics (for example human-to-human "
|
|
1025
|
+
"realtime collaboration versus working with an AI agent) — do "
|
|
1026
|
+
"not read them as segment evidence for this product without "
|
|
1027
|
+
"checking the quote itself."
|
|
1028
|
+
)
|
|
1029
|
+
add("")
|
|
1030
|
+
|
|
1031
|
+
add("### Search coverage")
|
|
1032
|
+
add("")
|
|
1033
|
+
add("| Query | Source | Items retrieved | Marker-matched quotes | Stored |")
|
|
1034
|
+
add("| --- | --- | ---: | ---: | ---: |")
|
|
1035
|
+
for summary in search_summaries:
|
|
1036
|
+
add(
|
|
1037
|
+
f"| {summary['query']} | {summary['source']} | "
|
|
1038
|
+
f"{summary['retrieved']} | {summary['matched']} | "
|
|
1039
|
+
f"{summary['stored']} |"
|
|
1040
|
+
)
|
|
1041
|
+
add("")
|
|
1042
|
+
|
|
1043
|
+
if communities:
|
|
1044
|
+
add("### Community observations")
|
|
1045
|
+
add("")
|
|
1046
|
+
context_col = " Product-context |" if product_context else ""
|
|
1047
|
+
add(
|
|
1048
|
+
"| Community | Quotes | Payment | Seeking | Pain | Price-sens |"
|
|
1049
|
+
f"{context_col} Votes on quotes | Latest |"
|
|
1050
|
+
)
|
|
1051
|
+
add(
|
|
1052
|
+
"| --- | ---: | ---: | ---: | ---: | ---: |"
|
|
1053
|
+
+ (" ---: |" if product_context else "")
|
|
1054
|
+
+ " ---: | --- |"
|
|
1055
|
+
)
|
|
1056
|
+
for row in communities:
|
|
1057
|
+
context_cell = (
|
|
1058
|
+
f" {row['product_context_quotes']} |" if product_context else ""
|
|
1059
|
+
)
|
|
1060
|
+
add(
|
|
1061
|
+
f"| {row['community']} | {row['quotes']} | {row['wtp_quotes']} | "
|
|
1062
|
+
f"{row['seeking_quotes']} | {row['pain_quotes']} | "
|
|
1063
|
+
f"{row['price_sensitivity_quotes']} |{context_cell} "
|
|
1064
|
+
f"{row['votes_on_quotes']} | {row['latest_quote_date'] or '—'} |"
|
|
1065
|
+
)
|
|
1066
|
+
add("")
|
|
1067
|
+
add(
|
|
1068
|
+
"Counts describe marker matches in the stored sample, not buyer intent, "
|
|
1069
|
+
"relative evidence strength or a channel recommendation. A "
|
|
1070
|
+
"community's absence here means the searched queries surfaced "
|
|
1071
|
+
"nothing marker-matched — weak evidence, not proof of no "
|
|
1072
|
+
"demand."
|
|
1073
|
+
)
|
|
1074
|
+
add("")
|
|
1075
|
+
|
|
1076
|
+
wtp_quotes = [q for q in stored_quotes if "wtp" in q["categories"]]
|
|
1077
|
+
if wtp_quotes:
|
|
1078
|
+
add("### Payment-language quotes")
|
|
1079
|
+
add("")
|
|
1080
|
+
context_col = " Product-ctx |" if product_context else ""
|
|
1081
|
+
add(f"| Community | Date | Votes |{context_col} Quote |")
|
|
1082
|
+
add(
|
|
1083
|
+
"| --- | --- | ---: |"
|
|
1084
|
+
+ (" :-: |" if product_context else "")
|
|
1085
|
+
+ " --- |"
|
|
1086
|
+
)
|
|
1087
|
+
for quote in wtp_quotes[:6]:
|
|
1088
|
+
context_cell = (
|
|
1089
|
+
(" ✓ |" if quote.get("product_context") else " — |")
|
|
1090
|
+
if product_context
|
|
1091
|
+
else ""
|
|
1092
|
+
)
|
|
1093
|
+
add(
|
|
1094
|
+
f"| {quote['community']} | {quote['created'] or '—'} | "
|
|
1095
|
+
f"{quote['votes']} |{context_cell} {_quote_cell(quote)} |"
|
|
1096
|
+
)
|
|
1097
|
+
add("")
|
|
1098
|
+
|
|
1099
|
+
other_quotes = [
|
|
1100
|
+
q for q in stored_quotes if "wtp" not in q["categories"]
|
|
1101
|
+
]
|
|
1102
|
+
if other_quotes:
|
|
1103
|
+
add("### Seeking, pain, and price-sensitivity quotes")
|
|
1104
|
+
add("")
|
|
1105
|
+
context_col = " Product-ctx |" if product_context else ""
|
|
1106
|
+
add(f"| Community | Date | Votes | Markers |{context_col} Quote |")
|
|
1107
|
+
add(
|
|
1108
|
+
"| --- | --- | ---: | --- |"
|
|
1109
|
+
+ (" :-: |" if product_context else "")
|
|
1110
|
+
+ " --- |"
|
|
1111
|
+
)
|
|
1112
|
+
for quote in other_quotes[:10]:
|
|
1113
|
+
context_cell = (
|
|
1114
|
+
(" ✓ |" if quote.get("product_context") else " — |")
|
|
1115
|
+
if product_context
|
|
1116
|
+
else ""
|
|
1117
|
+
)
|
|
1118
|
+
add(
|
|
1119
|
+
f"| {quote['community']} | {quote['created'] or '—'} | "
|
|
1120
|
+
f"{quote['votes']} | {', '.join(quote['categories'])} |"
|
|
1121
|
+
f"{context_cell} {_quote_cell(quote)} |"
|
|
1122
|
+
)
|
|
1123
|
+
add("")
|
|
1124
|
+
|
|
1125
|
+
add("### Method and coverage")
|
|
1126
|
+
add("")
|
|
1127
|
+
add(
|
|
1128
|
+
f"- Spend recorded: {totals['actor_runs']} Reddit actor run(s), "
|
|
1129
|
+
f"{totals['hn_calls']} free HN API call(s), and "
|
|
1130
|
+
f"{totals['youtube_calls']} metered YouTube API call(s), "
|
|
1131
|
+
"accumulated across attempts; cached replays add no new spend."
|
|
1132
|
+
)
|
|
1133
|
+
add(
|
|
1134
|
+
f"- Caps: {MAX_QUERIES} queries, {REDDIT_ITEMS_PER_QUERY} Reddit "
|
|
1135
|
+
f"items per query, {HN_STORY_HITS}+{HN_COMMENT_HITS} HN hits per "
|
|
1136
|
+
f"query, {YT_VIDEOS_PER_QUERY}×{YT_COMMENTS_PER_VIDEO} YouTube "
|
|
1137
|
+
f"comments per query, {STORED_QUOTES_CAP} stored quotes "
|
|
1138
|
+
f"({STORED_PER_COMMUNITY_CAP} per community)."
|
|
1139
|
+
)
|
|
1140
|
+
add(
|
|
1141
|
+
"- Excluded deterministically: Show HN/Launch HN stories and "
|
|
1142
|
+
"self-promo Reddit posts (sellers, not askers), jobs-thread "
|
|
1143
|
+
"comments and separator-formatted classifieds, known bots. A "
|
|
1144
|
+
f"marker counts only within {_ANCHOR_PROXIMITY_WINDOW} characters "
|
|
1145
|
+
"of a non-generic query token, and money mentions count as payment "
|
|
1146
|
+
"language only near a payment word ($100M scale-talk does not)."
|
|
1147
|
+
)
|
|
1148
|
+
add(
|
|
1149
|
+
"- Sources excluded from this run (recorded as excluded_source "
|
|
1150
|
+
"rows in the evidence file): "
|
|
1151
|
+
+ "; ".join(f"{row['source']} — {row['reason']}" for row in excluded_sources)
|
|
1152
|
+
+ ". Communities over-represent complainers and early adopters; "
|
|
1153
|
+
"marker matching misses paraphrased demand and admits some noise."
|
|
1154
|
+
)
|
|
1155
|
+
add("")
|
|
1156
|
+
add("### Sources")
|
|
1157
|
+
add("")
|
|
1158
|
+
add(
|
|
1159
|
+
f"- Hacker News via the keyless Algolia API; Reddit via Apify "
|
|
1160
|
+
f"`{REDDIT_ACTOR}`; YouTube via the Data API v3 (search + comment "
|
|
1161
|
+
f"threads) — retrieved {retrieved_on}; the stored quote rows in "
|
|
1162
|
+
f"`{EVIDENCE_REL.as_posix()}` are exactly the set every table "
|
|
1163
|
+
"above is computed from (texts truncated for storage, markers "
|
|
1164
|
+
"matched on full text before truncation)."
|
|
1165
|
+
)
|
|
1166
|
+
return "\n".join(lines).strip() + "\n"
|
|
1167
|
+
|
|
1168
|
+
|
|
1169
|
+
# ── orchestration ──────────────────────────────────────────────────────────
|
|
1170
|
+
|
|
1171
|
+
@dataclass
|
|
1172
|
+
class RunInputs:
|
|
1173
|
+
queries: list[str]
|
|
1174
|
+
query_sources: dict[str, str]
|
|
1175
|
+
product_context: list[str]
|
|
1176
|
+
|
|
1177
|
+
|
|
1178
|
+
def collect_youtube(client: OsClient, query: str, published_after: str) -> tuple[list[dict[str, Any]], int]:
|
|
1179
|
+
"""One search plus comment pulls for the top videos. Raises on provider
|
|
1180
|
+
failure; the caller decides best-effort semantics."""
|
|
1181
|
+
search, _ = client.yt_call(
|
|
1182
|
+
"search",
|
|
1183
|
+
{
|
|
1184
|
+
"query": query,
|
|
1185
|
+
"max_results": YT_SEARCH_RESULTS,
|
|
1186
|
+
"order": "relevance",
|
|
1187
|
+
"published_after": published_after,
|
|
1188
|
+
},
|
|
1189
|
+
)
|
|
1190
|
+
videos = [v for v in (search.get("rows") or []) if isinstance(v, dict)]
|
|
1191
|
+
candidates: list[dict[str, Any]] = []
|
|
1192
|
+
retrieved = 0
|
|
1193
|
+
for video in videos[:YT_VIDEOS_PER_QUERY]:
|
|
1194
|
+
threads, _ = client.yt_call(
|
|
1195
|
+
"comment-threads",
|
|
1196
|
+
{
|
|
1197
|
+
"video_id": video.get("video_id"),
|
|
1198
|
+
"max_results": YT_COMMENTS_PER_VIDEO,
|
|
1199
|
+
"order": "relevance",
|
|
1200
|
+
},
|
|
1201
|
+
)
|
|
1202
|
+
rows = [r for r in (threads.get("rows") or []) if isinstance(r, dict)]
|
|
1203
|
+
retrieved += len(rows)
|
|
1204
|
+
candidates.extend(
|
|
1205
|
+
c
|
|
1206
|
+
for c in (normalize_youtube_comment(r, query, video) for r in rows)
|
|
1207
|
+
if c
|
|
1208
|
+
)
|
|
1209
|
+
return candidates, retrieved
|
|
1210
|
+
|
|
1211
|
+
|
|
1212
|
+
def _call_with_context(client: OsClient, query: str, label: str, fn: Callable[[], Any]) -> Any:
|
|
1213
|
+
"""Re-raise a provider-call failure with WHICH query/call failed and how
|
|
1214
|
+
much of the attempt's paid budget is already spent — the raw gateway
|
|
1215
|
+
envelope alone doesn't say either, so a human can't tell whether a retry
|
|
1216
|
+
is even possible without this."""
|
|
1217
|
+
try:
|
|
1218
|
+
return fn()
|
|
1219
|
+
except CommunityDemandError as exc:
|
|
1220
|
+
raise CommunityDemandError(
|
|
1221
|
+
f"{label} for `{query}` failed: {exc} — this attempt had "
|
|
1222
|
+
f"already started {client.actor_runs}/{client.max_actor_runs} "
|
|
1223
|
+
"actor run(s); see GET /me/limits?resource=apify:runs for "
|
|
1224
|
+
"remaining daily quota."
|
|
1225
|
+
) from exc
|
|
1226
|
+
|
|
1227
|
+
|
|
1228
|
+
def run_pipeline(client: OsClient, inputs: RunInputs) -> dict[str, Any]:
|
|
1229
|
+
if not inputs.queries:
|
|
1230
|
+
raise CommunityDemandError(
|
|
1231
|
+
"no seed queries available: pass --seed-keywords with 1-5 "
|
|
1232
|
+
"category or problem phrases derived from the research assignment (see the "
|
|
1233
|
+
"workflow doc), or complete the Keyword Economics section first"
|
|
1234
|
+
)
|
|
1235
|
+
|
|
1236
|
+
hn_cutoff_date = date.today() - timedelta(days=HN_RECENCY_DAYS)
|
|
1237
|
+
hn_cutoff = hn_cutoff_date.isoformat()
|
|
1238
|
+
# Midnight-UTC epoch: stable within a day, so same-day replays hit the
|
|
1239
|
+
# cache while a fresh attempt on a later day refetches.
|
|
1240
|
+
hn_cutoff_epoch = int(
|
|
1241
|
+
datetime.combine(hn_cutoff_date, datetime.min.time(), tzinfo=timezone.utc).timestamp()
|
|
1242
|
+
)
|
|
1243
|
+
yt_published_after = (
|
|
1244
|
+
date.today() - timedelta(days=YT_RECENCY_DAYS)
|
|
1245
|
+
).isoformat() + "T00:00:00Z"
|
|
1246
|
+
candidates: list[dict[str, Any]] = []
|
|
1247
|
+
retrieval_counts: list[dict[str, Any]] = []
|
|
1248
|
+
youtube_failure: str | None = None
|
|
1249
|
+
|
|
1250
|
+
for query in inputs.queries:
|
|
1251
|
+
story_hits, _ = client.hn_search(query, "story", HN_STORY_HITS, hn_cutoff_epoch)
|
|
1252
|
+
comment_hits, _ = client.hn_search(query, "comment", HN_COMMENT_HITS, hn_cutoff_epoch)
|
|
1253
|
+
reddit_rows, _ = _call_with_context(
|
|
1254
|
+
client,
|
|
1255
|
+
query,
|
|
1256
|
+
"Reddit",
|
|
1257
|
+
lambda: client.run_actor(
|
|
1258
|
+
REDDIT_ACTOR,
|
|
1259
|
+
{
|
|
1260
|
+
"searches": [query],
|
|
1261
|
+
"searchPosts": True,
|
|
1262
|
+
"searchComments": True,
|
|
1263
|
+
"searchCommunities": False,
|
|
1264
|
+
"searchUsers": False,
|
|
1265
|
+
"skipUserPosts": True,
|
|
1266
|
+
"includeMediaLinks": True,
|
|
1267
|
+
"sort": "relevance",
|
|
1268
|
+
"time": "year",
|
|
1269
|
+
"maxItems": REDDIT_ITEMS_PER_QUERY,
|
|
1270
|
+
"proxy": {"useApifyProxy": True},
|
|
1271
|
+
},
|
|
1272
|
+
limit=REDDIT_ITEMS_PER_QUERY,
|
|
1273
|
+
),
|
|
1274
|
+
)
|
|
1275
|
+
|
|
1276
|
+
hn_candidates = [
|
|
1277
|
+
c
|
|
1278
|
+
for c in (
|
|
1279
|
+
[normalize_hn_story(h, query) for h in story_hits]
|
|
1280
|
+
+ [normalize_hn_comment(h, query) for h in comment_hits]
|
|
1281
|
+
)
|
|
1282
|
+
if c
|
|
1283
|
+
]
|
|
1284
|
+
reddit_candidates = [
|
|
1285
|
+
c for c in (normalize_reddit_row(r, query) for r in reddit_rows) if c
|
|
1286
|
+
]
|
|
1287
|
+
candidates.extend(hn_candidates)
|
|
1288
|
+
candidates.extend(reddit_candidates)
|
|
1289
|
+
retrieval_counts.append(
|
|
1290
|
+
{
|
|
1291
|
+
"query": query,
|
|
1292
|
+
"source": "hackernews",
|
|
1293
|
+
"retrieved": len(story_hits) + len(comment_hits),
|
|
1294
|
+
"normalized": len(hn_candidates),
|
|
1295
|
+
}
|
|
1296
|
+
)
|
|
1297
|
+
retrieval_counts.append(
|
|
1298
|
+
{
|
|
1299
|
+
"query": query,
|
|
1300
|
+
"source": "reddit",
|
|
1301
|
+
"retrieved": len(reddit_rows),
|
|
1302
|
+
"normalized": len(reddit_candidates),
|
|
1303
|
+
}
|
|
1304
|
+
)
|
|
1305
|
+
if youtube_failure is None:
|
|
1306
|
+
try:
|
|
1307
|
+
yt_candidates, yt_retrieved = collect_youtube(
|
|
1308
|
+
client, query, yt_published_after
|
|
1309
|
+
)
|
|
1310
|
+
except CommunityDemandError as exc:
|
|
1311
|
+
# Best-effort: an undeployed provider or exhausted
|
|
1312
|
+
# fleet-shared quota degrades to a recorded exclusion, never
|
|
1313
|
+
# a silent gap and never a failed section.
|
|
1314
|
+
youtube_failure = f"unavailable during this run: {str(exc)[:200]}"
|
|
1315
|
+
else:
|
|
1316
|
+
candidates.extend(yt_candidates)
|
|
1317
|
+
retrieval_counts.append(
|
|
1318
|
+
{
|
|
1319
|
+
"query": query,
|
|
1320
|
+
"source": "youtube",
|
|
1321
|
+
"retrieved": yt_retrieved,
|
|
1322
|
+
"normalized": len(yt_candidates),
|
|
1323
|
+
}
|
|
1324
|
+
)
|
|
1325
|
+
|
|
1326
|
+
# The same item can match several queries: first query wins, once.
|
|
1327
|
+
seen_ids: set[str] = set()
|
|
1328
|
+
quotes: list[dict[str, Any]] = []
|
|
1329
|
+
for candidate in candidates:
|
|
1330
|
+
if candidate["id"] in seen_ids:
|
|
1331
|
+
continue
|
|
1332
|
+
seen_ids.add(candidate["id"])
|
|
1333
|
+
quote = to_quote_row(candidate, hn_cutoff, inputs.product_context)
|
|
1334
|
+
if quote:
|
|
1335
|
+
quotes.append(quote)
|
|
1336
|
+
|
|
1337
|
+
stored_quotes = select_stored_quotes(quotes)
|
|
1338
|
+
communities = summarize_communities(stored_quotes)
|
|
1339
|
+
|
|
1340
|
+
search_summaries = [
|
|
1341
|
+
{
|
|
1342
|
+
"type": "search_summary",
|
|
1343
|
+
"query": count["query"],
|
|
1344
|
+
"source": count["source"],
|
|
1345
|
+
"retrieved": count["retrieved"],
|
|
1346
|
+
"matched": sum(
|
|
1347
|
+
1
|
|
1348
|
+
for q in quotes
|
|
1349
|
+
if q["query"] == count["query"] and q["source"] == count["source"]
|
|
1350
|
+
),
|
|
1351
|
+
"stored": sum(
|
|
1352
|
+
1
|
|
1353
|
+
for q in stored_quotes
|
|
1354
|
+
if q["query"] == count["query"] and q["source"] == count["source"]
|
|
1355
|
+
),
|
|
1356
|
+
"tier": "A",
|
|
1357
|
+
}
|
|
1358
|
+
for count in retrieval_counts
|
|
1359
|
+
]
|
|
1360
|
+
|
|
1361
|
+
totals = client.ledger_totals()
|
|
1362
|
+
retrieved_on = (
|
|
1363
|
+
totals["first_retrieved_on"]
|
|
1364
|
+
if totals["first_retrieved_on"] == totals["last_retrieved_on"]
|
|
1365
|
+
else f"{totals['first_retrieved_on']}..{totals['last_retrieved_on']}"
|
|
1366
|
+
)
|
|
1367
|
+
|
|
1368
|
+
excluded_sources = excluded_source_rows(youtube_failure)
|
|
1369
|
+
evidence = {
|
|
1370
|
+
"schema_version": SCHEMA_VERSION,
|
|
1371
|
+
"section_id": SECTION_ID,
|
|
1372
|
+
"generated_by": Path(__file__).name,
|
|
1373
|
+
"retrieved_on": retrieved_on,
|
|
1374
|
+
"marker_list_version": MARKER_LIST_VERSION,
|
|
1375
|
+
"inputs": {
|
|
1376
|
+
"queries": inputs.queries,
|
|
1377
|
+
"query_sources": inputs.query_sources,
|
|
1378
|
+
"product_context": inputs.product_context,
|
|
1379
|
+
},
|
|
1380
|
+
"rows": (
|
|
1381
|
+
search_summaries
|
|
1382
|
+
+ communities
|
|
1383
|
+
+ stored_quotes
|
|
1384
|
+
+ excluded_sources
|
|
1385
|
+
),
|
|
1386
|
+
"totals": {
|
|
1387
|
+
"actor_runs": totals["actor_runs"],
|
|
1388
|
+
"provider_calls": totals["provider_calls"],
|
|
1389
|
+
"hn_calls": totals["hn_calls"],
|
|
1390
|
+
"youtube_calls": totals["youtube_calls"],
|
|
1391
|
+
},
|
|
1392
|
+
}
|
|
1393
|
+
|
|
1394
|
+
section_markdown = render_section(
|
|
1395
|
+
queries=inputs.queries,
|
|
1396
|
+
query_sources=inputs.query_sources,
|
|
1397
|
+
product_context=inputs.product_context,
|
|
1398
|
+
search_summaries=search_summaries,
|
|
1399
|
+
communities=communities,
|
|
1400
|
+
stored_quotes=stored_quotes,
|
|
1401
|
+
excluded_sources=excluded_sources,
|
|
1402
|
+
totals=totals,
|
|
1403
|
+
retrieved_on=retrieved_on,
|
|
1404
|
+
)
|
|
1405
|
+
return {
|
|
1406
|
+
"evidence": evidence,
|
|
1407
|
+
"section_markdown": section_markdown,
|
|
1408
|
+
"totals": totals,
|
|
1409
|
+
"attempt": {
|
|
1410
|
+
"actor_runs": client.actor_runs,
|
|
1411
|
+
"hn_calls": client.hn_calls,
|
|
1412
|
+
"yt_calls": client.yt_calls,
|
|
1413
|
+
"cache_hits": client.cache_hits,
|
|
1414
|
+
},
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
|
|
1418
|
+
def _write_atomic(path: Path, content: str) -> None:
|
|
1419
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
1420
|
+
tmp = path.with_suffix(path.suffix + ".tmp")
|
|
1421
|
+
tmp.write_text(content, encoding="utf-8")
|
|
1422
|
+
tmp.replace(path)
|
|
1423
|
+
|
|
1424
|
+
|
|
1425
|
+
def _canonicalize_legacy_headings(upsert: ModuleType, report_path: Path) -> None:
|
|
1426
|
+
"""Fold pre-rename legacy H2s of this section into the canonical heading.
|
|
1427
|
+
|
|
1428
|
+
A report materialized by an older bundle keeps `## Community demand`; the
|
|
1429
|
+
PRD extractor and this runner's upsert address the canonical H2 only, so a
|
|
1430
|
+
stale legacy heading would either fail extraction or make the section
|
|
1431
|
+
coexist under two names after a rerun. Rename the legacy H2 in place, or
|
|
1432
|
+
drop the stale legacy section when a canonical one already exists.
|
|
1433
|
+
"""
|
|
1434
|
+
if not report_path.is_file():
|
|
1435
|
+
return
|
|
1436
|
+
text = report_path.read_text(encoding="utf-8")
|
|
1437
|
+
changed = False
|
|
1438
|
+
for legacy in LEGACY_SECTION_HEADINGS:
|
|
1439
|
+
while True:
|
|
1440
|
+
offsets = upsert._h2_offsets(text)
|
|
1441
|
+
match = next(
|
|
1442
|
+
(
|
|
1443
|
+
(index, offset)
|
|
1444
|
+
for index, (name, offset) in enumerate(offsets)
|
|
1445
|
+
if name == legacy
|
|
1446
|
+
),
|
|
1447
|
+
None,
|
|
1448
|
+
)
|
|
1449
|
+
if match is None:
|
|
1450
|
+
break
|
|
1451
|
+
canonical_present = any(
|
|
1452
|
+
name == SECTION_HEADING for name, _ in offsets
|
|
1453
|
+
)
|
|
1454
|
+
index, start = match
|
|
1455
|
+
if canonical_present:
|
|
1456
|
+
end = (
|
|
1457
|
+
offsets[index + 1][1]
|
|
1458
|
+
if index + 1 < len(offsets)
|
|
1459
|
+
else len(text)
|
|
1460
|
+
)
|
|
1461
|
+
text = text[:start] + text[end:]
|
|
1462
|
+
else:
|
|
1463
|
+
line_end = text.find("\n", start)
|
|
1464
|
+
if line_end == -1:
|
|
1465
|
+
line_end = len(text)
|
|
1466
|
+
text = text[:start] + f"## {SECTION_HEADING}" + text[line_end:]
|
|
1467
|
+
changed = True
|
|
1468
|
+
if changed:
|
|
1469
|
+
_write_atomic(report_path, text)
|
|
1470
|
+
|
|
1471
|
+
|
|
1472
|
+
def run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
|
|
1473
|
+
workspace = workspace.resolve()
|
|
1474
|
+
io = _load_script("research-artifact-io.py", "pmr_artifact_io")
|
|
1475
|
+
with io.section_attempt(workspace, SECTION_ID, context_file=getattr(args, "context_file", None)):
|
|
1476
|
+
return _run(workspace, args)
|
|
1477
|
+
|
|
1478
|
+
|
|
1479
|
+
def _run(workspace: Path, args: argparse.Namespace) -> dict[str, Any]:
|
|
1480
|
+
workspace = workspace.resolve()
|
|
1481
|
+
runtime = workspace / RUNTIME_REL
|
|
1482
|
+
cache_dir = runtime / "cache"
|
|
1483
|
+
runtime.mkdir(parents=True, exist_ok=True)
|
|
1484
|
+
|
|
1485
|
+
api_key = (os.environ.get("OS_API_KEY") or "").strip()
|
|
1486
|
+
if not api_key:
|
|
1487
|
+
raise CommunityDemandError("OS_API_KEY is required")
|
|
1488
|
+
os_url = (os.environ.get("OS_URL") or DEFAULT_OS_URL).strip() or DEFAULT_OS_URL
|
|
1489
|
+
|
|
1490
|
+
explicit = parse_terms(args.seed_keywords or "", MAX_QUERIES)
|
|
1491
|
+
discovered = queries_from_keyword_evidence(workspace, MAX_QUERIES)
|
|
1492
|
+
queries, query_sources = select_queries(explicit, discovered, args.max_queries)
|
|
1493
|
+
# One actor run and two HN calls per query: never admit more queries
|
|
1494
|
+
# than the required budgets can finish in one attempt (YouTube is
|
|
1495
|
+
# best-effort and merely stops when its own budget runs out).
|
|
1496
|
+
queries = queries[: max(1, min(args.max_actor_runs, args.max_hn_calls // 2))]
|
|
1497
|
+
query_sources = {q: query_sources[q] for q in queries}
|
|
1498
|
+
|
|
1499
|
+
inputs = RunInputs(
|
|
1500
|
+
queries=queries,
|
|
1501
|
+
query_sources=query_sources,
|
|
1502
|
+
product_context=parse_context_tokens(args.product_context or ""),
|
|
1503
|
+
)
|
|
1504
|
+
client = OsClient(
|
|
1505
|
+
os_url=os_url,
|
|
1506
|
+
api_key=api_key,
|
|
1507
|
+
cache_dir=cache_dir,
|
|
1508
|
+
max_actor_runs=args.max_actor_runs,
|
|
1509
|
+
max_hn_calls=args.max_hn_calls,
|
|
1510
|
+
max_yt_calls=args.max_yt_calls,
|
|
1511
|
+
)
|
|
1512
|
+
|
|
1513
|
+
outcome = run_pipeline(client, inputs)
|
|
1514
|
+
|
|
1515
|
+
evidence_path = workspace / EVIDENCE_REL
|
|
1516
|
+
evidence_json = json.dumps(outcome["evidence"], indent=1, ensure_ascii=False)
|
|
1517
|
+
_write_atomic(evidence_path, evidence_json)
|
|
1518
|
+
|
|
1519
|
+
section_file = runtime / "result.md"
|
|
1520
|
+
_write_atomic(section_file, outcome["section_markdown"])
|
|
1521
|
+
|
|
1522
|
+
upsert = _load_script("upsert-markdown-section.py", "pmr_upsert_markdown_section")
|
|
1523
|
+
report_path = workspace / REPORT_REL
|
|
1524
|
+
report_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1525
|
+
_canonicalize_legacy_headings(upsert, report_path)
|
|
1526
|
+
upsert.upsert_markdown_section(report_path, SECTION_HEADING, outcome["section_markdown"])
|
|
1527
|
+
|
|
1528
|
+
persist = _load_script("persist-research-report.py", "pmr_persist_research_report")
|
|
1529
|
+
persistence = persist.persist_research_report(
|
|
1530
|
+
workspace,
|
|
1531
|
+
commit_message=f"research: update {SECTION_HEADING.lower()} section",
|
|
1532
|
+
extra_paths=(Path(_artifact_path("egbe-docs/research/market/evidence/community-demand.json")),),
|
|
1533
|
+
)
|
|
1534
|
+
|
|
1535
|
+
state = {
|
|
1536
|
+
"schema_version": SCHEMA_VERSION,
|
|
1537
|
+
"section_id": SECTION_ID,
|
|
1538
|
+
"status": "completed",
|
|
1539
|
+
"queries": inputs.queries,
|
|
1540
|
+
"query_sources": inputs.query_sources,
|
|
1541
|
+
"product_context": inputs.product_context,
|
|
1542
|
+
"evidence_path": EVIDENCE_REL.as_posix(),
|
|
1543
|
+
"evidence_sha256": hashlib.sha256(evidence_json.encode("utf-8")).hexdigest(),
|
|
1544
|
+
"result_path": (RUNTIME_REL / "result.md").as_posix(),
|
|
1545
|
+
"result_sha256": hashlib.sha256(outcome["section_markdown"].encode("utf-8")).hexdigest(),
|
|
1546
|
+
"provider_calls": outcome["totals"]["provider_calls"],
|
|
1547
|
+
"actor_runs": outcome["totals"]["actor_runs"],
|
|
1548
|
+
"hn_calls": outcome["totals"]["hn_calls"],
|
|
1549
|
+
"youtube_calls": outcome["totals"]["youtube_calls"],
|
|
1550
|
+
"actor_starts_recorded": outcome["totals"].get("actor_starts_recorded", 0),
|
|
1551
|
+
"attempt_actor_runs": outcome["attempt"]["actor_runs"],
|
|
1552
|
+
"attempt_hn_calls": outcome["attempt"]["hn_calls"],
|
|
1553
|
+
"attempt_yt_calls": outcome["attempt"]["yt_calls"],
|
|
1554
|
+
"attempt_cache_hits": outcome["attempt"]["cache_hits"],
|
|
1555
|
+
"report_commit_sha": persistence.commit_sha,
|
|
1556
|
+
"publication_mode": persistence.publication_mode,
|
|
1557
|
+
"research_context": _load_script("research-artifact-io.py", "pmr_input_io").context_binding(workspace, SECTION_ID),
|
|
1558
|
+
"error": None,
|
|
1559
|
+
}
|
|
1560
|
+
_write_atomic(runtime / "request.json", json.dumps(state, indent=1))
|
|
1561
|
+
return state
|
|
1562
|
+
|
|
1563
|
+
|
|
1564
|
+
def main() -> int:
|
|
1565
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
1566
|
+
parser.add_argument("--context-file", type=Path, help="Explicit context for this study; no project files are discovered implicitly.")
|
|
1567
|
+
parser.add_argument("--workspace", type=Path, required=True)
|
|
1568
|
+
parser.add_argument(
|
|
1569
|
+
"--seed-keywords",
|
|
1570
|
+
default="",
|
|
1571
|
+
help="Comma-separated category AND problem phrases derived from "
|
|
1572
|
+
"the research assignment; merged ahead of the top keywords discovered by Keyword "
|
|
1573
|
+
"Economics.",
|
|
1574
|
+
)
|
|
1575
|
+
parser.add_argument(
|
|
1576
|
+
"--product-context",
|
|
1577
|
+
default="",
|
|
1578
|
+
help="Comma-separated mechanism tokens from the research assignment (e.g. 'ai, "
|
|
1579
|
+
"agent'); quotes mentioning them are flagged as speaking to THIS "
|
|
1580
|
+
"product's mechanism rather than the bare category.",
|
|
1581
|
+
)
|
|
1582
|
+
parser.add_argument("--max-queries", type=int, default=MAX_QUERIES)
|
|
1583
|
+
parser.add_argument("--max-actor-runs", type=int, default=MAX_ACTOR_RUNS_PER_ATTEMPT)
|
|
1584
|
+
parser.add_argument("--max-hn-calls", type=int, default=MAX_HN_CALLS_PER_ATTEMPT)
|
|
1585
|
+
parser.add_argument("--max-yt-calls", type=int, default=MAX_YT_CALLS_PER_ATTEMPT)
|
|
1586
|
+
args = parser.parse_args()
|
|
1587
|
+
|
|
1588
|
+
try:
|
|
1589
|
+
state = run(args.workspace, args)
|
|
1590
|
+
except (CommunityDemandError, ValueError) as exc:
|
|
1591
|
+
print(f"{DISPLAY_NAME}: {exc}", file=sys.stderr)
|
|
1592
|
+
return 1
|
|
1593
|
+
print(
|
|
1594
|
+
f"{DISPLAY_NAME}: completed — {state['actor_runs']} actor run(s), "
|
|
1595
|
+
f"{state['hn_calls']} HN call(s), {state['youtube_calls']} YouTube "
|
|
1596
|
+
f"call(s), evidence at {state['evidence_path']}, report commit "
|
|
1597
|
+
f"{state['report_commit_sha'][:12]}"
|
|
1598
|
+
)
|
|
1599
|
+
return 0
|
|
1600
|
+
|
|
1601
|
+
|
|
1602
|
+
if __name__ == "__main__":
|
|
1603
|
+
raise SystemExit(main())
|