@egbe-ai/opencode 0.0.0-stage → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +124 -2
- package/index.js +5 -0
- package/package.json +55 -4
- package/plugin.js +230 -0
- package/session-log.js +315 -0
- package/skills/.egbe-catalog/manifest.json +1 -0
- package/skills/egbe-os-analytics/SKILL.md +499 -0
- package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
- package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
- package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
- package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
- package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
- package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
- package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
- package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
- package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
- package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
- package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
- package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
- package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
- package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
- package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
- package/skills/egbe-os-db/SKILL.md +174 -0
- package/skills/egbe-os-deploy/SKILL.md +412 -0
- package/skills/egbe-os-design/SKILL.md +182 -0
- package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
- package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
- package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
- package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
- package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
- package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
- package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
- package/skills/egbe-os-destination-readiness/references/report.md +116 -0
- package/skills/egbe-os-domains/SKILL.md +189 -0
- package/skills/egbe-os-errors/SKILL.md +259 -0
- package/skills/egbe-os-git/SKILL.md +275 -0
- package/skills/egbe-os-git/references/issues.md +33 -0
- package/skills/egbe-os-google-ads/SKILL.md +163 -0
- package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
- package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
- package/skills/egbe-os-google-ads/references/conversions.md +153 -0
- package/skills/egbe-os-google-ads/references/delivery.md +128 -0
- package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
- package/skills/egbe-os-google-ads/references/reporting.md +105 -0
- package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
- package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
- package/skills/egbe-os-google-auth/SKILL.md +285 -0
- package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
- package/skills/egbe-os-images/SKILL.md +163 -0
- package/skills/egbe-os-images/references/stock-images.md +21 -0
- package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
- package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
- package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
- package/skills/egbe-os-llm/SKILL.md +160 -0
- package/skills/egbe-os-mail/SKILL.md +180 -0
- package/skills/egbe-os-me/SKILL.md +119 -0
- package/skills/egbe-os-meta-ads/SKILL.md +114 -0
- package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
- package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
- package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
- package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
- package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
- package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
- package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
- package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
- package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
- package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
- package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
- package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
- package/skills/egbe-os-notifications/SKILL.md +78 -0
- package/skills/egbe-os-openai-ads/SKILL.md +149 -0
- package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
- package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
- package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
- package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
- package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
- package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
- package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
- package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
- package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
- package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
- package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
- package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
- package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
- package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
- package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
- package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
- package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
- package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
- package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
- package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
- package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
- package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
- package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
- package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
- package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
- package/skills/egbe-os-payments/SKILL.md +215 -0
- package/skills/egbe-os-payments/agents/openai.yaml +4 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
- package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
- package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
- package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
- package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
- package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
- package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
- package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
- package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
- package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
- package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
- package/skills/egbe-os-playbook/SKILL.md +79 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
- package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
- package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
- package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
- package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
- package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
- package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
- package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
- package/skills/egbe-os-playbook/playbooks/product.md +67 -0
- package/skills/egbe-os-playbook/references/catalog.md +127 -0
- package/skills/egbe-os-playbook/references/changes.md +167 -0
- package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
- package/skills/egbe-os-product-auth/SKILL.md +151 -0
- package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
- package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
- package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
- package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
- package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
- package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
- package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
- package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
- package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
- package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
- package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
- package/skills/egbe-os-product-build/SKILL.md +111 -0
- package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-build/references/capabilities.md +25 -0
- package/skills/egbe-os-product-build/references/engineering.md +72 -0
- package/skills/egbe-os-product-build/references/execution.md +55 -0
- package/skills/egbe-os-product-build/references/foundations.md +93 -0
- package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
- package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
- package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
- package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
- package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
- package/skills/egbe-os-product-market-research/SKILL.md +159 -0
- package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
- package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
- package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
- package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
- package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
- package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
- package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
- package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
- package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
- package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
- package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
- package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
- package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
- package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
- package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
- package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
- package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
- package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
- package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
- package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
- package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
- package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
- package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
- package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
- package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
- package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
- package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
- package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
- package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
- package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
- package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
- package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
- package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
- package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
- package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
- package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
- package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
- package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
- package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
- package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
- package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
- package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
- package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
- package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
- package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
- package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
- package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
- package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
- package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
- package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
- package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
- package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
- package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
- package/skills/egbe-os-product-report/SKILL.md +150 -0
- package/skills/egbe-os-product-report/references/artifacts.md +84 -0
- package/skills/egbe-os-product-report/references/brief.md +34 -0
- package/skills/egbe-os-product-report/references/daily.md +76 -0
- package/skills/egbe-os-product-report/references/sources.md +68 -0
- package/skills/egbe-os-product-routines/SKILL.md +166 -0
- package/skills/egbe-os-product-routines/references/catalog.md +22 -0
- package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
- package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
- package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
- package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
- package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
- package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
- package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
- package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
- package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
- package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
- package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
- package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
- package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
- package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
- package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
- package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
- package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
- package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
- package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
- package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
- package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
- package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
- package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
- package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
- package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
- package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
- package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
- package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
- package/skills/egbe-os-report-ui/SKILL.md +133 -0
- package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
- package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
- package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
- package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
- package/skills/egbe-os-report-ui/references/components.md +209 -0
- package/skills/egbe-os-research/SKILL.md +125 -0
- package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
- package/skills/egbe-os-scrape/SKILL.md +160 -0
- package/skills/egbe-os-search/SKILL.md +182 -0
- package/skills/egbe-os-search/references/google-places.md +81 -0
- package/skills/egbe-os-support/SKILL.md +149 -0
- package/skills/egbe-os-telemetry/SKILL.md +344 -0
- package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
- package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
- package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
- package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
- package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
- package/skills/egbe-os-tools/SKILL.md +180 -0
- package/skills/egbe-os-tools/references/theona.md +26 -0
- package/skills/egbe-os-wallet/SKILL.md +181 -0
- package/skills/egbe-setup/EGBE.md +484 -0
- package/skills/egbe-setup/SKILL.md +116 -0
- package/skills/egbe-setup/references/project.md +80 -0
- package/state.js +37 -0
- package/update.js +361 -0
|
@@ -0,0 +1,1543 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Lifecycle Capture runner (internal section of the competitor-led process).
|
|
3
|
+
|
|
4
|
+
What the competitors send after someone signs up: the welcome sequence, its
|
|
5
|
+
cadence, offers, discounts and deadlines. The claw's own mailbox is the
|
|
6
|
+
secret shopper: one ordinary plus-addressed alias per competitor, plus stable
|
|
7
|
+
per-funnel aliases imported from section 05
|
|
8
|
+
(`<local>+NN@<domain>`, NN from 10–99), so every message is attributed by the
|
|
9
|
+
address it was sent to.
|
|
10
|
+
|
|
11
|
+
Two phases, one command:
|
|
12
|
+
|
|
13
|
+
- Start: the runner plans one alias per competitor and names the entry
|
|
14
|
+
points from an agent's reviewed discovery; the assigned human subscribes (newsletter, free account, trial
|
|
15
|
+
or waitlist that needs no payment) in the browser and the agent records each
|
|
16
|
+
subscription in `subscriptions.json`. The runner validates the file and
|
|
17
|
+
publishes the subscriptions. Until every competitor is subscribed or
|
|
18
|
+
skipped with a reason, the receipt stays `pending`.
|
|
19
|
+
- Harvest: the runner reads the inbox by alias through OS (`GET /mail/inbox`
|
|
20
|
+
with `to` and `since`), reads new messages once each, extracts cadence,
|
|
21
|
+
offers, discounts and deadlines, and republishes the section. Run it
|
|
22
|
+
again as mail accumulates; a later harvest replaces the earlier one.
|
|
23
|
+
|
|
24
|
+
Policy, approved for this section only: research sign-ups on competitor
|
|
25
|
+
sites with the claw's plus-addressed mailbox alias; no payments and no card
|
|
26
|
+
details; no agent-created accounts or required phone; no CAPTCHA bypass;
|
|
27
|
+
no real third-party identities; unsubscribe when
|
|
28
|
+
the capture ends. Message bodies stay in the runtime cache; the evidence
|
|
29
|
+
carries subjects and extracted facts, never bodies or excerpts.
|
|
30
|
+
|
|
31
|
+
Usage:
|
|
32
|
+
python3 run-lifecycle-capture.py --workspace . --product-domain product.com # start, or harvest once subscribed
|
|
33
|
+
python3 run-lifecycle-capture.py --workspace . --product-domain product.com --harvest # harvest explicitly
|
|
34
|
+
python3 run-lifecycle-capture.py --workspace . --funnel-mail --refresh # same-day 05 mail
|
|
35
|
+
"""
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import sys as _artifact_sys
|
|
39
|
+
from pathlib import Path as _ArtifactPath
|
|
40
|
+
_artifact_sys.path.insert(0, str(_ArtifactPath(__file__).resolve().parent))
|
|
41
|
+
from egbe_artifacts import artifact_path as _artifact_path
|
|
42
|
+
|
|
43
|
+
import runpy as _egbe_runpy
|
|
44
|
+
from pathlib import Path as _EgbePath
|
|
45
|
+
_egbe_adapter = _EgbePath(__file__).with_name("connection_identity.py")
|
|
46
|
+
_connection_headers = _egbe_runpy.run_path(str(_egbe_adapter))["connection_headers"] if _egbe_adapter.is_file() else lambda _url: {}
|
|
47
|
+
|
|
48
|
+
import argparse
|
|
49
|
+
import fcntl
|
|
50
|
+
import hashlib
|
|
51
|
+
import html
|
|
52
|
+
import importlib.util
|
|
53
|
+
import json
|
|
54
|
+
import os
|
|
55
|
+
import re
|
|
56
|
+
import sys
|
|
57
|
+
import tempfile
|
|
58
|
+
import urllib.error
|
|
59
|
+
import urllib.parse
|
|
60
|
+
import urllib.request
|
|
61
|
+
import uuid
|
|
62
|
+
from collections import Counter
|
|
63
|
+
from dataclasses import dataclass, field
|
|
64
|
+
from datetime import date, datetime, timedelta, timezone
|
|
65
|
+
from email.utils import getaddresses, parseaddr, parsedate_to_datetime
|
|
66
|
+
from pathlib import Path
|
|
67
|
+
from typing import Any, Callable
|
|
68
|
+
|
|
69
|
+
SCHEMA_VERSION = 1
|
|
70
|
+
SECTION_ID = "lifecycle_capture"
|
|
71
|
+
SECTION_HEADING = "Lifecycle capture"
|
|
72
|
+
DISPLAY_NAME = "Lifecycle Capture"
|
|
73
|
+
DEFAULT_OS_URL = "https://os.hz.egbe.dev"
|
|
74
|
+
RUNTIME_REL = Path(_artifact_path("egbe-docs/research/market/work")) / SECTION_ID
|
|
75
|
+
EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/lifecycle-capture.json"))
|
|
76
|
+
REPORT_REL = Path(_artifact_path("egbe-docs/research/market/full-report.md"))
|
|
77
|
+
FUNNEL_TEARDOWN_EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/funnel-teardown.json"))
|
|
78
|
+
SUBSCRIPTIONS_FILENAME = "subscriptions.json"
|
|
79
|
+
PLAN_FILENAME = "plan.json"
|
|
80
|
+
ENTRY_POINTS_FILENAME = "entry-points.json"
|
|
81
|
+
OPERATOR_ACTIONS_FILENAME = "operator-actions.json"
|
|
82
|
+
IDENTITY_REL = Path(_artifact_path("egbe-docs/research/market/work/identity-registry.json"))
|
|
83
|
+
FUNNEL_HANDOFF_REL = Path(_artifact_path("egbe-docs/research/market/work/funnel_teardown/lifecycle-handoff.json"))
|
|
84
|
+
FUNNEL_EVENTS_FILENAME = "funnel-events.json"
|
|
85
|
+
FUNNEL_KINDS = ("lead_form", "quiz", "webinar_registration")
|
|
86
|
+
ALIAS_PREFIX = "lc-"
|
|
87
|
+
|
|
88
|
+
MAX_COMPETITORS = 10
|
|
89
|
+
MAX_INBOX_CALLS_PER_ATTEMPT = 30 # one per subscription per harvest; each is one IMAP login at the mail server
|
|
90
|
+
MAX_MESSAGE_READS_PER_ATTEMPT = 60 # bodies are read once and cached for good
|
|
91
|
+
INBOX_PAGE = 50
|
|
92
|
+
HARVEST_AFTER_DAYS = 14 # the sequence worth reading has had time to arrive
|
|
93
|
+
EARLY_HARVEST_AFTER_DAYS = 3
|
|
94
|
+
EXCERPT_CHARS = 160
|
|
95
|
+
SUBSCRIPTION_KINDS = ("newsletter", "account", "trial", "waitlist", "lead_form")
|
|
96
|
+
|
|
97
|
+
_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
|
98
|
+
_URL_RE = re.compile(r"^https?://[^\s<>\"']+$", re.IGNORECASE)
|
|
99
|
+
_CONFIRM_RE = re.compile(r"\b(?:confirm|verify|verification|activate|validate)\b", re.IGNORECASE)
|
|
100
|
+
_DEADLINE_RE = re.compile(r"\b(?:ends? (?:today|tonight|tomorrow|soon|in \d+ (?:hours?|days?))|expires?(?: in \d+ (?:hours?|days?))?|last (?:chance|day|call)|"
|
|
101
|
+
r"today only|\d+ hours? left|final hours|limited time|don't miss|hurry|offer ends|sale ends|closing soon)\b", re.IGNORECASE)
|
|
102
|
+
_DISCOUNT_RE = re.compile(r"(?<!\d)(\d{1,2})\s?%\s?(?:off|discount|savings?)|(?:save|discount of)\s*(?:up to\s*)?(\d{1,2})\s?%", re.IGNORECASE)
|
|
103
|
+
_AMOUNT = r"\d{1,3}(?:[., ]\d{3})+(?:[.,]\d{1,2})?|\d+(?:[.,]\d{1,2})?"
|
|
104
|
+
_PRICE_RE = re.compile(rf"(?:[$€£]\s?(?:{_AMOUNT})(?!\d)(?:\s*(?:/\s?(?:mo|month|yr|year)\b|per\s+(?:month|year)|a\s+(?:month|year)))?"
|
|
105
|
+
rf"|(?<!\d)(?:{_AMOUNT})\s?(?:USD|EUR|GBP|€|£)(?![a-z]))", re.IGNORECASE)
|
|
106
|
+
_FREE_RE = re.compile(r"\b(?:free trial|for free|100% free|try (?:it )?free|free (?:plan|month|week|course|lesson)|no credit card)\b", re.IGNORECASE)
|
|
107
|
+
_UNSUB_RE = re.compile(r"unsubscribe|manage (?:your )?(?:email )?preferences|opt[- ]out", re.IGNORECASE)
|
|
108
|
+
_PROGRESS_RE = re.compile(r"\b(?:your progress|pick up where|continue (?:your|learning|the course)|you(?:'| a)re almost|streak|keep going|don't stop|still waiting)\b", re.IGNORECASE)
|
|
109
|
+
_LINK_RE = re.compile(r"https?://[^\s<>\"')\]]+", re.IGNORECASE)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _load_script(filename: str, module_name: str):
|
|
113
|
+
path = Path(__file__).with_name(filename)
|
|
114
|
+
spec = importlib.util.spec_from_file_location(module_name, path)
|
|
115
|
+
if spec is None or spec.loader is None:
|
|
116
|
+
raise RuntimeError(f"cannot load {filename}")
|
|
117
|
+
module = importlib.util.module_from_spec(spec)
|
|
118
|
+
sys.dont_write_bytecode = True
|
|
119
|
+
sys.modules[module_name] = module
|
|
120
|
+
spec.loader.exec_module(module)
|
|
121
|
+
return module
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
CF = _load_script("run-channel-footprint.py", "pmr_channel_footprint_for_lifecycle_capture")
|
|
125
|
+
LifecycleCaptureError = CF.ChannelFootprintError
|
|
126
|
+
BudgetExhausted = CF.BudgetExhausted
|
|
127
|
+
registrable_domain = CF.registrable_domain
|
|
128
|
+
brand_token = CF.brand_token
|
|
129
|
+
START = _load_script("lifecycle-start.py", "pmr_lifecycle_start")
|
|
130
|
+
FUNNEL = _load_script("funnel-model.py", "pmr_funnel_model_for_lifecycle_mail")
|
|
131
|
+
|
|
132
|
+
# Mail-specific access material, not a general personal-data filter. Ordinary
|
|
133
|
+
# bodies, addresses, offer codes and campaign URLs remain portable evidence.
|
|
134
|
+
_MAIL_LINK_RE = re.compile(r"(?:https?://|https?%3a%2f%2f)[^\s<>\"')\]]+", re.IGNORECASE)
|
|
135
|
+
_MAIL_SECRET_FIELDS = frozenset({"password", "access_token", "refresh_token", "id_token", "auth_token",
|
|
136
|
+
"token", "otp", "verification_code", "confirmation_code", "recovery_code"})
|
|
137
|
+
_MAIL_CREDENTIAL_LABEL = (r"(?:(?:one[- ]time|verification|confirmation|security|login|sign[- ]in|access|authentication)\s+(?:code|password)"
|
|
138
|
+
r"|(?:temporary\s+)?password|(?:access|refresh|auth)[-_ ]token|OTP)")
|
|
139
|
+
_MAIL_GAP = r"(?:\s|<[^>]*>)*"
|
|
140
|
+
_MAIL_CREDENTIAL_RE = re.compile(
|
|
141
|
+
rf"(\b{_MAIL_CREDENTIAL_LABEL}{_MAIL_GAP}(?:(?:is\b|[:=]){_MAIL_GAP})?[\"']?)"
|
|
142
|
+
r"([A-Za-z0-9][A-Za-z0-9._~+/=-]{2,})", re.IGNORECASE)
|
|
143
|
+
_MAIL_CREDENTIAL_BEFORE_LABEL_RE = re.compile(
|
|
144
|
+
rf"\b([A-Za-z0-9][A-Za-z0-9._~+/=-]{{2,}})(\s+is\s+(?:(?:your|the)\s+)?{_MAIL_CREDENTIAL_LABEL}\b)", re.IGNORECASE)
|
|
145
|
+
_MAIL_BEARER_RE = re.compile(r"\bBearer\s+[A-Za-z0-9._~+/=-]+", re.IGNORECASE)
|
|
146
|
+
_MAIL_SIGNIN_SUBJECT_RE = re.compile(r"\b(?:sign[- ]?in|log[- ]?in)\b", re.IGNORECASE)
|
|
147
|
+
_MAIL_ACCESS_RE = re.compile(r"\b(?:magic[- ]link|password reset|reset (?:your )?password|"
|
|
148
|
+
r"one[- ]time (?:code|password)|(?:verification|confirmation|security|access|authentication|login|sign[- ]in) code|OTP|"
|
|
149
|
+
r"code (?:to|for) (?:sign[- ]?in|log[- ]?in)|"
|
|
150
|
+
r"(?:confirm|verify|activate) (?:your )?(?:email|account))\b", re.IGNORECASE)
|
|
151
|
+
# Access messages often separate the label and value (including in their HTML
|
|
152
|
+
# alternative). Apply these only to message content, never IDs, dates or addresses.
|
|
153
|
+
_MAIL_NUMERIC_CODE_RE = re.compile(r"(?<![\w$€£./:-])(?:\d{3}[ \t\u00a0-]\d{3}|\d{4,10})(?![\w/-])")
|
|
154
|
+
_MAIL_CODE_LINE_RE = re.compile(r"(^|>)([ \t]*)([A-Z0-9]{4,16})([ \t]*)(?=$|<)", re.MULTILINE | re.IGNORECASE)
|
|
155
|
+
_MAIL_CONTENT_FIELDS = frozenset({"subject", "text", "html"})
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def portable_mail_response(value: Any, *, _access_message: bool = False, _content: bool = False) -> Any:
|
|
159
|
+
"""Remove usable access material before it reaches any versionable file."""
|
|
160
|
+
if isinstance(value, dict):
|
|
161
|
+
subject = str(value.get("subject") or "")
|
|
162
|
+
content = str(value.get("text") or "")[:600] + "\n" + re.sub(
|
|
163
|
+
r"<[^>]*>", " ", html.unescape(str(value.get("html") or "")))[:600]
|
|
164
|
+
_access_message = bool(_access_message or _CONFIRM_RE.search(subject)
|
|
165
|
+
or _MAIL_SIGNIN_SUBJECT_RE.search(subject)
|
|
166
|
+
or _MAIL_ACCESS_RE.search(subject + "\n" + content))
|
|
167
|
+
return {key: "[credential]" if key.lower().replace("-", "_") in _MAIL_SECRET_FIELDS and item
|
|
168
|
+
else portable_mail_response(item, _access_message=_access_message,
|
|
169
|
+
_content=key.lower() in _MAIL_CONTENT_FIELDS) for key, item in value.items()}
|
|
170
|
+
if isinstance(value, list):
|
|
171
|
+
return [portable_mail_response(item, _access_message=_access_message, _content=_content) for item in value]
|
|
172
|
+
if not isinstance(value, str):
|
|
173
|
+
return value
|
|
174
|
+
|
|
175
|
+
def link(match: re.Match[str]) -> str:
|
|
176
|
+
raw = match.group(0)
|
|
177
|
+
decoded = raw
|
|
178
|
+
for _ in range(3):
|
|
179
|
+
decoded = urllib.parse.unquote(html.unescape(decoded))
|
|
180
|
+
try:
|
|
181
|
+
parts = urllib.parse.urlsplit(decoded)
|
|
182
|
+
fragment_keys = {key.lower() for key, _ in urllib.parse.parse_qsl(parts.fragment)}
|
|
183
|
+
# Access mail can wrap its credential link in an opaque click URL.
|
|
184
|
+
# Preserve the body, but keep all its usable links in the local original.
|
|
185
|
+
sensitive = (_access_message or FUNNEL._private_mail_url(decoded, [], True) or parts.username or parts.password
|
|
186
|
+
or fragment_keys & (_MAIL_SECRET_FIELDS | {"code", "key", "auth", "signature"})
|
|
187
|
+
or re.search(r"/(?:login|sign-?in|reset-password|password-reset|recover|recovery|unsubscribe)(?:/|$)", parts.path, re.I))
|
|
188
|
+
except ValueError:
|
|
189
|
+
sensitive = True
|
|
190
|
+
return "[access-link]" if sensitive else raw
|
|
191
|
+
|
|
192
|
+
decoded = html.unescape(value) if _content else value
|
|
193
|
+
cleaned = _MAIL_LINK_RE.sub(link, decoded)
|
|
194
|
+
if _access_message and _content:
|
|
195
|
+
cleaned = _MAIL_NUMERIC_CODE_RE.sub("[credential]", cleaned)
|
|
196
|
+
cleaned = _MAIL_CODE_LINE_RE.sub(
|
|
197
|
+
lambda match: match.group(1) + match.group(2) + "[credential]" + match.group(4)
|
|
198
|
+
if any(c.isdigit() for c in match.group(3)) else match.group(0), cleaned)
|
|
199
|
+
cleaned = _MAIL_CREDENTIAL_BEFORE_LABEL_RE.sub(lambda match: "[credential]" + match.group(2), cleaned)
|
|
200
|
+
cleaned = _MAIL_BEARER_RE.sub("Bearer [credential]", _MAIL_CREDENTIAL_RE.sub(
|
|
201
|
+
lambda match: match.group(1) + "[credential]", cleaned))
|
|
202
|
+
return value if cleaned == decoded else cleaned
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
# ── client ──────────────────────────────────────────────────────────────────
|
|
206
|
+
|
|
207
|
+
@dataclass
|
|
208
|
+
class OsClient(CF.OsClient):
|
|
209
|
+
"""The Channel Footprint client plus GET calls to the OS mail routes. The
|
|
210
|
+
mailbox address is cached for the runtime; inbox listings are cached per
|
|
211
|
+
harvest day; message bodies are cached once for good."""
|
|
212
|
+
|
|
213
|
+
max_inbox_calls: int = MAX_INBOX_CALLS_PER_ATTEMPT
|
|
214
|
+
max_message_reads: int = MAX_MESSAGE_READS_PER_ATTEMPT
|
|
215
|
+
inbox_calls: int = 0
|
|
216
|
+
message_reads: int = 0
|
|
217
|
+
shared_dir: Path | None = None
|
|
218
|
+
secret_dir: Path | None = None
|
|
219
|
+
get: Callable[..., dict[str, Any]] | None = None
|
|
220
|
+
inbox_checks: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
221
|
+
last_get_at: str | None = None
|
|
222
|
+
|
|
223
|
+
def _isolate_mail_credentials(self, entry: dict[str, Any]) -> dict[str, Any]:
|
|
224
|
+
original = entry["response"]
|
|
225
|
+
portable = portable_mail_response(original)
|
|
226
|
+
if portable == original:
|
|
227
|
+
return entry
|
|
228
|
+
if isinstance(original, dict) and isinstance(original.get("text"), str):
|
|
229
|
+
portable["mail_content_metrics"] = {"links": len(_LINK_RE.findall(original["text"])),
|
|
230
|
+
"word_count": len(original["text"].split()),
|
|
231
|
+
"unsubscribe_link": bool(_UNSUB_RE.search(original["text"])),
|
|
232
|
+
"confirmation_in_prefix": bool(_CONFIRM_RE.search(original["text"][:600]))}
|
|
233
|
+
# Store the original only inside the existing ignored .local boundary,
|
|
234
|
+
# before even a temporary versionable cache file can contain credentials.
|
|
235
|
+
secret_dir = self.secret_dir or self._shared() / ".local" / "mail-cache"
|
|
236
|
+
if ".local" not in secret_dir.parts:
|
|
237
|
+
raise LifecycleCaptureError("credential-bearing mail requires a .local cache directory")
|
|
238
|
+
for parent in (secret_dir, *secret_dir.parents):
|
|
239
|
+
if parent.is_symlink():
|
|
240
|
+
raise LifecycleCaptureError("credential-bearing mail cache must not use symlinks")
|
|
241
|
+
if parent.name == ".local":
|
|
242
|
+
break
|
|
243
|
+
secret_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
244
|
+
encoded = json.dumps(entry, ensure_ascii=False, sort_keys=True)
|
|
245
|
+
digest = hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
|
246
|
+
with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=secret_dir, suffix=".tmp", delete=False) as handle:
|
|
247
|
+
temporary = Path(handle.name)
|
|
248
|
+
try:
|
|
249
|
+
handle.write(encoded)
|
|
250
|
+
handle.flush()
|
|
251
|
+
os.fsync(handle.fileno())
|
|
252
|
+
os.replace(temporary, secret_dir / f"{digest}.json")
|
|
253
|
+
finally:
|
|
254
|
+
temporary.unlink(missing_ok=True)
|
|
255
|
+
return {**entry, "response": portable, "credentials_isolated": True}
|
|
256
|
+
|
|
257
|
+
def _shared(self) -> Path:
|
|
258
|
+
path = self.shared_dir or self.cache_dir
|
|
259
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
260
|
+
return path
|
|
261
|
+
|
|
262
|
+
def _http_get(self, path: str, params: dict[str, Any] | None = None, timeout: float = 60.0) -> dict[str, Any]:
|
|
263
|
+
query = urllib.parse.urlencode({k: v for k, v in (params or {}).items() if v not in (None, "")})
|
|
264
|
+
url = f"{self.os_url.rstrip('/')}{path}" + (f"?{query}" if query else "")
|
|
265
|
+
req = urllib.request.Request(url, headers={"Authorization": f"Bearer {self.api_key}", "Accept": "application/json"}, method="GET")
|
|
266
|
+
try:
|
|
267
|
+
for _name, _value in _connection_headers(req.full_url).items():
|
|
268
|
+
req.add_unredirected_header(_name, _value)
|
|
269
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
270
|
+
envelope = json.loads(resp.read().decode("utf-8"))
|
|
271
|
+
except urllib.error.HTTPError as exc:
|
|
272
|
+
# Upstream bodies may echo mail access material; status is enough
|
|
273
|
+
# for retry/coverage decisions and is safe for caller output.
|
|
274
|
+
raise LifecycleCaptureError(f"OS {path} failed: HTTP {exc.code}") from exc
|
|
275
|
+
except urllib.error.URLError as exc:
|
|
276
|
+
raise CF.OsUnreachable(f"OS {path} unreachable: {exc.reason}") from exc
|
|
277
|
+
except (OSError, ValueError) as exc:
|
|
278
|
+
raise LifecycleCaptureError(f"OS {path} failed in transit: {type(exc).__name__}: {str(exc)[:200]}") from exc
|
|
279
|
+
data = envelope.get("data")
|
|
280
|
+
if envelope.get("code") not in (None, 200) or not isinstance(data, dict):
|
|
281
|
+
raise LifecycleCaptureError(f"OS {path} returned no valid mail data")
|
|
282
|
+
return data
|
|
283
|
+
|
|
284
|
+
def _cached_get(self, kind: str, path: str, params: dict[str, Any], *, cache_dir: Path, budget_attr: str, budget_max: int, label: str,
|
|
285
|
+
timeout: float) -> tuple[dict[str, Any], str]:
|
|
286
|
+
digest = hashlib.sha256(json.dumps({"kind": kind, "path": path, "params": params}, sort_keys=True).encode("utf-8")).hexdigest()
|
|
287
|
+
cache_file = cache_dir / f"{digest}.json"
|
|
288
|
+
self.touched.add(digest)
|
|
289
|
+
entry = self._cache_read(cache_file)
|
|
290
|
+
if entry is not None:
|
|
291
|
+
self.cache_hits += 1
|
|
292
|
+
self.last_get_at = entry.get("retrieved_at")
|
|
293
|
+
response = entry.get("response")
|
|
294
|
+
if portable_mail_response(response) != response:
|
|
295
|
+
raise LifecycleCaptureError("saved mail cache contains access material; secure the local original and sanitize the cache before resuming")
|
|
296
|
+
return (response if isinstance(response, dict) else {}), entry.get("retrieved_on") or date.today().isoformat()
|
|
297
|
+
if getattr(self, budget_attr) >= budget_max:
|
|
298
|
+
raise BudgetExhausted(f"{label} budget exhausted ({budget_max} per attempt); {self.cache_writes} new answer(s) cached in this attempt; rerun to continue from the cache")
|
|
299
|
+
setattr(self, budget_attr, getattr(self, budget_attr) + 1)
|
|
300
|
+
transport = self.get or self._http_get
|
|
301
|
+
attempts = 0
|
|
302
|
+
while True:
|
|
303
|
+
try:
|
|
304
|
+
response = transport(path, params, timeout)
|
|
305
|
+
break
|
|
306
|
+
except LifecycleCaptureError as exc:
|
|
307
|
+
unreachable = isinstance(exc, CF.OsUnreachable)
|
|
308
|
+
if attempts >= 1 or not (unreachable or re.search(r"HTTP 5\d\d", str(exc))):
|
|
309
|
+
raise
|
|
310
|
+
attempts += 1
|
|
311
|
+
(self.sleep or __import__("time").sleep)(CF.OS_UNREACHABLE_RETRY_DELAY_SECONDS if unreachable else CF.FREE_RETRY_DELAY_SECONDS)
|
|
312
|
+
retrieved_on = date.today().isoformat()
|
|
313
|
+
self.last_get_at = datetime.now().astimezone().isoformat()
|
|
314
|
+
entry = self._isolate_mail_credentials({"kind": kind, "path": path, "params": params, "response": response,
|
|
315
|
+
"retrieved_on": retrieved_on, "retrieved_at": self.last_get_at})
|
|
316
|
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
317
|
+
tmp = cache_file.with_name(cache_file.name + ".tmp")
|
|
318
|
+
tmp.write_text(json.dumps(entry, ensure_ascii=False), encoding="utf-8")
|
|
319
|
+
os.replace(tmp, cache_file)
|
|
320
|
+
self.cache_writes += 1
|
|
321
|
+
return entry["response"], retrieved_on
|
|
322
|
+
|
|
323
|
+
def mail_account(self) -> str:
|
|
324
|
+
response, _ = self._cached_get("mail_account", "/mail/account", {}, cache_dir=self._shared(), budget_attr="inbox_calls",
|
|
325
|
+
budget_max=self.max_inbox_calls, label="mail account", timeout=30.0)
|
|
326
|
+
email = str(response.get("email") or "").strip().lower()
|
|
327
|
+
if "@" not in email:
|
|
328
|
+
raise LifecycleCaptureError("the claw has no mailbox: GET /mail/account returned no address")
|
|
329
|
+
return email
|
|
330
|
+
|
|
331
|
+
def reserve_aliases(self, capture_id: str, domains: list[str], mailbox: str) -> dict[str, str]:
|
|
332
|
+
"""The OS registry atomically reserves neutral aliases across workspaces.
|
|
333
|
+
A retry uses the same durable capture id, including after a lost response."""
|
|
334
|
+
payload = {"capture_id": capture_id, "domains": domains}
|
|
335
|
+
data = self._http_post("/mail/lifecycle-aliases/reserve", payload)
|
|
336
|
+
aliases = data.get("aliases")
|
|
337
|
+
if str(data.get("mailbox") or "").lower() != mailbox or not isinstance(aliases, dict):
|
|
338
|
+
raise LifecycleCaptureError("alias reservation returned a different mailbox or no mapping")
|
|
339
|
+
local, _, host = mailbox.partition("@")
|
|
340
|
+
for domain in domains:
|
|
341
|
+
alias = str(aliases.get(domain) or "").lower()
|
|
342
|
+
if not re.fullmatch(re.escape(local) + r"\+[1-9][0-9]@" + re.escape(host), alias):
|
|
343
|
+
raise LifecycleCaptureError("alias reservation returned an invalid neutral alias")
|
|
344
|
+
if len({aliases[d] for d in domains}) != len(domains):
|
|
345
|
+
raise LifecycleCaptureError("alias reservation reused a number for different competitors")
|
|
346
|
+
return {d: str(aliases[d]).lower() for d in domains}
|
|
347
|
+
|
|
348
|
+
def _http_post(self, path: str, payload: dict[str, Any], timeout: float = 30.0) -> dict[str, Any]:
|
|
349
|
+
req = urllib.request.Request(f"{self.os_url.rstrip('/')}{path}", data=json.dumps(payload).encode(),
|
|
350
|
+
headers={"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"}, method="POST")
|
|
351
|
+
try:
|
|
352
|
+
for _name, _value in _connection_headers(req.full_url).items():
|
|
353
|
+
req.add_unredirected_header(_name, _value)
|
|
354
|
+
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
355
|
+
envelope = json.loads(response.read().decode())
|
|
356
|
+
except urllib.error.HTTPError as exc:
|
|
357
|
+
raise LifecycleCaptureError(f"alias reservation failed: HTTP {exc.code}; the neutral number pool may be exhausted or unavailable") from exc
|
|
358
|
+
except (urllib.error.URLError, OSError, ValueError) as exc:
|
|
359
|
+
raise LifecycleCaptureError(f"alias reservation unavailable: {type(exc).__name__}; rerun with the same capture") from exc
|
|
360
|
+
if envelope.get("code") != 200 or not isinstance(envelope.get("data"), dict):
|
|
361
|
+
raise LifecycleCaptureError("alias reservation returned no mapping; rerun with the same capture")
|
|
362
|
+
return envelope["data"]
|
|
363
|
+
|
|
364
|
+
def inbox(self, alias: str, since: str) -> tuple[list[dict[str, Any]], str]:
|
|
365
|
+
response, retrieved_on = self._cached_get("inbox", "/mail/inbox", {"to": alias, "since": since, "limit": INBOX_PAGE}, cache_dir=self.cache_dir,
|
|
366
|
+
budget_attr="inbox_calls", budget_max=self.max_inbox_calls, label="inbox listing", timeout=90.0)
|
|
367
|
+
messages = response.get("messages")
|
|
368
|
+
if not isinstance(messages, list):
|
|
369
|
+
raise LifecycleCaptureError("OS inbox response has no messages list")
|
|
370
|
+
self.inbox_checks[alias] = {"checked_at": self.last_get_at, "inbox_since": since, "query_succeeded": True}
|
|
371
|
+
return [m for m in messages if isinstance(m, dict) and m.get("uid")], retrieved_on
|
|
372
|
+
|
|
373
|
+
def read_message(self, uid: str) -> dict[str, Any]:
|
|
374
|
+
response, _ = self._cached_get("message", f"/mail/message/{urllib.parse.quote(str(uid))}", {}, cache_dir=self._shared() / "messages",
|
|
375
|
+
budget_attr="message_reads", budget_max=self.max_message_reads, label="message read", timeout=90.0)
|
|
376
|
+
return response
|
|
377
|
+
|
|
378
|
+
def ledger_totals(self) -> dict[str, Any]:
|
|
379
|
+
counts = {"mail_account": 0, "inbox": 0, "message": 0}
|
|
380
|
+
dates: list[str] = []
|
|
381
|
+
for base in (self.cache_dir, self._shared(), self._shared() / "messages"):
|
|
382
|
+
if not base.is_dir():
|
|
383
|
+
continue
|
|
384
|
+
for digest in sorted(self.touched):
|
|
385
|
+
path = base / f"{digest}.json"
|
|
386
|
+
if not path.is_file():
|
|
387
|
+
continue
|
|
388
|
+
try:
|
|
389
|
+
entry = json.loads(path.read_text(encoding="utf-8"))
|
|
390
|
+
except (OSError, json.JSONDecodeError):
|
|
391
|
+
continue
|
|
392
|
+
kind = entry.get("kind")
|
|
393
|
+
if kind in counts:
|
|
394
|
+
counts[kind] += 1
|
|
395
|
+
if entry.get("retrieved_on"):
|
|
396
|
+
dates.append(str(entry["retrieved_on"]))
|
|
397
|
+
total = sum(counts.values())
|
|
398
|
+
return {**counts, "provider_calls": total, "observations": total,
|
|
399
|
+
"first_retrieved_on": min(dates) if dates else date.today().isoformat(),
|
|
400
|
+
"last_retrieved_on": max(dates) if dates else date.today().isoformat()}
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
# ── plan and subscriptions ──────────────────────────────────────────────────
|
|
404
|
+
|
|
405
|
+
def alias_for(mailbox: str, domain: str) -> str:
|
|
406
|
+
"""One alias per registrable domain: `lc-coursera-org`, `lc-mural-co`.
|
|
407
|
+
The suffix keeps two competitors with the same brand stem apart."""
|
|
408
|
+
local, _, host = mailbox.partition("@")
|
|
409
|
+
tag = ALIAS_PREFIX + re.sub(r"[^a-z0-9-]", "", registrable_domain(domain).replace(".", "-"))[:48]
|
|
410
|
+
return f"{local}+{tag}@{host}"
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def funnel_lineage(item: dict[str, Any]) -> dict[str, Any]:
|
|
414
|
+
"""Public attribution; mailbox addresses and submission receipts stay private."""
|
|
415
|
+
return {key: item[key] for key in ("funnel_id", "campaign_refs", "path_refs") if key in item}
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def identity_registry(workspace: Path, mailbox: str, aliases: dict[str, str]) -> dict[str, Any]:
|
|
419
|
+
"""Reserve ordinary 02 aliases without replacing the stable identities from 05."""
|
|
420
|
+
path = workspace / IDENTITY_REL
|
|
421
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
422
|
+
with path.with_name(path.name + ".lock").open("a") as lock:
|
|
423
|
+
fcntl.flock(lock, fcntl.LOCK_EX)
|
|
424
|
+
if path.is_file():
|
|
425
|
+
try:
|
|
426
|
+
registry = json.loads(path.read_text(encoding="utf-8"))
|
|
427
|
+
except (OSError, ValueError) as exc:
|
|
428
|
+
raise LifecycleCaptureError("identity registry is not valid JSON") from exc
|
|
429
|
+
if not isinstance(registry, dict) or registry.get("schema_version") != 1 or registry.get("mailbox") != mailbox or not isinstance(registry.get("identities"), list):
|
|
430
|
+
raise LifecycleCaptureError("identity registry needs schema_version 1 and the current research mailbox")
|
|
431
|
+
else:
|
|
432
|
+
registry = {"schema_version": 1, "mailbox": mailbox, "identities": []}
|
|
433
|
+
identities = registry["identities"]
|
|
434
|
+
if any(not isinstance(item, dict) for item in identities):
|
|
435
|
+
raise LifecycleCaptureError("identity registry identities must be objects")
|
|
436
|
+
if len({item.get("alias") for item in identities}) != len(identities):
|
|
437
|
+
raise LifecycleCaptureError("identity registry aliases must be unique")
|
|
438
|
+
for domain, alias in aliases.items():
|
|
439
|
+
matches = [item for item in identities if item.get("alias") == alias]
|
|
440
|
+
if any(item.get("domain") != domain or item.get("funnel_id") is not None
|
|
441
|
+
or item.get("kind") not in {"lifecycle", "lifecycle_reserved"} for item in matches):
|
|
442
|
+
raise LifecycleCaptureError("ordinary lifecycle alias collides with another research identity")
|
|
443
|
+
# A restored, actually recorded legacy subscription can replace an
|
|
444
|
+
# unused neutral plan alias. Preserve that address's reservation,
|
|
445
|
+
# but keep one current mapping for the domain-scoped receipt hash.
|
|
446
|
+
# Changing a used mapping still invalidates its previous receipt.
|
|
447
|
+
for item in identities:
|
|
448
|
+
if item.get("kind") == "lifecycle" and item.get("domain") == domain and item.get("alias") != alias:
|
|
449
|
+
item.update(kind="lifecycle_reserved", identity_id="lifecycle-reservation:" + domain + ":" +
|
|
450
|
+
hashlib.sha256(item["alias"].encode()).hexdigest()[:20])
|
|
451
|
+
if matches:
|
|
452
|
+
matches[0].update(kind="lifecycle", identity_id=f"lifecycle:{domain}")
|
|
453
|
+
else:
|
|
454
|
+
identities.append({"identity_id": f"lifecycle:{domain}", "domain": domain, "funnel_id": None, "alias": alias, "kind": "lifecycle"})
|
|
455
|
+
_write_atomic(path, json.dumps(registry, indent=2, ensure_ascii=False) + "\n")
|
|
456
|
+
return registry
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def load_funnel_handoff(workspace: Path, mailbox: str, registry: dict[str, Any]) -> dict[str, Any]:
|
|
460
|
+
"""Import only journalled, approved registrations with reserved per-funnel aliases.
|
|
461
|
+
|
|
462
|
+
This accepts several funnels for one competitor. It neither creates accounts
|
|
463
|
+
nor relaxes the form/phone rules of ordinary Lifecycle Capture subscriptions.
|
|
464
|
+
"""
|
|
465
|
+
path = workspace / FUNNEL_HANDOFF_REL
|
|
466
|
+
if not path.is_file():
|
|
467
|
+
return {"subscriptions": [], "events": []}
|
|
468
|
+
try:
|
|
469
|
+
raw = json.loads(path.read_text(encoding="utf-8"))
|
|
470
|
+
except (OSError, ValueError) as exc:
|
|
471
|
+
raise LifecycleCaptureError("funnel lifecycle handoff is not valid JSON") from exc
|
|
472
|
+
if not isinstance(raw, dict) or raw.get("schema_version") != 1 or not isinstance(raw.get("subscriptions"), list) or not isinstance(raw.get("events", []), list):
|
|
473
|
+
raise LifecycleCaptureError("funnel lifecycle handoff needs schema_version 1, subscriptions and events lists")
|
|
474
|
+
subscriptions: list[dict[str, Any]] = []
|
|
475
|
+
seen: set[str] = set()
|
|
476
|
+
for item in raw["subscriptions"]:
|
|
477
|
+
if not isinstance(item, dict):
|
|
478
|
+
raise LifecycleCaptureError("funnel subscription must be an object")
|
|
479
|
+
funnel_id = str(item.get("funnel_id") or "")
|
|
480
|
+
domain = registrable_domain(str(item.get("domain") or ""))
|
|
481
|
+
alias = str(item.get("alias") or "").strip().lower()
|
|
482
|
+
if not funnel_id or not domain or funnel_id in seen:
|
|
483
|
+
raise LifecycleCaptureError("funnel handoff needs one subscription per distinct funnel_id")
|
|
484
|
+
seen.add(funnel_id)
|
|
485
|
+
local, _, host = mailbox.partition("@")
|
|
486
|
+
if not re.fullmatch(re.escape(local) + r"\+\d{2,}@" + re.escape(host), alias):
|
|
487
|
+
raise LifecycleCaptureError("funnel handoff alias must be a numeric plus alias of the research mailbox")
|
|
488
|
+
if not any(identity.get("funnel_id") == funnel_id and identity.get("domain") == domain and identity.get("alias") == alias for identity in registry["identities"]):
|
|
489
|
+
raise LifecycleCaptureError("funnel handoff alias is not reserved for this funnel in the identity registry")
|
|
490
|
+
if not item.get("approval_ref") or not item.get("submission_ref"):
|
|
491
|
+
raise LifecycleCaptureError("funnel handoff requires approval_ref and submission_ref")
|
|
492
|
+
if item.get("kind") not in FUNNEL_KINDS:
|
|
493
|
+
raise LifecycleCaptureError("funnel handoff accepts lead_form, quiz or webinar_registration only")
|
|
494
|
+
try:
|
|
495
|
+
subscribed = date.fromisoformat(str(item.get("subscribed_on") or ""))
|
|
496
|
+
except ValueError as exc:
|
|
497
|
+
raise LifecycleCaptureError("funnel handoff needs a valid subscribed_on date") from exc
|
|
498
|
+
if subscribed > date.today():
|
|
499
|
+
raise LifecycleCaptureError("funnel handoff subscription is in the future")
|
|
500
|
+
entry_url = str(item.get("entry_url") or "")
|
|
501
|
+
if not _URL_RE.fullmatch(entry_url):
|
|
502
|
+
raise LifecycleCaptureError("funnel handoff entry_url needs its approved HTTP(S) form URL")
|
|
503
|
+
for key in ("campaign_refs", "path_refs"):
|
|
504
|
+
if not isinstance(item.get(key), list) or not item[key] or any(not isinstance(ref, str) or not ref for ref in item[key]):
|
|
505
|
+
raise LifecycleCaptureError(f"funnel handoff needs nonempty {key}")
|
|
506
|
+
if item.get("confirmed") is not None and not isinstance(item["confirmed"], bool):
|
|
507
|
+
raise LifecycleCaptureError("funnel handoff confirmed must be boolean or null")
|
|
508
|
+
expectations = item.get("email_expectations", {})
|
|
509
|
+
if not isinstance(expectations, dict) or any(expectations.get(key) is not None and type(expectations[key]) is not bool for key in ("confirmation", "result")):
|
|
510
|
+
raise LifecycleCaptureError("funnel handoff email expectations must be boolean or null")
|
|
511
|
+
if any(expectations.get(key) is not None for key in ("confirmation", "result")) and not str(expectations.get("source_ref") or "").startswith("browser:"):
|
|
512
|
+
raise LifecycleCaptureError("funnel handoff email expectations need a browser observation reference")
|
|
513
|
+
email = _load_script("funnel-email-review.py", "pmr_lifecycle_handoff_email")
|
|
514
|
+
if expectations.get("wait_until") is not None and (email.timestamp(expectations["wait_until"]) is None or
|
|
515
|
+
not any(expectations.get(key) is True for key in ("confirmation", "result")) or
|
|
516
|
+
not str(expectations.get("source_ref") or "").startswith("browser:")):
|
|
517
|
+
raise LifecycleCaptureError("expected email deadline needs an observed timestamp and browser reference")
|
|
518
|
+
if "submission_checks" in item:
|
|
519
|
+
checks = item["submission_checks"]
|
|
520
|
+
if not isinstance(checks, list) or not checks:
|
|
521
|
+
raise LifecycleCaptureError("funnel handoff submission_checks must describe each submitted path")
|
|
522
|
+
for check in checks:
|
|
523
|
+
if not isinstance(check, dict) or not check.get("submission_ref") or email.timestamp(check.get("submitted_at")) is None or check.get("path_id") not in item["path_refs"] or not isinstance(check.get("campaign_refs"), list) or not check["campaign_refs"] or not set(check["campaign_refs"]).issubset(item["campaign_refs"]):
|
|
524
|
+
raise LifecycleCaptureError("funnel handoff submission check is not attributable to its subscription")
|
|
525
|
+
per_submission = check.get("email_expectations", expectations)
|
|
526
|
+
if not isinstance(per_submission, dict) or any(type(per_submission.get(key)) not in (bool, type(None)) for key in ("confirmation", "result")) or (check.get("confirmed") is not None and type(check["confirmed"]) is not bool):
|
|
527
|
+
raise LifecycleCaptureError("funnel handoff submission expectations/confirmation must be observed booleans or unknown")
|
|
528
|
+
if any(per_submission.get(key) is not None for key in ("confirmation", "result")) and not str(per_submission.get("source_ref") or "").startswith("browser:"):
|
|
529
|
+
raise LifecycleCaptureError("funnel handoff submission expectations need their browser source")
|
|
530
|
+
if per_submission.get("wait_until") is not None and (email.timestamp(per_submission["wait_until"]) is None or email.timestamp(per_submission["wait_until"]) < email.timestamp(check["submitted_at"]) or not any(per_submission.get(key) is True for key in ("confirmation", "result"))):
|
|
531
|
+
raise LifecycleCaptureError("funnel handoff submission email deadline is invalid")
|
|
532
|
+
if item.get("unsubscribed_on") is not None:
|
|
533
|
+
try:
|
|
534
|
+
unsubscribed = date.fromisoformat(item["unsubscribed_on"])
|
|
535
|
+
except (ValueError, TypeError) as exc:
|
|
536
|
+
raise LifecycleCaptureError("funnel handoff has an invalid unsubscribed_on") from exc
|
|
537
|
+
if not subscribed <= unsubscribed <= date.today():
|
|
538
|
+
raise LifecycleCaptureError("funnel handoff has an invalid unsubscribed_on")
|
|
539
|
+
subscriptions.append({**item, "domain": domain, "alias": alias, "confirmed": item.get("confirmed"), "unsubscribed_on": item.get("unsubscribed_on"), "notes": item.get("notes")})
|
|
540
|
+
events: list[dict[str, Any]] = []
|
|
541
|
+
for event in raw.get("events", []):
|
|
542
|
+
if not isinstance(event, dict) or not event.get("event_id") or not event.get("occurred_at") or not event.get("source_ref"):
|
|
543
|
+
raise LifecycleCaptureError("funnel handoff events need event_id, occurred_at and source_ref")
|
|
544
|
+
subscription = next((item for item in subscriptions if item["funnel_id"] == event.get("funnel_id")), None)
|
|
545
|
+
if subscription is None or event.get("alias") != subscription["alias"] or event.get("type") not in ("email_confirmation", "email_result"):
|
|
546
|
+
raise LifecycleCaptureError("funnel handoff event is not attributed to a reserved subscription")
|
|
547
|
+
if not isinstance(event.get("campaign_refs"), list) or not set(event["campaign_refs"]).issubset(subscription["campaign_refs"]):
|
|
548
|
+
raise LifecycleCaptureError("funnel handoff event campaign refs do not match its subscription")
|
|
549
|
+
if event.get("path_id") is not None and event["path_id"] not in subscription["path_refs"]:
|
|
550
|
+
raise LifecycleCaptureError("funnel handoff event path does not match its subscription")
|
|
551
|
+
if event["type"] == "email_confirmation" and event.get("verification_status") == "confirmed":
|
|
552
|
+
subscription["confirmed"] = True
|
|
553
|
+
events.append(event)
|
|
554
|
+
return {"subscriptions": subscriptions, "events": events}
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
_RESULT_RE = re.compile(r"\b(?:your (?:quiz |assessment |audit |test )?results?|your (?:personalized|personalised) (?:plan|report)|assessment complete|quiz complete)\b", re.IGNORECASE)
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def capture_funnel_events(client: OsClient, workspace: Path, handoff: dict[str, Any], *, refresh: bool = False) -> dict[str, Any]:
|
|
561
|
+
"""Read same-day confirmation/results without completing section 02.
|
|
562
|
+
|
|
563
|
+
--refresh uses a new listing snapshot, even on the same day; message bodies
|
|
564
|
+
remain cached once. Receiving a confirmation is never treated as clicking it.
|
|
565
|
+
"""
|
|
566
|
+
runtime = workspace / RUNTIME_REL
|
|
567
|
+
path = runtime / FUNNEL_EVENTS_FILENAME
|
|
568
|
+
previous = json.loads(path.read_text()) if path.is_file() else {"events": []}
|
|
569
|
+
events = {str(event["event_id"]): event for event in previous.get("events", [])}
|
|
570
|
+
events.update({str(event["event_id"]): event for event in handoff["events"]})
|
|
571
|
+
day = date.today().isoformat()
|
|
572
|
+
snapshot_path = runtime / "funnel-mail-snapshot.json"
|
|
573
|
+
snapshot_state = json.loads(snapshot_path.read_text()) if snapshot_path.is_file() else {}
|
|
574
|
+
if snapshot_state.get("day") == day and (not snapshot_state.get("completed") or not refresh):
|
|
575
|
+
snapshot = snapshot_state["snapshot"]
|
|
576
|
+
else:
|
|
577
|
+
snapshot = datetime.now(timezone.utc).strftime("%H%M%S%f") if refresh else "initial"
|
|
578
|
+
_write_atomic(snapshot_path, json.dumps({"day": day, "snapshot": snapshot, "completed": False}))
|
|
579
|
+
client.cache_dir = runtime / "cache" / "funnel-mail" / day / snapshot
|
|
580
|
+
coverage: list[dict[str, Any]] = []
|
|
581
|
+
messages, _ = harvest(client, handoff["subscriptions"], date.today(), coverage)
|
|
582
|
+
for message in messages:
|
|
583
|
+
subscription = next(item for item in handoff["subscriptions"] if item["alias"] == message["alias"])
|
|
584
|
+
body = client.read_message(message["uid"])
|
|
585
|
+
kind = "email_confirmation" if message["is_confirmation"] else "email_result" if _RESULT_RE.search(message["subject"] + "\n" + str(body.get("text") or "")[:1000]) else None
|
|
586
|
+
if kind is None:
|
|
587
|
+
continue
|
|
588
|
+
if not message["received_at"]:
|
|
589
|
+
CF._coverage_gap(coverage, "message", [subscription["domain"]], "confirmation/result arrival time was not observed")
|
|
590
|
+
continue
|
|
591
|
+
digest = hashlib.sha256(f"{subscription['funnel_id']}:{message['uid']}:{kind}".encode()).hexdigest()[:20]
|
|
592
|
+
event_id = f"mail-{digest}"
|
|
593
|
+
if event_id in events and events[event_id].get("verification_status") == "confirmed":
|
|
594
|
+
continue
|
|
595
|
+
events[event_id] = {"event_id": event_id, "type": kind, "funnel_id": subscription["funnel_id"],
|
|
596
|
+
"path_id": subscription["path_refs"][0] if len(subscription["path_refs"]) == 1 else None,
|
|
597
|
+
"campaign_refs": subscription["campaign_refs"], "alias": subscription["alias"],
|
|
598
|
+
"occurred_at": message["received_at"], "source_ref": f"mail:{message['uid']}", "message_ref": message["uid"],
|
|
599
|
+
"verification_status": "received", "result": {"subject": message["subject"], "excerpt": _excerpt(str(body.get("text") or ""))} if kind == "email_result" else None}
|
|
600
|
+
pending, checks = [], list(previous.get("checks", []))
|
|
601
|
+
email = _load_script("funnel-email-review.py", "pmr_lifecycle_email_review")
|
|
602
|
+
for subscription in handoff["subscriptions"]:
|
|
603
|
+
query = client.inbox_checks.get(subscription["alias"], {})
|
|
604
|
+
check = {"funnel_id": subscription["funnel_id"], "path_refs": subscription["path_refs"],
|
|
605
|
+
"campaign_refs": subscription["campaign_refs"], "alias": subscription["alias"],
|
|
606
|
+
"checked_at": query.get("checked_at"), "inbox_since": query.get("inbox_since") or
|
|
607
|
+
(date.fromisoformat(subscription["subscribed_on"]) - timedelta(days=1)).isoformat(),
|
|
608
|
+
"query_succeeded": query.get("query_succeeded") is True,
|
|
609
|
+
"source_ref": f"inbox_snapshot:{day}:{snapshot}"}
|
|
610
|
+
if check not in checks:
|
|
611
|
+
checks.append(check)
|
|
612
|
+
review = email.review(subscription, {"checks": checks, "events": list(events.values())}, harvest_after_days=HARVEST_AFTER_DAYS)
|
|
613
|
+
if review["pending"]:
|
|
614
|
+
reason = ("mailbox_check_failed" if query.get("query_succeeded") is not True else
|
|
615
|
+
"mailbox_check_time_unknown" if not query.get("checked_at") else review["detail"])
|
|
616
|
+
pending.append({"funnel_id": subscription["funnel_id"], "path_refs": subscription["path_refs"], "alias": subscription["alias"], "reason": reason})
|
|
617
|
+
if review["gaps"]:
|
|
618
|
+
coverage.append({"type": "coverage", **funnel_lineage(subscription), "status": "email_coverage_gap",
|
|
619
|
+
"detail": review["detail"], "gaps": review["gaps"], "unknown": review["unknown"], "terminal": not review["pending"], "tier": "C"})
|
|
620
|
+
result = {"schema_version": 1, "events": sorted(events.values(), key=lambda event: str(event.get("occurred_at") or "")),
|
|
621
|
+
"checks": checks, "pending": pending, "coverage": coverage, "checked_at": datetime.now(timezone.utc).isoformat()}
|
|
622
|
+
_write_atomic(path, json.dumps(result, indent=2, ensure_ascii=False) + "\n")
|
|
623
|
+
_write_atomic(snapshot_path, json.dumps({"day": day, "snapshot": snapshot, "completed": True}))
|
|
624
|
+
return result
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def entry_points(workspace: Path, domains: list[str]) -> dict[str, list[str]]:
|
|
628
|
+
"""Sign-up, lead-form, pricing and home addresses per competitor from a
|
|
629
|
+
completed Funnel Teardown; the home page otherwise."""
|
|
630
|
+
points: dict[str, list[str]] = {d: [f"https://{d}/"] for d in domains}
|
|
631
|
+
evidence = _load_script("research-evidence.py", "pmr_research_evidence_for_lifecycle_entry_points")
|
|
632
|
+
try:
|
|
633
|
+
outcome = evidence.inspect_section(workspace, "funnel_teardown", require_materialized=False)
|
|
634
|
+
except ValueError:
|
|
635
|
+
return points
|
|
636
|
+
if outcome.get("status") != "completed":
|
|
637
|
+
return points
|
|
638
|
+
try:
|
|
639
|
+
document = json.loads((workspace / FUNNEL_TEARDOWN_EVIDENCE_REL).read_text(encoding="utf-8"))
|
|
640
|
+
rows = document.get("rows") or []
|
|
641
|
+
except (OSError, json.JSONDecodeError, AttributeError):
|
|
642
|
+
return points
|
|
643
|
+
preferred = ("signup_page", "lead_form", "dedicated_landing", "pricing_page", "homepage")
|
|
644
|
+
browser_contract = document.get("schema_version", 1) >= 2
|
|
645
|
+
for domain in domains:
|
|
646
|
+
entries = {r.get("address") for r in rows if isinstance(r, dict) and r.get("type") == "funnel_walk"
|
|
647
|
+
and r.get("domain") == domain and r.get("role") != "own"}
|
|
648
|
+
pages = []
|
|
649
|
+
for row in rows:
|
|
650
|
+
if not isinstance(row, dict) or row.get("type") != "landing_page" or row.get("domain") != domain or row.get("role") == "own":
|
|
651
|
+
continue
|
|
652
|
+
address = str(row.get("address") or "")
|
|
653
|
+
if not _URL_RE.fullmatch(address) or registrable_domain(urllib.parse.urlsplit(address).hostname or "") != domain:
|
|
654
|
+
continue
|
|
655
|
+
if browser_contract:
|
|
656
|
+
# V2 publishes browser screens without the legacy fetch_status.
|
|
657
|
+
# A personalized result/confirmation is not a sign-up entry.
|
|
658
|
+
if row.get("source") != "browser" or not (row.get("page_type") in preferred or row.get("lead_fields") or address in entries):
|
|
659
|
+
continue
|
|
660
|
+
elif row.get("fetch_status") != "ok":
|
|
661
|
+
continue
|
|
662
|
+
pages.append(row)
|
|
663
|
+
pages.sort(key=lambda r: (preferred.index(r.get("page_type")) if r.get("page_type") in preferred else len(preferred),
|
|
664
|
+
0 if browser_contract else -(r.get("weight") or 0)))
|
|
665
|
+
chosen = [r["address"] for r in pages[:4]]
|
|
666
|
+
if chosen:
|
|
667
|
+
points[domain] = list(dict.fromkeys(chosen + points[domain]))[:5]
|
|
668
|
+
return points
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def load_subscriptions(path: Path, domains: list[str], aliases: dict[str, str], planned: dict[str, dict[str, Any]] | None = None) -> dict[str, Any] | None:
|
|
672
|
+
"""The agent's subscriptions file, strictly validated. Missing means no
|
|
673
|
+
subscriptions yet. Every alias must be the planned alias of its
|
|
674
|
+
competitor: the capture never uses another address."""
|
|
675
|
+
if not path.is_file():
|
|
676
|
+
return None
|
|
677
|
+
try:
|
|
678
|
+
raw = json.loads(path.read_text(encoding="utf-8"))
|
|
679
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
680
|
+
raise LifecycleCaptureError(f"{path.name} is not valid JSON: {exc}") from exc
|
|
681
|
+
if not isinstance(raw, dict) or raw.get("schema_version") != 1 or not isinstance(raw.get("subscriptions"), list):
|
|
682
|
+
raise LifecycleCaptureError(f"{path.name} must be an object with schema_version 1 and a subscriptions list")
|
|
683
|
+
subscriptions: list[dict[str, Any]] = []
|
|
684
|
+
seen: set[str] = set()
|
|
685
|
+
for index, item in enumerate(raw["subscriptions"], 1):
|
|
686
|
+
label = f"subscription #{index}"
|
|
687
|
+
if not isinstance(item, dict):
|
|
688
|
+
raise LifecycleCaptureError(f"{label} must be an object")
|
|
689
|
+
domain = registrable_domain(str(item.get("domain") or ""))
|
|
690
|
+
if domain not in domains:
|
|
691
|
+
continue # recorded for a competitor outside this run; kept in the file, not published here
|
|
692
|
+
if domain in seen:
|
|
693
|
+
raise LifecycleCaptureError(f"{label} subscribes `{domain}` twice")
|
|
694
|
+
seen.add(domain)
|
|
695
|
+
alias = str(item.get("alias") or "").strip().lower()
|
|
696
|
+
if alias != aliases[domain]:
|
|
697
|
+
raise LifecycleCaptureError(f"{label} must use the planned alias {aliases[domain]} (got {item.get('alias')!r})")
|
|
698
|
+
subscribed_on = str(item.get("subscribed_on") or "")
|
|
699
|
+
if not _DATE_RE.match(subscribed_on):
|
|
700
|
+
raise LifecycleCaptureError(f"{label} needs subscribed_on as YYYY-MM-DD")
|
|
701
|
+
try:
|
|
702
|
+
subscribed = date.fromisoformat(subscribed_on)
|
|
703
|
+
except ValueError as exc:
|
|
704
|
+
raise LifecycleCaptureError(f"{label} has an invalid subscribed_on date") from exc
|
|
705
|
+
if subscribed > date.today():
|
|
706
|
+
raise LifecycleCaptureError(f"{label} is subscribed in the future")
|
|
707
|
+
entry_url = str(item.get("entry_url") or "").strip()
|
|
708
|
+
has_reviewed_entry = planned is not None and any(planned[domain].get(key) for key in ("selected", "fallback"))
|
|
709
|
+
if not _URL_RE.match(entry_url) or (not has_reviewed_entry and registrable_domain(urllib.parse.urlsplit(entry_url).hostname or "") != domain):
|
|
710
|
+
raise LifecycleCaptureError(f"{label} needs entry_url as an https address on {domain}")
|
|
711
|
+
kind = str(item.get("kind") or "").strip().lower()
|
|
712
|
+
if kind not in SUBSCRIPTION_KINDS:
|
|
713
|
+
raise LifecycleCaptureError(f"{label} needs kind in {', '.join(SUBSCRIPTION_KINDS)}")
|
|
714
|
+
confirmed = item.get("confirmed", None)
|
|
715
|
+
if confirmed is not None and not isinstance(confirmed, bool):
|
|
716
|
+
raise LifecycleCaptureError(f"{label} has a non-boolean confirmed")
|
|
717
|
+
unsubscribed_on = item.get("unsubscribed_on")
|
|
718
|
+
if unsubscribed_on is not None:
|
|
719
|
+
if not isinstance(unsubscribed_on, str) or not _DATE_RE.match(unsubscribed_on) or not (subscribed <= date.fromisoformat(unsubscribed_on) <= date.today()):
|
|
720
|
+
raise LifecycleCaptureError(f"{label} has an invalid unsubscribed_on date")
|
|
721
|
+
bound_entry = START.subscription_entry({**item, "domain": domain}, planned[domain]) if planned is not None else {}
|
|
722
|
+
subscriptions.append({"domain": domain, "alias": alias, "subscribed_on": subscribed_on, "entry_url": entry_url, "kind": kind, **bound_entry,
|
|
723
|
+
"confirmed": confirmed, "unsubscribed_on": unsubscribed_on,
|
|
724
|
+
"notes": " ".join(str(item.get("notes") or "").split())[:240] or None})
|
|
725
|
+
if len({s["alias"] for s in subscriptions}) != len(subscriptions):
|
|
726
|
+
raise LifecycleCaptureError("two subscriptions share one alias; every competitor gets its own")
|
|
727
|
+
skipped: list[dict[str, str]] = []
|
|
728
|
+
for index, item in enumerate(raw.get("skipped") or [], 1):
|
|
729
|
+
if not isinstance(item, dict):
|
|
730
|
+
raise LifecycleCaptureError(f"skipped entry #{index} must be an object")
|
|
731
|
+
domain = registrable_domain(str(item.get("domain") or ""))
|
|
732
|
+
reason = " ".join(str(item.get("reason") or "").split())
|
|
733
|
+
if not domain or not reason:
|
|
734
|
+
raise LifecycleCaptureError(f"skipped entry #{index} needs a domain and a reason")
|
|
735
|
+
if domain not in domains:
|
|
736
|
+
continue
|
|
737
|
+
if domain in seen:
|
|
738
|
+
raise LifecycleCaptureError(f"skipped entry #{index} names `{domain}`, which is also subscribed")
|
|
739
|
+
skipped.append({"domain": domain, "reason": reason[:240]})
|
|
740
|
+
return {"subscriptions": subscriptions, "skipped": skipped}
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
# ── harvest ─────────────────────────────────────────────────────────────────
|
|
744
|
+
|
|
745
|
+
def _parse_received(message: dict[str, Any]) -> datetime | None:
|
|
746
|
+
for key in ("received_at", "date"):
|
|
747
|
+
value = message.get(key)
|
|
748
|
+
if not isinstance(value, str) or not value.strip():
|
|
749
|
+
continue
|
|
750
|
+
try:
|
|
751
|
+
if key == "received_at":
|
|
752
|
+
return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)
|
|
753
|
+
return parsedate_to_datetime(value).astimezone(timezone.utc)
|
|
754
|
+
except (ValueError, TypeError):
|
|
755
|
+
continue
|
|
756
|
+
return None
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
def _excerpt(text: str) -> str:
|
|
760
|
+
"""The opening words without any link: confirmation and tracking links
|
|
761
|
+
carry per-recipient tokens and never leave the runtime cache."""
|
|
762
|
+
plain = " ".join(_LINK_RE.sub("[link]", text or "").split())
|
|
763
|
+
return plain[:EXCERPT_CHARS] + ("…" if len(plain) > EXCERPT_CHARS else "")
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
_ADDRESS_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
|
767
|
+
|
|
768
|
+
|
|
769
|
+
PRIVATE_ROW_FIELDS = ("alias", "uid", "message_ref", "source_ref", "approval_ref", "submission_ref")
|
|
770
|
+
_MESSAGE_REF_RE = re.compile(r"/mail/message/[^\s\"')\]]+|\buid\s+[A-Za-z0-9_-]+", re.IGNORECASE)
|
|
771
|
+
_WELCOME_RE = re.compile(r"\bwelcome\b", re.IGNORECASE)
|
|
772
|
+
|
|
773
|
+
|
|
774
|
+
def private_strings(aliases: Any, mailbox: str) -> tuple[str, ...]:
|
|
775
|
+
"""Every spelling of an address that must not be published: the planned
|
|
776
|
+
aliases and the mailbox, plain and percent-encoded (an alias echoed in a
|
|
777
|
+
confirmation link or a server error comes back encoded)."""
|
|
778
|
+
out: list[str] = []
|
|
779
|
+
for value in [*aliases, mailbox]:
|
|
780
|
+
value = str(value or "").strip()
|
|
781
|
+
if not value:
|
|
782
|
+
continue
|
|
783
|
+
out += [value, urllib.parse.quote(value, safe=""), value.replace("+", "%2B").replace("@", "%40"),
|
|
784
|
+
value.replace("@", "%40"), value.replace("+", "%2B")] # half-encoded spellings a server may echo
|
|
785
|
+
return tuple(dict.fromkeys(out))
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
def public_text(value: Any, private: tuple[str, ...]) -> Any:
|
|
789
|
+
"""Published text never carries an address or a message reference."""
|
|
790
|
+
if isinstance(value, str):
|
|
791
|
+
return _MESSAGE_REF_RE.sub("a message", _scrub(value, private))
|
|
792
|
+
if isinstance(value, list):
|
|
793
|
+
return [public_text(v, private) for v in value]
|
|
794
|
+
if isinstance(value, dict):
|
|
795
|
+
return {k: public_text(v, private) for k, v in value.items()}
|
|
796
|
+
return value
|
|
797
|
+
|
|
798
|
+
|
|
799
|
+
def public_row(row: dict[str, Any], private: tuple[str, ...] = ()) -> dict[str, Any]:
|
|
800
|
+
"""The evidence copy of a row: the mailbox alias and the message id stay
|
|
801
|
+
in the runtime folder (plan.json, subscriptions.json, the cache), and
|
|
802
|
+
every published string is scrubbed of addresses and message references."""
|
|
803
|
+
return {k: public_text(v, private) for k, v in row.items() if k not in PRIVATE_ROW_FIELDS}
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def public_entry_url(url: str) -> str:
|
|
807
|
+
"""The entry point without its query and fragment (a confirmation link
|
|
808
|
+
can carry the address)."""
|
|
809
|
+
parts = urllib.parse.urlsplit(url)
|
|
810
|
+
return urllib.parse.urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
|
|
811
|
+
|
|
812
|
+
|
|
813
|
+
def _scrub(value: str, aliases: tuple[str, ...]) -> str:
|
|
814
|
+
"""No mailbox address in published text: the alias, the mailbox or any
|
|
815
|
+
other address a sender echoes back becomes `[address]`."""
|
|
816
|
+
out = value
|
|
817
|
+
for alias in aliases:
|
|
818
|
+
if alias:
|
|
819
|
+
out = re.sub(re.escape(alias), "[address]", out, flags=re.IGNORECASE)
|
|
820
|
+
return _ADDRESS_RE.sub("[address]", out)
|
|
821
|
+
|
|
822
|
+
|
|
823
|
+
def recipient_addresses(item: dict[str, Any]) -> set[str]:
|
|
824
|
+
"""Every address the listing names as a recipient (To and Delivered-To),
|
|
825
|
+
parsed and normalised, so attribution compares whole addresses."""
|
|
826
|
+
found: set[str] = set()
|
|
827
|
+
for field in ("to", "delivered_to"):
|
|
828
|
+
raw = item.get(field)
|
|
829
|
+
values = raw if isinstance(raw, list) else [raw]
|
|
830
|
+
for value in values:
|
|
831
|
+
if not isinstance(value, str) or not value.strip():
|
|
832
|
+
continue
|
|
833
|
+
for _, address in getaddresses([value]):
|
|
834
|
+
address = address.strip().lower()
|
|
835
|
+
if "@" in address:
|
|
836
|
+
found.add(address)
|
|
837
|
+
return found
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
def message_row(listing: dict[str, Any], body: dict[str, Any], subscription: dict[str, Any], index: int) -> dict[str, Any]:
|
|
841
|
+
text = str(body.get("text") or "")
|
|
842
|
+
aliases = (subscription["alias"], subscription["alias"].partition("+")[0] + "@" + subscription["alias"].partition("@")[2])
|
|
843
|
+
subject = _scrub(" ".join(str(listing.get("subject") or body.get("subject") or "").split()), aliases)
|
|
844
|
+
sender_name, sender_address = parseaddr(str(listing.get("from") or body.get("from") or ""))
|
|
845
|
+
received = _parse_received(listing) or _parse_received(body)
|
|
846
|
+
subscribed = date.fromisoformat(subscription["subscribed_on"])
|
|
847
|
+
# The inbox is asked from the day before the subscription (server days are
|
|
848
|
+
# UTC, the recorded date is local); a message from that margin is day 0.
|
|
849
|
+
day_offset = max(0, (received.date() - subscribed).days) if received else None
|
|
850
|
+
haystack = f"{subject}\n{text}"
|
|
851
|
+
discounts = [int(a or b) for a, b in _DISCOUNT_RE.findall(haystack) if (a or b)]
|
|
852
|
+
prices = list(dict.fromkeys(m.group(0).strip() for m in _PRICE_RE.finditer(haystack)))[:4]
|
|
853
|
+
deadline = _DEADLINE_RE.search(haystack)
|
|
854
|
+
is_confirmation = bool(_CONFIRM_RE.search(subject)) or (index == 1 and
|
|
855
|
+
body.get("mail_content_metrics", {}).get("confirmation_in_prefix", bool(_CONFIRM_RE.search(text[:600]))))
|
|
856
|
+
is_offer = bool(discounts or prices or _FREE_RE.search(haystack))
|
|
857
|
+
kind = "confirmation" if is_confirmation else "offer" if is_offer else "nudge" if _PROGRESS_RE.search(haystack) else "welcome" if index <= 2 else "content"
|
|
858
|
+
is_welcome = kind == "welcome" or (not is_confirmation and bool(_WELCOME_RE.search(subject)))
|
|
859
|
+
return {
|
|
860
|
+
**funnel_lineage(subscription),
|
|
861
|
+
"type": "lifecycle_message", "domain": subscription["domain"], "alias": subscription["alias"], "uid": str(listing.get("uid")),
|
|
862
|
+
"received_at": received.isoformat().replace("+00:00", "Z") if received else None, "day_offset": day_offset,
|
|
863
|
+
"hour_utc": received.hour if received else None, "sender_domain": registrable_domain(sender_address.partition("@")[2]) if "@" in sender_address else None,
|
|
864
|
+
"subject": subject[:200], "kind": kind, "is_confirmation": is_confirmation, "is_offer": is_offer, "is_welcome": is_welcome,
|
|
865
|
+
"discount_percent": max(discounts) if discounts else None, "prices": prices, "free_mention": bool(_FREE_RE.search(haystack)),
|
|
866
|
+
"deadline": deadline.group(0).lower() if deadline else None,
|
|
867
|
+
"unsubscribe_link": body.get("mail_content_metrics", {}).get("unsubscribe_link", bool(_UNSUB_RE.search(text))),
|
|
868
|
+
"links": body.get("mail_content_metrics", {}).get("links", len(_LINK_RE.findall(text))),
|
|
869
|
+
"word_count": body.get("mail_content_metrics", {}).get("word_count", len(text.split())), "tier": "A",
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
|
|
873
|
+
def sequence_end(subscription: dict[str, Any], harvested_on: date) -> date:
|
|
874
|
+
"""The last day a sequence is observed: the harvest cutoff, or the
|
|
875
|
+
unsubscribe date when it came first."""
|
|
876
|
+
if subscription.get("unsubscribed_on"):
|
|
877
|
+
return min(harvested_on, date.fromisoformat(subscription["unsubscribed_on"]))
|
|
878
|
+
return harvested_on
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
def sequence_row(subscription: dict[str, Any], messages: list[dict[str, Any]], harvested_on: date, truncated: bool = False) -> dict[str, Any]:
|
|
882
|
+
subscribed = date.fromisoformat(subscription["subscribed_on"])
|
|
883
|
+
end = sequence_end(subscription, harvested_on)
|
|
884
|
+
days_covered = max(0, (end - subscribed).days)
|
|
885
|
+
offsets = sorted(m["day_offset"] for m in messages if isinstance(m.get("day_offset"), int))
|
|
886
|
+
offers = [m for m in messages if m["is_offer"]]
|
|
887
|
+
gaps = [b - a for a, b in zip(offsets, offsets[1:])]
|
|
888
|
+
kinds = Counter(m["kind"] for m in messages)
|
|
889
|
+
return {
|
|
890
|
+
**funnel_lineage(subscription),
|
|
891
|
+
"type": "lifecycle_sequence", "domain": subscription["domain"], "alias": subscription["alias"], "kind": subscription["kind"],
|
|
892
|
+
"subscribed_on": subscription["subscribed_on"], "harvested_on": harvested_on.isoformat(), "days_covered": days_covered,
|
|
893
|
+
"observation_status": "full" if days_covered >= HARVEST_AFTER_DAYS else "partial", "target_days": HARVEST_AFTER_DAYS,
|
|
894
|
+
"observed_from": subscribed.isoformat(), "observed_through": end.isoformat(),
|
|
895
|
+
"early_harvest_due_on": (subscribed + timedelta(days=EARLY_HARVEST_AFTER_DAYS)).isoformat(),
|
|
896
|
+
"full_harvest_due_on": (subscribed + timedelta(days=HARVEST_AFTER_DAYS)).isoformat(),
|
|
897
|
+
"coverage_complete": not truncated and subscription.get("confirmed") is not False,
|
|
898
|
+
"listing_succeeded": True,
|
|
899
|
+
"messages": len(messages), "messages_per_week": round(len(messages) * 7 / days_covered, 2) if days_covered >= 7 else None,
|
|
900
|
+
"first_message_day": offsets[0] if offsets else None, "last_message_day": offsets[-1] if offsets else None,
|
|
901
|
+
"first_welcome_day": min((m["day_offset"] for m in messages if m.get("is_welcome") and isinstance(m.get("day_offset"), int)), default=None),
|
|
902
|
+
"median_gap_days": sorted(gaps)[len(gaps) // 2] if gaps else None, "confirmations": kinds.get("confirmation", 0),
|
|
903
|
+
"offers": len(offers), "first_offer_day": min((m["day_offset"] for m in offers if isinstance(m.get("day_offset"), int)), default=None),
|
|
904
|
+
"max_discount_percent": max((m["discount_percent"] for m in messages if m["discount_percent"]), default=None),
|
|
905
|
+
"deadlines": sum(1 for m in messages if m["deadline"]), "nudges": kinds.get("nudge", 0),
|
|
906
|
+
"prices": list(dict.fromkeys(p for m in messages for p in m["prices"]))[:6],
|
|
907
|
+
"sender_domains": list(dict.fromkeys(m["sender_domain"] for m in messages if m["sender_domain"]))[:4],
|
|
908
|
+
"unsubscribe_present": any(m["unsubscribe_link"] for m in messages), "subjects": [m["subject"] for m in messages[:8]],
|
|
909
|
+
"listing_truncated": truncated, "tier": "A" if messages else "C",
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
_PATTERN_LABELS = {
|
|
914
|
+
"double_opt_in": "A confirmation step before the sequence starts",
|
|
915
|
+
"welcome_same_day": "A welcome message within a day",
|
|
916
|
+
"offer_in_first_week": "A paid offer within the first seven days",
|
|
917
|
+
"discount_offered": "A percentage discount is offered",
|
|
918
|
+
"deadline_used": "A deadline or scarcity line is used",
|
|
919
|
+
"nudges_sent": "Progress or come-back nudges are sent",
|
|
920
|
+
"daily_or_more": "Five or more messages a week",
|
|
921
|
+
"weekly_or_less": "At most one message a week",
|
|
922
|
+
"prices_named": "Prices are named inside the emails",
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
|
|
926
|
+
def lifecycle_patterns(sequences: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
927
|
+
active = [s for s in sequences if s["messages"]]
|
|
928
|
+
if not active:
|
|
929
|
+
return []
|
|
930
|
+
hits: dict[str, list[str]] = {k: [] for k in _PATTERN_LABELS}
|
|
931
|
+
for s in active:
|
|
932
|
+
d = s["domain"]
|
|
933
|
+
if s["confirmations"]:
|
|
934
|
+
hits["double_opt_in"].append(d)
|
|
935
|
+
if s.get("first_welcome_day") is not None and s["first_welcome_day"] <= 1:
|
|
936
|
+
hits["welcome_same_day"].append(d)
|
|
937
|
+
if s["first_offer_day"] is not None and s["first_offer_day"] <= 7:
|
|
938
|
+
hits["offer_in_first_week"].append(d)
|
|
939
|
+
if s["max_discount_percent"]:
|
|
940
|
+
hits["discount_offered"].append(d)
|
|
941
|
+
if s["deadlines"]:
|
|
942
|
+
hits["deadline_used"].append(d)
|
|
943
|
+
if s["nudges"]:
|
|
944
|
+
hits["nudges_sent"].append(d)
|
|
945
|
+
if s["days_covered"] >= 7 and (s["messages_per_week"] or 0) >= 5:
|
|
946
|
+
hits["daily_or_more"].append(d)
|
|
947
|
+
elif s["days_covered"] >= 7 and (s["messages_per_week"] or 0) <= 1:
|
|
948
|
+
hits["weekly_or_less"].append(d)
|
|
949
|
+
if s["prices"]:
|
|
950
|
+
hits["prices_named"].append(d)
|
|
951
|
+
rows = []
|
|
952
|
+
for pattern, domains in hits.items():
|
|
953
|
+
if not domains:
|
|
954
|
+
continue
|
|
955
|
+
supporting = [s for s in active if s["domain"] in domains]
|
|
956
|
+
windows = [s for s in supporting if s.get("observed_from") and s.get("observed_through")]
|
|
957
|
+
rows.append({"type": "lifecycle_pattern", "pattern": pattern, "label": _PATTERN_LABELS[pattern], "competitors": sorted(set(domains)),
|
|
958
|
+
"competitors_with": len(set(domains)), "competitors_total": len({s["domain"] for s in active}), "share": round(len(set(domains)) / len({s["domain"] for s in active}), 2),
|
|
959
|
+
"recurring": len(set(domains)) >= 2, "eligible": True,
|
|
960
|
+
"observation_status": "full" if all(s["days_covered"] >= HARVEST_AFTER_DAYS for s in supporting) else "partial",
|
|
961
|
+
"target_days": HARVEST_AFTER_DAYS, "days_covered": min(s["days_covered"] for s in supporting),
|
|
962
|
+
"observed_from": min((s["observed_from"] for s in windows), default=None),
|
|
963
|
+
"observed_through": max((s["observed_through"] for s in windows), default=None),
|
|
964
|
+
"coverage_complete": all(s.get("coverage_complete", True) for s in supporting),
|
|
965
|
+
"tier": "B" if len(set(domains)) >= 2 else "C"})
|
|
966
|
+
rows.sort(key=lambda r: (-r["competitors_with"], r["pattern"]))
|
|
967
|
+
return rows
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def harvest(client: OsClient, subscriptions: list[dict[str, Any]], harvested_on: date, coverage: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
971
|
+
"""Every recorded subscription is harvested, unsubscribed ones included:
|
|
972
|
+
their mail is still in the mailbox and the sequence they produced is the
|
|
973
|
+
evidence. Messages after the harvest day or after the unsubscribe date are
|
|
974
|
+
left out."""
|
|
975
|
+
messages_out: list[dict[str, Any]] = []
|
|
976
|
+
sequences: list[dict[str, Any]] = []
|
|
977
|
+
for subscription in subscriptions:
|
|
978
|
+
if date.fromisoformat(subscription["subscribed_on"]) > harvested_on:
|
|
979
|
+
coverage.append({"type": "coverage", "domain": subscription["domain"], "source": "inbox", "status": "pending",
|
|
980
|
+
"detail": "subscription had not started at this historical cutoff", "tier": "C"})
|
|
981
|
+
continue
|
|
982
|
+
since = (date.fromisoformat(subscription["subscribed_on"]) - timedelta(days=1)).isoformat()
|
|
983
|
+
try:
|
|
984
|
+
listing, _ = client.inbox(subscription["alias"], since)
|
|
985
|
+
except BudgetExhausted:
|
|
986
|
+
raise
|
|
987
|
+
except LifecycleCaptureError as exc:
|
|
988
|
+
if isinstance(exc, CF.OsUnreachable) or re.search(r"HTTP (?:401|402|403|429)\b", str(exc)):
|
|
989
|
+
raise # the key, the mailbox or the quota is refused: not a fact about this competitor
|
|
990
|
+
CF._coverage_gap(coverage, "inbox", [subscription["domain"]], CF._provider_detail(exc))
|
|
991
|
+
failed_sequence = sequence_row(subscription, [], harvested_on)
|
|
992
|
+
failed_sequence["coverage_complete"] = False
|
|
993
|
+
failed_sequence["listing_succeeded"] = False
|
|
994
|
+
sequences.append(failed_sequence)
|
|
995
|
+
continue
|
|
996
|
+
alias = subscription["alias"].lower()
|
|
997
|
+
end = sequence_end(subscription, harvested_on)
|
|
998
|
+
kept = []
|
|
999
|
+
for item in listing:
|
|
1000
|
+
if alias not in recipient_addresses(item):
|
|
1001
|
+
continue # the server filter is a substring match; keep exact alias hits only
|
|
1002
|
+
received = _parse_received(item)
|
|
1003
|
+
if received is not None and received.date() > end:
|
|
1004
|
+
continue
|
|
1005
|
+
kept.append(item)
|
|
1006
|
+
truncated = len(listing) >= INBOX_PAGE
|
|
1007
|
+
if truncated:
|
|
1008
|
+
CF._coverage_gap(coverage, "inbox", [subscription["domain"]],
|
|
1009
|
+
f"the inbox listing returned the newest {INBOX_PAGE} messages since {since}; earlier messages of this sequence may be missing")
|
|
1010
|
+
rows: list[dict[str, Any]] = []
|
|
1011
|
+
ordered = sorted(kept, key=lambda m: (str(m.get("received_at") or ""), str(m.get("uid"))))
|
|
1012
|
+
for index, item in enumerate(ordered, 1):
|
|
1013
|
+
try:
|
|
1014
|
+
body = client.read_message(str(item["uid"]))
|
|
1015
|
+
except BudgetExhausted:
|
|
1016
|
+
raise
|
|
1017
|
+
except LifecycleCaptureError as exc:
|
|
1018
|
+
if not re.search(r"HTTP (?:404|410)\b", str(exc)):
|
|
1019
|
+
raise # only a message that vanished between the listing and the read is a fact about this sequence
|
|
1020
|
+
status_match = re.search(r"HTTP (\d{3})", str(exc))
|
|
1021
|
+
CF._coverage_gap(coverage, "message", [subscription["domain"]], f"a message of this sequence could not be read: HTTP {status_match.group(1) if status_match else 'error'}")
|
|
1022
|
+
continue
|
|
1023
|
+
rows.append(message_row(item, body, subscription, index))
|
|
1024
|
+
messages_out.extend(rows)
|
|
1025
|
+
sequence = sequence_row(subscription, rows, harvested_on, truncated)
|
|
1026
|
+
if any(g.get("domain") == subscription["domain"] and g.get("status") != "ok" for g in coverage):
|
|
1027
|
+
sequence["coverage_complete"] = False
|
|
1028
|
+
sequences.append(sequence)
|
|
1029
|
+
return messages_out, sequences
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
# ── rendering ───────────────────────────────────────────────────────────────
|
|
1033
|
+
|
|
1034
|
+
def _cell(value: Any, cap: int = 90) -> str:
|
|
1035
|
+
return CF._cell(value, cap)
|
|
1036
|
+
|
|
1037
|
+
|
|
1038
|
+
def render_section(*, domains: list[str], mailbox_domain: str, subscriptions: list[dict[str, Any]], skipped: list[dict[str, str]],
|
|
1039
|
+
sequences: list[dict[str, Any]], messages: list[dict[str, Any]], patterns: list[dict[str, Any]], coverage: list[dict[str, Any]],
|
|
1040
|
+
phase: str, harvested_on: str | None, totals: dict[str, Any]) -> str:
|
|
1041
|
+
lines: list[str] = []
|
|
1042
|
+
add = lines.append
|
|
1043
|
+
if phase == "started":
|
|
1044
|
+
add(f"Research subscriptions started at {len({s['domain'] for s in subscriptions})} of {len(domains)} competitor(s) with separate plus-addressed aliases on the claw's `@{mailbox_domain}` mailbox; "
|
|
1045
|
+
f"the early harvest reads the inbox after {EARLY_HARVEST_AFTER_DAYS} days and the full window after {HARVEST_AFTER_DAYS} days from each actual sign-up.")
|
|
1046
|
+
else:
|
|
1047
|
+
add(f"Competitor email sequences harvested on {harvested_on}: {len(messages)} message(s) from {sum(1 for s in sequences if s['messages'])} of {len(subscriptions)} subscription(s), "
|
|
1048
|
+
f"attributed by the exact subscription alias and, where present, its funnel and campaign references.")
|
|
1049
|
+
fully_observed = sequences and len(sequences) == len(subscriptions) and all(s["observation_status"] == "full" for s in sequences)
|
|
1050
|
+
pending_signups = any(g.get("source") == "subscription" and g.get("status") == "pending" for g in coverage)
|
|
1051
|
+
add("Observation: " + ("full" if fully_observed and not pending_signups else "partial") +
|
|
1052
|
+
"; conclusions apply only to the stated observation windows. Full duration does not imply complete inbox coverage.")
|
|
1053
|
+
add("")
|
|
1054
|
+
add("### Subscriptions")
|
|
1055
|
+
add("")
|
|
1056
|
+
add("| Competitor | Kind | Entry point | Subscribed on | Confirmed | Status | Notes |")
|
|
1057
|
+
add("| --- | --- | --- | --- | --- | --- | --- |")
|
|
1058
|
+
for s in subscriptions:
|
|
1059
|
+
status = f"unsubscribed {s['unsubscribed_on']}" if s.get("unsubscribed_on") else "active"
|
|
1060
|
+
confirmed = "yes" if s.get("confirmed") else "no" if s.get("confirmed") is False else "not observed"
|
|
1061
|
+
add(f"| {s['domain']} | {s['kind'].replace('_', ' ')} | {_cell(s['entry_url'].replace('https://', ''), 60)} | {s['subscribed_on']} | {confirmed} | {status} | {_cell(s.get('notes') or '—', 80)} |")
|
|
1062
|
+
for s in skipped:
|
|
1063
|
+
add(f"| {s['domain']} | — | — | — | — | skipped | {_cell(s['reason'], 80)} |")
|
|
1064
|
+
add("")
|
|
1065
|
+
if phase == "harvested":
|
|
1066
|
+
add("### Sequences by competitor")
|
|
1067
|
+
add("")
|
|
1068
|
+
add("| Competitor | Observation | Window | Coverage | Days covered | Messages | Per week | First message (day) | First offer (day) | Max discount | Deadlines | Nudges | Prices named |")
|
|
1069
|
+
add("| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |")
|
|
1070
|
+
for s in sequences:
|
|
1071
|
+
add(f"| {s['domain']} | {s['observation_status']} | {s['observed_from']}..{s['observed_through']} | {'complete' if s['coverage_complete'] else 'gaps'} | {s['days_covered']} | {s['messages']} | {s['messages_per_week'] if s['messages_per_week'] is not None else '—'} | "
|
|
1072
|
+
f"{s['first_message_day'] if s['first_message_day'] is not None else '—'} | {s['first_offer_day'] if s['first_offer_day'] is not None else '—'} | "
|
|
1073
|
+
f"{str(s['max_discount_percent']) + '%' if s['max_discount_percent'] else '—'} | {s['deadlines']} | {s['nudges']} | {_cell(', '.join(s['prices']) or '—', 60)} |")
|
|
1074
|
+
add("")
|
|
1075
|
+
for s in sequences:
|
|
1076
|
+
rows = [m for m in messages if m["domain"] == s["domain"] and m.get("funnel_id") == s.get("funnel_id")]
|
|
1077
|
+
if not rows:
|
|
1078
|
+
continue
|
|
1079
|
+
add(f"#### {s['domain']}" + (f" — {s['funnel_id']}" if s.get("funnel_id") else ""))
|
|
1080
|
+
add("")
|
|
1081
|
+
add("| Day | Kind | Subject | Offer or deadline |")
|
|
1082
|
+
add("| ---: | --- | --- | --- |")
|
|
1083
|
+
for m in rows:
|
|
1084
|
+
offer = ", ".join(x for x in ([f"{m['discount_percent']}% off"] if m["discount_percent"] else []) + m["prices"][:2] + ([m["deadline"]] if m["deadline"] else []))
|
|
1085
|
+
add(f"| {m['day_offset'] if m['day_offset'] is not None else '—'} | {m['kind']} | {_cell(m['subject'] or '(no subject)', 90)} | {_cell(offer or '—', 60)} |")
|
|
1086
|
+
add("")
|
|
1087
|
+
add("### Recurring lifecycle patterns")
|
|
1088
|
+
add("")
|
|
1089
|
+
if patterns:
|
|
1090
|
+
add("| Pattern | Competitors | Observation | Share |")
|
|
1091
|
+
add("| --- | --- | --- | ---: |")
|
|
1092
|
+
for r in patterns:
|
|
1093
|
+
add(f"| {r['label']} | {', '.join(r['competitors'])} ({r['competitors_with']} of {r['competitors_total']}) | {r['observation_status']}, {r['days_covered']} days | {int(r['share'] * 100)}% |")
|
|
1094
|
+
else:
|
|
1095
|
+
add("No message has arrived yet; patterns need at least one harvested sequence.")
|
|
1096
|
+
add("")
|
|
1097
|
+
add("### Method and policy")
|
|
1098
|
+
add("")
|
|
1099
|
+
add("- One neutral plus-addressed alias per competitor per capture, reserved uniquely for the mailbox; every message is attributed by an exact To or Delivered-To address match; "
|
|
1100
|
+
"stable per-funnel aliases imported from section 05 retain their attribution; the inbox is read through OS by alias and start date, each message body once. Bodies stay in the runtime cache; the section carries subjects, "
|
|
1101
|
+
f"and extracted facts. Reads in this cutoff: {totals.get('inbox', 0)} inbox listing(s), {totals.get('message', 0)} message(s).")
|
|
1102
|
+
add("- Research sign-ups only: newsletters, free accounts, trials and waitlists that need no payment; no card details, no CAPTCHA bypass, no real "
|
|
1103
|
+
"third-party identities; no required phone. A human with verified access to the planned aliases and mailbox signs up, confirms and unsubscribes; never engage sales representatives. Replicate observed offer structure; cadence needs at least a week of observation. Never copy competitor wording or claims.")
|
|
1104
|
+
gaps = [g for g in coverage if g.get("status") != "ok"]
|
|
1105
|
+
if gaps:
|
|
1106
|
+
add("- Not read in this run: " + "; ".join(f"{g['domain']} ({g['detail']})" for g in gaps[:10]) + ".")
|
|
1107
|
+
add("")
|
|
1108
|
+
add("### Sources")
|
|
1109
|
+
add("")
|
|
1110
|
+
add("- The claw's mailbox through OS `GET /mail/inbox` (filters `to` and `since`) and `GET /mail/message/{uid}`; subscriptions recorded by the executing agent.")
|
|
1111
|
+
return "\n".join(lines).strip() + "\n"
|
|
1112
|
+
|
|
1113
|
+
|
|
1114
|
+
# ── orchestration ───────────────────────────────────────────────────────────
|
|
1115
|
+
|
|
1116
|
+
def _arg(args: Any, name: str, default: Any) -> Any:
|
|
1117
|
+
value = getattr(args, name, None)
|
|
1118
|
+
return default if value is None else value
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
def _write_atomic(path: Path, content: str) -> None:
|
|
1122
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
1123
|
+
tmp = path.with_name(path.name + ".tmp")
|
|
1124
|
+
tmp.write_text(content, encoding="utf-8")
|
|
1125
|
+
os.replace(tmp, path)
|
|
1126
|
+
|
|
1127
|
+
|
|
1128
|
+
def resolve_harvest_day(runtime: Path, override: str, refresh: bool, not_before: date | None = None) -> date:
|
|
1129
|
+
"""A harvest is a day's snapshot of the inbox: reruns on the same cutoff
|
|
1130
|
+
replay the cache; `--refresh` reads the inbox again once the current
|
|
1131
|
+
harvest completed."""
|
|
1132
|
+
marker = runtime / "harvest-as-of.json"
|
|
1133
|
+
try:
|
|
1134
|
+
state = json.loads(marker.read_text(encoding="utf-8"))
|
|
1135
|
+
current = date.fromisoformat(state["as_of"])
|
|
1136
|
+
completed = bool(state.get("completed"))
|
|
1137
|
+
except (OSError, ValueError, KeyError, json.JSONDecodeError):
|
|
1138
|
+
current, completed = None, False
|
|
1139
|
+
if override:
|
|
1140
|
+
if not _DATE_RE.match(override):
|
|
1141
|
+
raise LifecycleCaptureError("--as-of must be YYYY-MM-DD")
|
|
1142
|
+
chosen = date.fromisoformat(override)
|
|
1143
|
+
if chosen > date.today():
|
|
1144
|
+
raise LifecycleCaptureError("--as-of cannot be in the future")
|
|
1145
|
+
if not_before is not None and chosen < not_before:
|
|
1146
|
+
raise LifecycleCaptureError(f"--as-of cannot be before the first subscription day ({not_before.isoformat()})")
|
|
1147
|
+
elif current is None or (completed and (refresh or current < date.today())):
|
|
1148
|
+
# A harvest is a day's snapshot: a completed one is replayed only on
|
|
1149
|
+
# its own day; the next day's run reads the inbox again.
|
|
1150
|
+
chosen = date.today()
|
|
1151
|
+
else:
|
|
1152
|
+
chosen = current
|
|
1153
|
+
if chosen != current:
|
|
1154
|
+
completed = False
|
|
1155
|
+
_write_atomic(marker, json.dumps({"as_of": chosen.isoformat(), "completed": completed}))
|
|
1156
|
+
return chosen
|
|
1157
|
+
|
|
1158
|
+
|
|
1159
|
+
def mark_harvest_completed(runtime: Path, as_of: date) -> None:
|
|
1160
|
+
_write_atomic(runtime / "harvest-as-of.json", json.dumps({"as_of": as_of.isoformat(), "completed": True}))
|
|
1161
|
+
|
|
1162
|
+
|
|
1163
|
+
def _plan(mailbox: str, domains: list[str], aliases: dict[str, str], points: dict[str, list[str]], subscriptions_path: Path,
|
|
1164
|
+
*, capture_id: str, reviewed: dict[str, dict[str, Any]], previous: dict[str, Any], product_domain: str) -> dict[str, Any]:
|
|
1165
|
+
old = {c["domain"]: c for c in previous.get("competitors", []) if isinstance(c, dict) and "domain" in c}
|
|
1166
|
+
competitors = []
|
|
1167
|
+
for d in domains:
|
|
1168
|
+
entry = reviewed.get(d) or ({k: old[d].get(k) for k in ("selected", "fallback", "skip_reason", "discovery_status", "legacy")} if d in old else {})
|
|
1169
|
+
urls = [v["entry_url"] for v in (entry.get("selected"), entry.get("fallback")) if v]
|
|
1170
|
+
competitors.append({"domain": d, "alias": aliases[d], "entry_points": urls or points.get(d) or [f"https://{d}/"],
|
|
1171
|
+
"discovery_status": "pending", **entry})
|
|
1172
|
+
return {
|
|
1173
|
+
"schema_version": SCHEMA_VERSION, "section_id": SECTION_ID, "mailbox": mailbox, "subscriptions_path": subscriptions_path.as_posix(),
|
|
1174
|
+
"capture_id": capture_id, "product_domain": product_domain, "kinds": list(SUBSCRIPTION_KINDS),
|
|
1175
|
+
"early_harvest_after_days": EARLY_HARVEST_AFTER_DAYS, "harvest_after_days": HARVEST_AFTER_DAYS,
|
|
1176
|
+
"competitors": competitors,
|
|
1177
|
+
"rules": [
|
|
1178
|
+
"Assign signups to any authorized human with verified access to the planned aliases and mailbox, regardless of role. Use the planned alias only at the reviewed selected entry or fallback; no payment is allowed. If nobody has that access, leave actions pending.",
|
|
1179
|
+
"Never enter a required phone, card or payment details, never bypass a CAPTCHA or bot check, never use a real third-party identity; try a verified fallback or record a skip reason.",
|
|
1180
|
+
"The assigned human confirms double opt-in from the inbox and reports the result; the agent records confirmed: true. Never reply to sales representatives.",
|
|
1181
|
+
f"Record one entry per competitor in {subscriptions_path.name}: domain, alias, subscribed_on, entry_url, kind, confirmed, notes; competitors you could not subscribe go to `skipped` with a reason.",
|
|
1182
|
+
"After the full harvest the assigned human unsubscribes through the link in the emails and reports the actual date; record unsubscribed_on.",
|
|
1183
|
+
],
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
|
|
1187
|
+
def subscription_scope(item: dict[str, Any]) -> str:
|
|
1188
|
+
return item["domain"] + (":funnel:" + item["funnel_id"] if item.get("funnel_id") else "")
|
|
1189
|
+
|
|
1190
|
+
|
|
1191
|
+
def write_operator_actions(runtime: Path, plan: dict[str, Any], subscriptions: list[dict[str, Any]], skipped: list[dict[str, Any]], harvested_on: str | None,
|
|
1192
|
+
harvested_domains: list[str] | None = None) -> None:
|
|
1193
|
+
path = runtime / OPERATOR_ACTIONS_FILENAME
|
|
1194
|
+
previous = START.read_object(path)
|
|
1195
|
+
actions, harvests = [], []
|
|
1196
|
+
for competitor in plan["competitors"]:
|
|
1197
|
+
domain = competitor["domain"]
|
|
1198
|
+
observed = [s for s in subscriptions if s["domain"] == domain]
|
|
1199
|
+
for subscription in observed or [None]:
|
|
1200
|
+
scope = subscription_scope(subscription) if subscription else domain
|
|
1201
|
+
old = {**previous}
|
|
1202
|
+
for key in ("actions", "harvests"):
|
|
1203
|
+
old[key] = [{**item, "id": domain + item["id"][len(scope):]}
|
|
1204
|
+
for item in previous.get(key, []) if item.get("id", "").startswith(scope + ":")]
|
|
1205
|
+
generated = START.operator_actions({**plan, "competitors": [competitor]}, [subscription] if subscription else [], skipped,
|
|
1206
|
+
harvested_on, old, [domain] if scope in (harvested_domains or []) else [])
|
|
1207
|
+
for key, output in (("actions", actions), ("harvests", harvests)):
|
|
1208
|
+
for item in generated[key]:
|
|
1209
|
+
item["id"] = scope + item["id"][len(domain):]
|
|
1210
|
+
if item.get("blocked_by"):
|
|
1211
|
+
item["blocked_by"] = scope + item["blocked_by"][len(domain):]
|
|
1212
|
+
if subscription:
|
|
1213
|
+
item.update(funnel_lineage(subscription))
|
|
1214
|
+
output.append(item)
|
|
1215
|
+
_write_atomic(path, json.dumps({"schema_version": 1, "section_id": SECTION_ID, "capture_id": plan["capture_id"],
|
|
1216
|
+
"product_domain": plan.get("product_domain", ""), "actions": actions, "harvests": harvests}, indent=1))
|
|
1217
|
+
|
|
1218
|
+
|
|
1219
|
+
PHASE_FILENAME = "phase.json"
|
|
1220
|
+
|
|
1221
|
+
|
|
1222
|
+
def _previous_phase(runtime: Path) -> str | None:
|
|
1223
|
+
"""The phase of the last completed publication (a durable marker: the
|
|
1224
|
+
receipt itself is reset to pending at the start of every attempt)."""
|
|
1225
|
+
try:
|
|
1226
|
+
state = json.loads((runtime / PHASE_FILENAME).read_text(encoding="utf-8"))
|
|
1227
|
+
except (OSError, json.JSONDecodeError):
|
|
1228
|
+
state = {}
|
|
1229
|
+
if state.get("phase") in ("started", "harvested"):
|
|
1230
|
+
return str(state["phase"])
|
|
1231
|
+
if (runtime / "harvest-as-of.json").is_file():
|
|
1232
|
+
return "harvested"
|
|
1233
|
+
return None
|
|
1234
|
+
|
|
1235
|
+
|
|
1236
|
+
def _pending(runtime: Path, reason: str, *, extra: dict[str, Any], client: OsClient) -> dict[str, Any]:
|
|
1237
|
+
state = {"schema_version": SCHEMA_VERSION, "section_id": SECTION_ID, "status": "pending", "error": None, "reason": reason, **extra,
|
|
1238
|
+
"research_context": {"path": (RUNTIME_REL / "input-context.json").as_posix(),
|
|
1239
|
+
"sha256": hashlib.sha256((runtime / "input-context.json").read_bytes()).hexdigest()},
|
|
1240
|
+
"attempt_inbox_calls": client.inbox_calls, "attempt_message_reads": client.message_reads, "attempt_cache_hits": client.cache_hits}
|
|
1241
|
+
_write_atomic(runtime / "request.json", json.dumps(state, indent=1))
|
|
1242
|
+
return state
|
|
1243
|
+
|
|
1244
|
+
|
|
1245
|
+
def _run(workspace: Path, args: Any) -> dict[str, Any]:
|
|
1246
|
+
api_key = (os.environ.get("OS_API_KEY") or "").strip()
|
|
1247
|
+
if not api_key:
|
|
1248
|
+
raise LifecycleCaptureError("OS_API_KEY is required")
|
|
1249
|
+
os_url = (os.environ.get("OS_URL") or DEFAULT_OS_URL).strip() or DEFAULT_OS_URL
|
|
1250
|
+
runtime = workspace / RUNTIME_REL
|
|
1251
|
+
runtime.mkdir(parents=True, exist_ok=True)
|
|
1252
|
+
if bool(_arg(args, "funnel_mail", False)):
|
|
1253
|
+
client = OsClient(os_url=os_url, api_key=api_key, cache_dir=runtime / "cache" / "start", shared_dir=runtime / "cache" / "shared",
|
|
1254
|
+
secret_dir=workspace / REPORT_REL.parent / ".local" / SECTION_ID / "mail-cache",
|
|
1255
|
+
max_inbox_calls=int(_arg(args, "max_inbox_calls", MAX_INBOX_CALLS_PER_ATTEMPT)),
|
|
1256
|
+
max_message_reads=int(_arg(args, "max_message_reads", MAX_MESSAGE_READS_PER_ATTEMPT)), key_scope=f"{SECTION_ID}:")
|
|
1257
|
+
mailbox = client.mail_account()
|
|
1258
|
+
registry = identity_registry(workspace, mailbox, {})
|
|
1259
|
+
handoff = load_funnel_handoff(workspace, mailbox, registry)
|
|
1260
|
+
result = capture_funnel_events(client, workspace, handoff, refresh=bool(_arg(args, "refresh", False)))
|
|
1261
|
+
return {"status": "pending" if result["pending"] or any(not gap.get("terminal") for gap in result["coverage"]) else "completed", "phase": "funnel-mail",
|
|
1262
|
+
"events_path": (RUNTIME_REL / FUNNEL_EVENTS_FILENAME).as_posix(), "events": len(result["events"]), "pending": result["pending"],
|
|
1263
|
+
"coverage": result["coverage"], "subscriptions": len(handoff["subscriptions"])}
|
|
1264
|
+
product_domain = registrable_domain(str(_arg(args, "product_domain", "") or "")) if _arg(args, "product_domain", "") else ""
|
|
1265
|
+
max_competitors = max(1, int(_arg(args, "max_competitors", MAX_COMPETITORS)))
|
|
1266
|
+
explicit = [d for d in CF.parse_domains(str(_arg(args, "domains", "") or ""), CF.MAX_EXPLICIT_DOMAINS) if d != product_domain]
|
|
1267
|
+
evidence = _load_script("research-evidence.py", "pmr_research_evidence_for_lifecycle_capture")
|
|
1268
|
+
tiered = evidence.competitor_set_inputs(workspace, consumer=SECTION_ID, cap=MAX_COMPETITORS)
|
|
1269
|
+
rejected = evidence.competitor_set_excluded_domains(workspace) - set(explicit)
|
|
1270
|
+
discovered = evidence.keyword_inputs(workspace, consumer=SECTION_ID, row_type="competitor_domain", field="domain", cap=MAX_COMPETITORS, exclude=rejected)
|
|
1271
|
+
domains: list[str] = []
|
|
1272
|
+
domain_sources: dict[str, str] = {}
|
|
1273
|
+
for source, candidates in (("explicit", explicit), ("competitor_set", tiered), ("keyword_economics", discovered)):
|
|
1274
|
+
for domain in candidates:
|
|
1275
|
+
domain = registrable_domain(domain)
|
|
1276
|
+
if len(domains) < max_competitors and domain and domain != product_domain and domain not in domains:
|
|
1277
|
+
domains.append(domain)
|
|
1278
|
+
domain_sources[domain] = source
|
|
1279
|
+
if not domains:
|
|
1280
|
+
raise LifecycleCaptureError("no competitor domains available: pass --domains, or complete the Competitor Set (or Keyword Economics) section first")
|
|
1281
|
+
|
|
1282
|
+
subscriptions_path = Path(_arg(args, "subscriptions", "") or (runtime / SUBSCRIPTIONS_FILENAME))
|
|
1283
|
+
if not subscriptions_path.is_absolute():
|
|
1284
|
+
subscriptions_path = workspace / subscriptions_path
|
|
1285
|
+
want_harvest = bool(_arg(args, "harvest", False))
|
|
1286
|
+
want_start = bool(_arg(args, "start", False))
|
|
1287
|
+
client = OsClient(os_url=os_url, api_key=api_key, cache_dir=runtime / "cache" / "start", shared_dir=runtime / "cache" / "shared",
|
|
1288
|
+
secret_dir=workspace / REPORT_REL.parent / ".local" / SECTION_ID / "mail-cache",
|
|
1289
|
+
max_inbox_calls=int(_arg(args, "max_inbox_calls", MAX_INBOX_CALLS_PER_ATTEMPT)),
|
|
1290
|
+
max_message_reads=int(_arg(args, "max_message_reads", MAX_MESSAGE_READS_PER_ATTEMPT)), key_scope=f"{SECTION_ID}:")
|
|
1291
|
+
mailbox = client.mail_account()
|
|
1292
|
+
previous_plan = START.read_object(runtime / PLAN_FILENAME)
|
|
1293
|
+
if previous_plan and previous_plan.get("mailbox") != mailbox:
|
|
1294
|
+
raise LifecycleCaptureError("this capture belongs to another mailbox; use a fresh research workspace")
|
|
1295
|
+
if previous_plan.get("product_domain") and previous_plan["product_domain"] != product_domain:
|
|
1296
|
+
raise LifecycleCaptureError("this capture belongs to another product; use a fresh research workspace")
|
|
1297
|
+
marker = START.read_object(runtime / "capture.json")
|
|
1298
|
+
capture_id = str(previous_plan.get("capture_id") or marker.get("capture_id") or uuid.uuid4())
|
|
1299
|
+
try:
|
|
1300
|
+
uuid.UUID(capture_id)
|
|
1301
|
+
except ValueError as exc:
|
|
1302
|
+
raise LifecycleCaptureError("capture_id must be a UUID") from exc
|
|
1303
|
+
# Persist before the network call so a lost reservation response is safely retryable.
|
|
1304
|
+
_write_atomic(runtime / "capture.json", json.dumps({"capture_id": capture_id}))
|
|
1305
|
+
existing_subscriptions = START.read_object(subscriptions_path)
|
|
1306
|
+
active_legacy = {registrable_domain(str(s.get("domain") or "")): str(s.get("alias") or "").strip().lower()
|
|
1307
|
+
for s in existing_subscriptions.get("subscriptions", []) if isinstance(s, dict)}
|
|
1308
|
+
legacy_aliases = {c["domain"]: c["alias"] for c in previous_plan.get("competitors", [])
|
|
1309
|
+
if c.get("domain") in domains and active_legacy.get(c["domain"]) == c.get("alias") == alias_for(mailbox, c["domain"])}
|
|
1310
|
+
for c in previous_plan.get("competitors", []):
|
|
1311
|
+
c["legacy"] = c.get("domain") in legacy_aliases
|
|
1312
|
+
reserve_domains = [d for d in domains if d not in legacy_aliases]
|
|
1313
|
+
aliases = {**legacy_aliases, **(client.reserve_aliases(capture_id, reserve_domains, mailbox) if reserve_domains else {})}
|
|
1314
|
+
if len(set(aliases.values())) != len(aliases):
|
|
1315
|
+
raise LifecycleCaptureError("two competitors map to one alias; name them with distinct registrable domains")
|
|
1316
|
+
registry = identity_registry(workspace, mailbox, aliases)
|
|
1317
|
+
handoff = load_funnel_handoff(workspace, mailbox, registry)
|
|
1318
|
+
points = entry_points(workspace, domains)
|
|
1319
|
+
discovery_path = Path(_arg(args, "entry_points", "") or (runtime / ENTRY_POINTS_FILENAME))
|
|
1320
|
+
if not discovery_path.is_absolute():
|
|
1321
|
+
discovery_path = workspace / discovery_path
|
|
1322
|
+
reviewed = START.discovery(discovery_path, domains, registrable_domain)
|
|
1323
|
+
plan = _plan(mailbox, domains, aliases, points, subscriptions_path, capture_id=capture_id, reviewed=reviewed,
|
|
1324
|
+
previous=previous_plan, product_domain=product_domain)
|
|
1325
|
+
plan["imported_funnel_subscriptions"] = handoff["subscriptions"]
|
|
1326
|
+
imported_domains = {item["domain"] for item in handoff["subscriptions"] if item["domain"] in domains}
|
|
1327
|
+
_write_atomic(runtime / PLAN_FILENAME, json.dumps(plan, indent=1, ensure_ascii=False))
|
|
1328
|
+
signup_plan = {**plan, "competitors": [c for c in plan["competitors"] if c["domain"] not in imported_domains]}
|
|
1329
|
+
signup_list = START.render_signups(signup_plan)
|
|
1330
|
+
for item in handoff["subscriptions"]:
|
|
1331
|
+
if item["domain"] in imported_domains:
|
|
1332
|
+
signup_list += f"\n## {item['domain']} — {item['funnel_id']}\n\nAlready registered through approved section 05; do not register again.\nAddress: `{item['alias']}`\nEntry: {item['entry_url']}\n"
|
|
1333
|
+
_write_atomic(runtime / "lifecycle-signups.md", signup_list)
|
|
1334
|
+
planned = {c["domain"]: c for c in plan["competitors"]}
|
|
1335
|
+
recorded = load_subscriptions(subscriptions_path, domains, aliases, planned)
|
|
1336
|
+
if imported_domains:
|
|
1337
|
+
recorded = recorded or {"subscriptions": [], "skipped": []}
|
|
1338
|
+
recorded["subscriptions"].extend(item for item in handoff["subscriptions"] if item["domain"] in domains)
|
|
1339
|
+
recorded["skipped"] = [item for item in recorded["skipped"] if item["domain"] not in imported_domains]
|
|
1340
|
+
phase_marker = START.read_object(runtime / PHASE_FILENAME)
|
|
1341
|
+
previous_windows = phase_marker.get("subscription_windows") or {}
|
|
1342
|
+
old_scopes = phase_marker.get("harvested_scopes", phase_marker.get("harvested_domains", []))
|
|
1343
|
+
still_observed = [subscription_scope(s) for s in (recorded or {}).get("subscriptions", [])
|
|
1344
|
+
if subscription_scope(s) in old_scopes
|
|
1345
|
+
and previous_windows.get(subscription_scope(s)) == {"alias": s["alias"], "subscribed_on": s["subscribed_on"]}]
|
|
1346
|
+
write_operator_actions(runtime, plan, (recorded or {}).get("subscriptions", []), (recorded or {}).get("skipped", []),
|
|
1347
|
+
phase_marker.get("harvested_on"), still_observed)
|
|
1348
|
+
accounted = {s["domain"] for s in (recorded or {}).get("subscriptions", [])} | {s["domain"] for s in (recorded or {}).get("skipped", [])}
|
|
1349
|
+
missing = [d for d in domains if d not in accounted]
|
|
1350
|
+
skipped_domains = {s["domain"] for s in (recorded or {}).get("skipped", [])}
|
|
1351
|
+
discovery_missing = [d for d in domains if d not in skipped_domains and d not in imported_domains and planned[d].get("discovery_status") != "reviewed" and not planned[d].get("legacy")]
|
|
1352
|
+
if discovery_missing and not (want_harvest and (recorded or {}).get("subscriptions")):
|
|
1353
|
+
return _pending(runtime, "awaiting reviewed entry-point discovery", extra={"capture_id": capture_id,
|
|
1354
|
+
"plan_path": (RUNTIME_REL / PLAN_FILENAME).as_posix(), "subscriptions_path": subscriptions_path.as_posix(),
|
|
1355
|
+
"entry_points_path": discovery_path.as_posix(), "missing_entry_points": discovery_missing,
|
|
1356
|
+
"unsubscribed": missing, "mailbox_domain": mailbox.partition("@")[2]}, client=client)
|
|
1357
|
+
if recorded is None or missing:
|
|
1358
|
+
if want_harvest and not (recorded or {}).get("subscriptions"):
|
|
1359
|
+
raise LifecycleCaptureError("harvest needs at least one recorded subscription first")
|
|
1360
|
+
if not want_harvest:
|
|
1361
|
+
return _pending(runtime, "awaiting research subscriptions for the planned aliases",
|
|
1362
|
+
extra={"plan_path": (RUNTIME_REL / PLAN_FILENAME).as_posix(), "subscriptions_path": subscriptions_path.as_posix(),
|
|
1363
|
+
"capture_id": capture_id, "unsubscribed": missing or domains, "mailbox_domain": mailbox.partition("@")[2]}, client=client)
|
|
1364
|
+
subscriptions, skipped = recorded["subscriptions"], recorded["skipped"]
|
|
1365
|
+
coverage: list[dict[str, Any]] = []
|
|
1366
|
+
for d in missing:
|
|
1367
|
+
coverage.append({"type": "coverage", "domain": d, "source": "subscription", "status": "pending",
|
|
1368
|
+
"detail": "entry-point discovery is pending; no observation window has started" if d in discovery_missing
|
|
1369
|
+
else "human sign-up is pending; no observation window has started", "tier": "C"})
|
|
1370
|
+
# The first run with every competitor recorded publishes the subscriptions
|
|
1371
|
+
# (phase started); later runs harvest. Flags override that order.
|
|
1372
|
+
previously = _previous_phase(runtime)
|
|
1373
|
+
if want_harvest:
|
|
1374
|
+
phase = "harvested"
|
|
1375
|
+
elif previously == "harvested":
|
|
1376
|
+
phase = "harvested" # A start refresh must never erase an already observed sequence.
|
|
1377
|
+
elif want_start or previously is None:
|
|
1378
|
+
phase = "started"
|
|
1379
|
+
else:
|
|
1380
|
+
phase = "harvested"
|
|
1381
|
+
if phase == "harvested" and not subscriptions:
|
|
1382
|
+
phase = "started"
|
|
1383
|
+
harvested_on: date | None = None
|
|
1384
|
+
messages: list[dict[str, Any]] = []
|
|
1385
|
+
sequences: list[dict[str, Any]] = []
|
|
1386
|
+
patterns: list[dict[str, Any]] = []
|
|
1387
|
+
if phase == "harvested":
|
|
1388
|
+
requested_as_of = str(_arg(args, "as_of", "") or "")
|
|
1389
|
+
if requested_as_of and phase_marker.get("harvested_on") and requested_as_of < phase_marker["harvested_on"]:
|
|
1390
|
+
raise LifecycleCaptureError("--as-of cannot replace a newer published harvest; read its private snapshot instead")
|
|
1391
|
+
harvested_on = resolve_harvest_day(runtime, str(_arg(args, "as_of", "") or ""), bool(_arg(args, "refresh", False)),
|
|
1392
|
+
not_before=min(date.fromisoformat(s["subscribed_on"]) for s in subscriptions))
|
|
1393
|
+
client.cache_dir = runtime / "cache" / "harvest" / harvested_on.isoformat()
|
|
1394
|
+
messages, sequences = harvest(client, subscriptions, harvested_on, coverage)
|
|
1395
|
+
patterns = lifecycle_patterns(sequences)
|
|
1396
|
+
else:
|
|
1397
|
+
coverage.append({"type": "coverage", "domain": "all", "source": "inbox", "status": "pending",
|
|
1398
|
+
"detail": f"subscriptions started; the inbox is harvested after {EARLY_HARVEST_AFTER_DAYS} days for early evidence and {HARVEST_AFTER_DAYS} days for the full window", "tier": "C"})
|
|
1399
|
+
for s in skipped:
|
|
1400
|
+
CF._coverage_gap(coverage, "subscription", [s["domain"]], f"skipped: {s['reason']}")
|
|
1401
|
+
totals = client.ledger_totals()
|
|
1402
|
+
if totals["observations"] == 0:
|
|
1403
|
+
raise LifecycleCaptureError("no OS mail call succeeded; rerun later")
|
|
1404
|
+
if phase == "harvested" and totals["inbox"] == 0:
|
|
1405
|
+
raise LifecycleCaptureError("no inbox listing succeeded in this harvest; rerun later")
|
|
1406
|
+
|
|
1407
|
+
private = private_strings([*aliases.values(), *(item["alias"] for item in subscriptions)], mailbox)
|
|
1408
|
+
subscription_rows = [public_row({**funnel_lineage(s), "type": "subscription", "domain": s["domain"], "kind": s["kind"], "entry_url": public_entry_url(s["entry_url"]),
|
|
1409
|
+
"entry_host": registrable_domain(urllib.parse.urlsplit(s["entry_url"]).hostname or ""), "subscribed_on": s["subscribed_on"],
|
|
1410
|
+
"entry_type": s.get("entry_type"), "source_url": public_entry_url(s.get("source_url") or s["entry_url"]),
|
|
1411
|
+
"early_harvest_due_on": (date.fromisoformat(s["subscribed_on"]) + timedelta(days=EARLY_HARVEST_AFTER_DAYS)).isoformat(),
|
|
1412
|
+
"full_harvest_due_on": (date.fromisoformat(s["subscribed_on"]) + timedelta(days=HARVEST_AFTER_DAYS)).isoformat(),
|
|
1413
|
+
"confirmed": s["confirmed"], "unsubscribed_on": s.get("unsubscribed_on"), "status": "unsubscribed" if s.get("unsubscribed_on") else "active",
|
|
1414
|
+
"notes": s.get("notes"), "tier": "A"}, private) for s in subscriptions]
|
|
1415
|
+
sequences = [public_row(r, private) for r in sequences]
|
|
1416
|
+
messages = [public_row(r, private) for r in messages]
|
|
1417
|
+
patterns = [public_row(r, private) for r in patterns]
|
|
1418
|
+
coverage = [public_row(r, private) for r in coverage]
|
|
1419
|
+
skipped = [public_text(s, private) for s in skipped]
|
|
1420
|
+
rows = subscription_rows + sequences + messages + patterns + coverage
|
|
1421
|
+
observation = {
|
|
1422
|
+
"observation_status": "pending" if phase == "started" else "full" if sequences and not missing and len(sequences) == len(subscriptions) and all(s["observation_status"] == "full" for s in sequences) else "partial",
|
|
1423
|
+
"target_days": HARVEST_AFTER_DAYS,
|
|
1424
|
+
"observed_from": min((s["observed_from"] for s in sequences), default=None),
|
|
1425
|
+
"observed_through": harvested_on.isoformat() if harvested_on else None,
|
|
1426
|
+
"days_covered": min((s["days_covered"] for s in sequences), default=0),
|
|
1427
|
+
"coverage_complete": bool(sequences) and not missing and not skipped and len(sequences) == len(subscriptions) and all(s["coverage_complete"] for s in sequences),
|
|
1428
|
+
"early_harvest_due_on": min((s["early_harvest_due_on"] for s in subscription_rows), default=None),
|
|
1429
|
+
"full_harvest_due_on": max((s["full_harvest_due_on"] for s in subscription_rows), default=None),
|
|
1430
|
+
}
|
|
1431
|
+
retrieved_on = totals["first_retrieved_on"] if totals["first_retrieved_on"] == totals["last_retrieved_on"] else f"{totals['first_retrieved_on']}..{totals['last_retrieved_on']}"
|
|
1432
|
+
evidence_doc = {
|
|
1433
|
+
"schema_version": SCHEMA_VERSION, "section_id": SECTION_ID, "generated_by": "run-lifecycle-capture.py", "retrieved_on": retrieved_on,
|
|
1434
|
+
"inputs": {"domains": domains, "domain_sources": domain_sources, "product_domain": product_domain, "mailbox_domain": mailbox.partition("@")[2],
|
|
1435
|
+
"capture_id": capture_id, "phase": phase, "harvested_on": harvested_on.isoformat() if harvested_on else None,
|
|
1436
|
+
"harvest_after_days": HARVEST_AFTER_DAYS, "early_harvest_after_days": EARLY_HARVEST_AFTER_DAYS, **observation},
|
|
1437
|
+
"rows": rows,
|
|
1438
|
+
"totals": {**totals, "subscriptions": len(subscriptions), "skipped": len(skipped), "messages": len(messages), "patterns": len(patterns)},
|
|
1439
|
+
}
|
|
1440
|
+
evidence_json = json.dumps(evidence_doc, indent=1, ensure_ascii=False)
|
|
1441
|
+
_write_atomic(workspace / EVIDENCE_REL, evidence_json)
|
|
1442
|
+
section_markdown = render_section(domains=domains, mailbox_domain=mailbox.partition("@")[2], subscriptions=subscription_rows, skipped=skipped, sequences=sequences,
|
|
1443
|
+
messages=messages, patterns=patterns, coverage=coverage, phase=phase,
|
|
1444
|
+
harvested_on=harvested_on.isoformat() if harvested_on else None, totals=totals)
|
|
1445
|
+
_write_atomic(runtime / "result.md", section_markdown)
|
|
1446
|
+
upsert = _load_script("upsert-markdown-section.py", "pmr_upsert_markdown_section_for_lifecycle_capture")
|
|
1447
|
+
report_path = workspace / REPORT_REL
|
|
1448
|
+
report_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1449
|
+
upsert.upsert_markdown_section(report_path, SECTION_HEADING, section_markdown)
|
|
1450
|
+
persist = _load_script("persist-research-report.py", "pmr_persist_research_report_for_lifecycle_capture")
|
|
1451
|
+
persistence = persist.persist_research_report(workspace, commit_message=f"research: update {SECTION_HEADING.lower()} section",
|
|
1452
|
+
extra_paths=(Path(_artifact_path("egbe-docs/research/market/evidence/lifecycle-capture.json")),))
|
|
1453
|
+
io = _load_script("research-artifact-io.py", "pmr_artifact_io_for_lifecycle_capture")
|
|
1454
|
+
state = {
|
|
1455
|
+
"schema_version": SCHEMA_VERSION, "section_id": SECTION_ID, "status": "completed", "error": None, "phase": phase,
|
|
1456
|
+
"capture_id": capture_id, **observation,
|
|
1457
|
+
"domains": domains, "domain_sources": domain_sources, "product_domain": product_domain, "mailbox_domain": mailbox.partition("@")[2],
|
|
1458
|
+
"harvested_on": harvested_on.isoformat() if harvested_on else None, "subscriptions": len(subscriptions), "skipped": len(skipped),
|
|
1459
|
+
"messages": len(messages), "patterns": len(patterns), "provider_calls": totals["provider_calls"], "observations": totals["observations"],
|
|
1460
|
+
"inbox_calls": totals["inbox"], "message_reads": totals["message"],
|
|
1461
|
+
"attempt_inbox_calls": client.inbox_calls, "attempt_message_reads": client.message_reads, "attempt_cache_hits": client.cache_hits,
|
|
1462
|
+
"subscriptions_sha256": hashlib.sha256(subscriptions_path.read_bytes()).hexdigest() if subscriptions_path.is_file() else None,
|
|
1463
|
+
"funnel_handoff_sha256": hashlib.sha256((workspace / FUNNEL_HANDOFF_REL).read_bytes()).hexdigest() if (workspace / FUNNEL_HANDOFF_REL).is_file() else None,
|
|
1464
|
+
"identity_registry_binding": _load_script("research_integrity.py", "pmr_lifecycle_identity_integrity").identity_binding(
|
|
1465
|
+
registry, lifecycle_domains=domains, funnel_ids=[s["funnel_id"] for s in subscriptions if s.get("funnel_id")]),
|
|
1466
|
+
"research_context": io.context_binding(workspace, SECTION_ID),
|
|
1467
|
+
"publication_mode": persistence.publication_mode, "report_commit_sha": persistence.commit_sha,
|
|
1468
|
+
"evidence_path": EVIDENCE_REL.as_posix(), "evidence_sha256": hashlib.sha256(evidence_json.encode("utf-8")).hexdigest(),
|
|
1469
|
+
"result_path": (RUNTIME_REL / "result.md").as_posix(), "result_sha256": hashlib.sha256(section_markdown.encode("utf-8")).hexdigest(),
|
|
1470
|
+
}
|
|
1471
|
+
_write_atomic(runtime / "request.json", json.dumps(state, indent=1))
|
|
1472
|
+
successful_scopes = [subscription_scope(s) for s in sequences if s["listing_succeeded"]]
|
|
1473
|
+
successful_domains = sorted({s["domain"] for s in sequences if s["listing_succeeded"]})
|
|
1474
|
+
_write_atomic(runtime / PHASE_FILENAME, json.dumps({"phase": phase, "published_on": date.today().isoformat(),
|
|
1475
|
+
"harvested_on": harvested_on.isoformat() if harvested_on else phase_marker.get("harvested_on"),
|
|
1476
|
+
"harvested_domains": successful_domains if harvested_on else sorted({s["domain"] for s in subscriptions if subscription_scope(s) in still_observed}),
|
|
1477
|
+
"harvested_scopes": successful_scopes if harvested_on else still_observed,
|
|
1478
|
+
"subscription_windows": {subscription_scope(s): {"alias": s["alias"], "subscribed_on": s["subscribed_on"]} for s in subscriptions}
|
|
1479
|
+
if harvested_on else previous_windows}))
|
|
1480
|
+
if harvested_on is not None:
|
|
1481
|
+
mark_harvest_completed(runtime, harvested_on)
|
|
1482
|
+
write_operator_actions(runtime, plan, subscriptions, recorded["skipped"], harvested_on.isoformat(), successful_scopes)
|
|
1483
|
+
# Immutable private provenance for handed-over packs, even for multiple revisions on one day.
|
|
1484
|
+
snapshot = runtime / "snapshots" / (harvested_on or date.today()).isoformat() / state["evidence_sha256"]
|
|
1485
|
+
for name, contents in (("evidence.json", evidence_json), ("result.md", section_markdown), ("request.json", json.dumps(state, indent=1))):
|
|
1486
|
+
path = snapshot / name
|
|
1487
|
+
if not path.exists():
|
|
1488
|
+
_write_atomic(path, contents)
|
|
1489
|
+
return state
|
|
1490
|
+
|
|
1491
|
+
|
|
1492
|
+
def run(workspace: Path, args: Any) -> dict[str, Any]:
|
|
1493
|
+
workspace = Path(workspace).resolve()
|
|
1494
|
+
io = _load_script("research-artifact-io.py", "pmr_artifact_io")
|
|
1495
|
+
if bool(_arg(args, "funnel_mail", False)):
|
|
1496
|
+
# A same-day mailbox check is a 05 continuation, not a publication or
|
|
1497
|
+
# a failed/completed receipt for the independent section 02 capture.
|
|
1498
|
+
with io.file_lock(workspace / RUNTIME_REL / ".run.lock", timeout=1):
|
|
1499
|
+
io.bind_context(workspace, SECTION_ID, context_file=getattr(args, "context_file", None))
|
|
1500
|
+
return _run(workspace, args)
|
|
1501
|
+
with io.section_attempt(workspace, SECTION_ID, context_file=getattr(args, "context_file", None)):
|
|
1502
|
+
try:
|
|
1503
|
+
return _run(workspace, args)
|
|
1504
|
+
except ValueError as exc:
|
|
1505
|
+
raise LifecycleCaptureError(str(exc)) from exc
|
|
1506
|
+
|
|
1507
|
+
|
|
1508
|
+
def main() -> int:
|
|
1509
|
+
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
1510
|
+
parser.add_argument("--workspace", type=Path, required=True)
|
|
1511
|
+
parser.add_argument("--context-file", type=Path, help="Optional selected research context; existing snapshots are reused on resume.")
|
|
1512
|
+
parser.add_argument("--domains", default="", help="Comma-separated competitor domains; merged ahead of the Competitor Set and Keyword Economics domains.")
|
|
1513
|
+
parser.add_argument("--product-domain", default="", help="The product's own domain; never subscribed to as a competitor.")
|
|
1514
|
+
parser.add_argument("--max-competitors", type=int, default=MAX_COMPETITORS)
|
|
1515
|
+
parser.add_argument("--subscriptions", default="", help=_artifact_path(f"Path to the subscriptions file (default egbe-docs/research/market/work/{SECTION_ID}/{SUBSCRIPTIONS_FILENAME})."))
|
|
1516
|
+
parser.add_argument("--entry-points", default="", help="Reviewed entry discovery JSON (default: the private runtime entry-points.json).")
|
|
1517
|
+
mode = parser.add_mutually_exclusive_group()
|
|
1518
|
+
mode.add_argument("--funnel-mail", action="store_true", help="Read confirmations/results for imported 05 aliases today; writes runtime events only, without completing 02.")
|
|
1519
|
+
mode.add_argument("--start", action="store_true", help="Publish recorded subscriptions before the first harvest; later refreshes preserve the harvested phase.")
|
|
1520
|
+
mode.add_argument("--harvest", action="store_true", help="Read the inbox by alias and publish the sequences (the default once subscriptions are recorded).")
|
|
1521
|
+
parser.add_argument("--as-of", default="", help="Harvest day YYYY-MM-DD (default: today for a new harvest, kept across reruns).")
|
|
1522
|
+
parser.add_argument("--refresh", action="store_true", help="For --funnel-mail, refresh today’s inbox listing; for ordinary harvest, advance a completed daily snapshot.")
|
|
1523
|
+
parser.add_argument("--max-inbox-calls", type=int, default=MAX_INBOX_CALLS_PER_ATTEMPT)
|
|
1524
|
+
parser.add_argument("--max-message-reads", type=int, default=MAX_MESSAGE_READS_PER_ATTEMPT)
|
|
1525
|
+
args = parser.parse_args()
|
|
1526
|
+
try:
|
|
1527
|
+
state = run(args.workspace, args)
|
|
1528
|
+
except (LifecycleCaptureError, ValueError) as exc:
|
|
1529
|
+
print(f"{DISPLAY_NAME}: {exc}", file=sys.stderr)
|
|
1530
|
+
return 1
|
|
1531
|
+
if state.get("phase") == "funnel-mail":
|
|
1532
|
+
print(json.dumps(state, indent=1))
|
|
1533
|
+
return 2 if state["status"] == "pending" else 0
|
|
1534
|
+
if state["status"] == "pending":
|
|
1535
|
+
print(f"{DISPLAY_NAME}: {state['reason']} — read {state['plan_path']} and the lifecycle workflow; "
|
|
1536
|
+
f"the assigned human's results go in {state['subscriptions_path']}. Continue other research while waiting.", file=sys.stderr)
|
|
1537
|
+
return 2
|
|
1538
|
+
print(json.dumps({k: state[k] for k in ("status", "phase", "domains", "subscriptions", "skipped", "messages", "patterns")}, indent=1))
|
|
1539
|
+
return 0
|
|
1540
|
+
|
|
1541
|
+
|
|
1542
|
+
if __name__ == "__main__":
|
|
1543
|
+
raise SystemExit(main())
|