@egbe-ai/opencode 0.0.0-stage → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +4 -0
- package/README.md +124 -2
- package/index.js +5 -0
- package/package.json +55 -4
- package/plugin.js +230 -0
- package/session-log.js +315 -0
- package/skills/.egbe-catalog/manifest.json +1 -0
- package/skills/egbe-os-analytics/SKILL.md +499 -0
- package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
- package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
- package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
- package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
- package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
- package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
- package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
- package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
- package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
- package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
- package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
- package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
- package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
- package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
- package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
- package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
- package/skills/egbe-os-db/SKILL.md +174 -0
- package/skills/egbe-os-deploy/SKILL.md +412 -0
- package/skills/egbe-os-design/SKILL.md +182 -0
- package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
- package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
- package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
- package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
- package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
- package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
- package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
- package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
- package/skills/egbe-os-destination-readiness/references/report.md +116 -0
- package/skills/egbe-os-domains/SKILL.md +189 -0
- package/skills/egbe-os-errors/SKILL.md +259 -0
- package/skills/egbe-os-git/SKILL.md +275 -0
- package/skills/egbe-os-git/references/issues.md +33 -0
- package/skills/egbe-os-google-ads/SKILL.md +163 -0
- package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
- package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
- package/skills/egbe-os-google-ads/references/conversions.md +153 -0
- package/skills/egbe-os-google-ads/references/delivery.md +128 -0
- package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
- package/skills/egbe-os-google-ads/references/reporting.md +105 -0
- package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
- package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
- package/skills/egbe-os-google-auth/SKILL.md +285 -0
- package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
- package/skills/egbe-os-images/SKILL.md +163 -0
- package/skills/egbe-os-images/references/stock-images.md +21 -0
- package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
- package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
- package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
- package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
- package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
- package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
- package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
- package/skills/egbe-os-llm/SKILL.md +160 -0
- package/skills/egbe-os-mail/SKILL.md +180 -0
- package/skills/egbe-os-me/SKILL.md +119 -0
- package/skills/egbe-os-meta-ads/SKILL.md +114 -0
- package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
- package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
- package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
- package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
- package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
- package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
- package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
- package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
- package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
- package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
- package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
- package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
- package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
- package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
- package/skills/egbe-os-notifications/SKILL.md +78 -0
- package/skills/egbe-os-openai-ads/SKILL.md +149 -0
- package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
- package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
- package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
- package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
- package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
- package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
- package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
- package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
- package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
- package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
- package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
- package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
- package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
- package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
- package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
- package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
- package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
- package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
- package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
- package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
- package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
- package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
- package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
- package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
- package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
- package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
- package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
- package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
- package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
- package/skills/egbe-os-payments/SKILL.md +215 -0
- package/skills/egbe-os-payments/agents/openai.yaml +4 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
- package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
- package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
- package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
- package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
- package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
- package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
- package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
- package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
- package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
- package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
- package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
- package/skills/egbe-os-playbook/SKILL.md +79 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
- package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
- package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
- package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
- package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
- package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
- package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
- package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
- package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
- package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
- package/skills/egbe-os-playbook/playbooks/product.md +67 -0
- package/skills/egbe-os-playbook/references/catalog.md +127 -0
- package/skills/egbe-os-playbook/references/changes.md +167 -0
- package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
- package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
- package/skills/egbe-os-product-auth/SKILL.md +151 -0
- package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
- package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
- package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
- package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
- package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
- package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
- package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
- package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
- package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
- package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
- package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
- package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
- package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
- package/skills/egbe-os-product-build/SKILL.md +111 -0
- package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-build/references/capabilities.md +25 -0
- package/skills/egbe-os-product-build/references/engineering.md +72 -0
- package/skills/egbe-os-product-build/references/execution.md +55 -0
- package/skills/egbe-os-product-build/references/foundations.md +93 -0
- package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
- package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
- package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
- package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
- package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
- package/skills/egbe-os-product-market-research/SKILL.md +159 -0
- package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
- package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
- package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
- package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
- package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
- package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
- package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
- package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
- package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
- package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
- package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
- package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
- package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
- package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
- package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
- package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
- package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
- package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
- package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
- package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
- package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
- package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
- package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
- package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
- package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
- package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
- package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
- package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
- package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
- package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
- package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
- package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
- package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
- package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
- package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
- package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
- package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
- package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
- package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
- package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
- package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
- package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
- package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
- package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
- package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
- package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
- package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
- package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
- package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
- package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
- package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
- package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
- package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
- package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
- package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
- package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
- package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
- package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
- package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
- package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
- package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
- package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
- package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
- package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
- package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
- package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
- package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
- package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
- package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
- package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
- package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
- package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
- package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
- package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
- package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
- package/skills/egbe-os-product-report/SKILL.md +150 -0
- package/skills/egbe-os-product-report/references/artifacts.md +84 -0
- package/skills/egbe-os-product-report/references/brief.md +34 -0
- package/skills/egbe-os-product-report/references/daily.md +76 -0
- package/skills/egbe-os-product-report/references/sources.md +68 -0
- package/skills/egbe-os-product-routines/SKILL.md +166 -0
- package/skills/egbe-os-product-routines/references/catalog.md +22 -0
- package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
- package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
- package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
- package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
- package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
- package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
- package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
- package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
- package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
- package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
- package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
- package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
- package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
- package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
- package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
- package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
- package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
- package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
- package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
- package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
- package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
- package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
- package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
- package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
- package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
- package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
- package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
- package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
- package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
- package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
- package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
- package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
- package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
- package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
- package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
- package/skills/egbe-os-report-ui/SKILL.md +133 -0
- package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
- package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
- package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
- package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
- package/skills/egbe-os-report-ui/references/components.md +209 -0
- package/skills/egbe-os-research/SKILL.md +125 -0
- package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
- package/skills/egbe-os-scrape/SKILL.md +160 -0
- package/skills/egbe-os-search/SKILL.md +182 -0
- package/skills/egbe-os-search/references/google-places.md +81 -0
- package/skills/egbe-os-support/SKILL.md +149 -0
- package/skills/egbe-os-telemetry/SKILL.md +344 -0
- package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
- package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
- package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
- package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
- package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
- package/skills/egbe-os-tools/SKILL.md +180 -0
- package/skills/egbe-os-tools/references/theona.md +26 -0
- package/skills/egbe-os-wallet/SKILL.md +181 -0
- package/skills/egbe-setup/EGBE.md +484 -0
- package/skills/egbe-setup/SKILL.md +116 -0
- package/skills/egbe-setup/references/project.md +80 -0
- package/state.js +37 -0
- package/update.js +361 -0
|
@@ -0,0 +1,927 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Collect Umami evidence and report per-session stage observations, not conversions.
|
|
3
|
+
|
|
4
|
+
Stdlib-only. Auth: project-local .env.local (OS_URL / OS_API_KEY), or environment.
|
|
5
|
+
Run from the project root, with the complete Egbe catalog installed.
|
|
6
|
+
|
|
7
|
+
python3 umami_funnel.py [--site <name-domain-or-id>] [--since 2026-07-01]
|
|
8
|
+
[--until 2026-08-01]
|
|
9
|
+
[--act-path /color] [--act-start-event generation_started]
|
|
10
|
+
[--act-event activated]
|
|
11
|
+
[--events-json evidence/events.json] [--json analysis/funnel.json]
|
|
12
|
+
python3 umami_funnel.py --from-events evidence/events.json --json analysis/replay.json
|
|
13
|
+
|
|
14
|
+
Output: totals (included vs excluded), traffic mix, depth/bounce, top paths,
|
|
15
|
+
custom events, stage counts, per-source sessions/arrivals, and compact,
|
|
16
|
+
filtered journeys for sessions with activity or exposure events. Raise
|
|
17
|
+
`--max-journey-events` when a longer timeline is needed. Repeated completion
|
|
18
|
+
events and a small allowlist of non-PII event properties are preserved by
|
|
19
|
+
joining Umami's paged event-data endpoint to event rows. Paths and canonical
|
|
20
|
+
visitor/visit IDs are anonymized before output. Exclude confirmed internal
|
|
21
|
+
activity and disclose suspects; completion is a technical proxy, never proof
|
|
22
|
+
of usable value. Filtering is described in references/data-recipes.md.
|
|
23
|
+
"""
|
|
24
|
+
import argparse
|
|
25
|
+
import collections
|
|
26
|
+
import datetime as dt
|
|
27
|
+
import hashlib
|
|
28
|
+
import json
|
|
29
|
+
import math
|
|
30
|
+
import os
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
import re
|
|
33
|
+
import runpy
|
|
34
|
+
import sys
|
|
35
|
+
import time
|
|
36
|
+
import urllib.parse
|
|
37
|
+
import urllib.error
|
|
38
|
+
import urllib.request
|
|
39
|
+
|
|
40
|
+
PAGE_SIZE, MAX_EVENT_PAGES = 500, 200
|
|
41
|
+
SIGNUP_PATHS = ("/signup", "/register", "/ru/signup", "/#/join")
|
|
42
|
+
POST_AUTH_PATHS = ("/dashboard", "/#/dashboard", "/#/app", "/account", "/admin")
|
|
43
|
+
CHECKOUT_PATHS = ("/checkout",)
|
|
44
|
+
PRICING_PATHS = ("/pricing", "/#pricing", "/#plans")
|
|
45
|
+
ENGAGE_EVENTS = {"campaign-visit", "engagement-scroll", "engagement-time", "engagement-click"}
|
|
46
|
+
EXPOSURE_EVENTS = {"section_view", "scroll_depth"}
|
|
47
|
+
SIGNUP_START_EVENTS = {"signup_started"}
|
|
48
|
+
SIGNUP_DONE_EVENTS = {"signup_completed"}
|
|
49
|
+
CHECKOUT_EVENTS = {"checkout_started"}
|
|
50
|
+
PAYWALL_EVENTS = {"paywall_view"}
|
|
51
|
+
TECHNICAL_START_EVENTS = {
|
|
52
|
+
"main_action_submit", "quiz_start", "request_form_start", "activation_started",
|
|
53
|
+
}
|
|
54
|
+
TECHNICAL_DONE_EVENTS = {
|
|
55
|
+
"activated", "main_action_success", "quiz_complete", "activation_completed",
|
|
56
|
+
}
|
|
57
|
+
SAFE_EVENT_FIELDS = {
|
|
58
|
+
"button", "cta", "action", "plan", "surface", "status", "hq",
|
|
59
|
+
"activation_type", "style", "room_type", "op", "count", "product_slug",
|
|
60
|
+
"cta_id", "cta_role", "button_id", "button_role", "input_type",
|
|
61
|
+
"result_type", "duration_ms", "checkout_type", "currency", "amount_cents",
|
|
62
|
+
"price_cents", "plan_name", "section", "percent", "thresholdSeconds", "position",
|
|
63
|
+
}
|
|
64
|
+
NUMERIC_EVENT_FIELDS = {
|
|
65
|
+
"count", "duration_ms", "amount_cents", "price_cents",
|
|
66
|
+
"percent", "thresholdSeconds", "position",
|
|
67
|
+
}
|
|
68
|
+
BOOLEAN_EVENT_FIELDS = {"hq"}
|
|
69
|
+
IDENTITY_EVENT_FIELDS = {"visitor_id", "visit_id"}
|
|
70
|
+
ASSET_PATH_PREFIXES = ("/_next", "/favicon", "/assets", "/static/")
|
|
71
|
+
SENSITIVE_PATH_PARENTS = {
|
|
72
|
+
"u", "user", "users", "r", "share", "invite", "reset", "verify",
|
|
73
|
+
"magic", "token", "session", "sessions",
|
|
74
|
+
}
|
|
75
|
+
EMAIL_ANY_RE = re.compile(r"[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}", re.IGNORECASE)
|
|
76
|
+
UUID_RE = re.compile(
|
|
77
|
+
r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
|
|
78
|
+
re.IGNORECASE,
|
|
79
|
+
)
|
|
80
|
+
JWT_RE = re.compile(r"[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}")
|
|
81
|
+
URL_RE = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*://\S+|\bwww\.\S+", re.IGNORECASE)
|
|
82
|
+
LONG_HEX_RE = re.compile(r"^[0-9a-f]{20,}$", re.IGNORECASE)
|
|
83
|
+
BASE64URL_RE = re.compile(r"^[A-Za-z0-9_-]{24,}$")
|
|
84
|
+
CONTROL_RE = re.compile(r"[\x00-\x1f\x7f]")
|
|
85
|
+
LONG_NUMBER_RE = re.compile(r"(?<!\d)\+?\d[\d .()-]{6,}\d(?!\d)")
|
|
86
|
+
SNAPSHOT_FORMAT = "egbe-umami-events.v1"
|
|
87
|
+
SAFE_UTM_FIELDS = ("utm_source", "utm_medium", "utm_campaign", "utm_content")
|
|
88
|
+
ANALYSIS_OPTIONS = (
|
|
89
|
+
"act_event", "act_start_event", "act_path", "internal_sources",
|
|
90
|
+
"internal_country", "dc_country", "max_journey_events",
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def os_connection():
|
|
95
|
+
root = Path(__file__).resolve().parents[1]
|
|
96
|
+
quickstart = "egbe-os-quickstart" if root.name.startswith("egbe-os-") else "quickstart"
|
|
97
|
+
helper_path = root.parent / quickstart / "scripts/egbe-curl.py"
|
|
98
|
+
if not helper_path.is_file():
|
|
99
|
+
sys.exit("Install the complete Egbe catalog through quickstart before collecting analytics")
|
|
100
|
+
helper = runpy.run_path(str(helper_path))
|
|
101
|
+
settings = Path.cwd() / ".env.local"
|
|
102
|
+
try:
|
|
103
|
+
env = helper["project_environment"](str(settings) if settings.exists() else None)
|
|
104
|
+
base, key = env.get("OS_URL", "").rstrip("/"), env.get("OS_API_KEY", "")
|
|
105
|
+
parsed = urllib.parse.urlsplit(base)
|
|
106
|
+
if not key or not parsed.hostname or parsed.scheme not in {"http", "https"} or parsed.query or parsed.fragment or parsed.username:
|
|
107
|
+
raise ValueError("OS_URL and OS_API_KEY are required")
|
|
108
|
+
args = helper["command"]([base + "/analytics"], env)
|
|
109
|
+
headers = dict(args[i + 1].split(": ", 1) for i, arg in enumerate(args) if arg == "-H")
|
|
110
|
+
headers["Authorization"] = "Bearer " + key
|
|
111
|
+
return base, headers
|
|
112
|
+
except (ValueError, OSError):
|
|
113
|
+
sys.exit("Check OS_URL / OS_API_KEY in .env.local and this agent's quickstart connection identity")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
117
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
118
|
+
return None # Never forward project credentials to a redirected host.
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def call(url, headers):
|
|
122
|
+
req = urllib.request.Request(url, headers=headers)
|
|
123
|
+
try:
|
|
124
|
+
with urllib.request.build_opener(NoRedirect()).open(req, timeout=60) as response:
|
|
125
|
+
result = json.load(response)
|
|
126
|
+
except urllib.error.HTTPError as error:
|
|
127
|
+
raise RuntimeError(f"OS analytics returned HTTP {error.code}; check access, app registration and OS release") from None
|
|
128
|
+
if not isinstance(result, dict) or result.get("code") != 200 or "data" not in result:
|
|
129
|
+
raise ValueError("Invalid OS analytics response envelope")
|
|
130
|
+
return result["data"]
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def event_property_value(prop):
|
|
134
|
+
for key in ("stringValue", "numberValue", "dateValue"):
|
|
135
|
+
if prop.get(key) is not None:
|
|
136
|
+
return prop[key]
|
|
137
|
+
return None
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def ephemeral_ref(kind, value, salt):
|
|
141
|
+
if not isinstance(value, (str, int, float)) or not str(value):
|
|
142
|
+
return None
|
|
143
|
+
digest = hashlib.sha256(
|
|
144
|
+
salt + b"\0" + kind.encode("utf-8") + b"\0" + str(value).encode("utf-8")
|
|
145
|
+
).hexdigest()
|
|
146
|
+
return digest[:12]
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def looks_opaque_token(value):
|
|
150
|
+
"""Distinguish likely secrets/IDs from ordinary long semantic slugs."""
|
|
151
|
+
if not isinstance(value, str):
|
|
152
|
+
return False
|
|
153
|
+
if LONG_HEX_RE.fullmatch(value) or JWT_RE.fullmatch(value):
|
|
154
|
+
return True
|
|
155
|
+
if not BASE64URL_RE.fullmatch(value):
|
|
156
|
+
return False
|
|
157
|
+
chunks = re.split(r"[-_]", value)
|
|
158
|
+
mixed_chunks = [
|
|
159
|
+
chunk for chunk in chunks
|
|
160
|
+
if any(char.isalpha() for char in chunk)
|
|
161
|
+
and any(char.isdigit() for char in chunk)
|
|
162
|
+
]
|
|
163
|
+
secret_prefix = (
|
|
164
|
+
len(chunks) >= 2
|
|
165
|
+
and ((chunks[0] in {"sk", "pk", "rk"} and chunks[1] in {"live", "test", "prod"})
|
|
166
|
+
or chunks[:2] in (["api", "key"], ["access", "token"]))
|
|
167
|
+
)
|
|
168
|
+
if (secret_prefix
|
|
169
|
+
or any(len(chunk) >= 12 for chunk in mixed_chunks)
|
|
170
|
+
or (len(mixed_chunks) >= 2 and sum(map(len, mixed_chunks)) >= 12)):
|
|
171
|
+
return True
|
|
172
|
+
if len(chunks) > 1 and all(
|
|
173
|
+
re.fullmatch(r"[a-z0-9]{1,16}", chunk) for chunk in chunks):
|
|
174
|
+
# Preserve readable kebab/snake-case route and event labels.
|
|
175
|
+
return False
|
|
176
|
+
compact = value.replace("-", "").replace("_", "")
|
|
177
|
+
classes = sum((
|
|
178
|
+
any(char.islower() for char in compact),
|
|
179
|
+
any(char.isupper() for char in compact),
|
|
180
|
+
any(char.isdigit() for char in compact),
|
|
181
|
+
))
|
|
182
|
+
return (classes >= 2 and len(set(compact)) >= 8) or len(set(compact)) >= 16
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def safe_event_value(key, value):
|
|
186
|
+
"""Return a short scalar only when its value has no obvious PII/secret."""
|
|
187
|
+
if isinstance(value, bool):
|
|
188
|
+
return value if key in BOOLEAN_EVENT_FIELDS else None
|
|
189
|
+
if isinstance(value, (int, float)):
|
|
190
|
+
return value if key in NUMERIC_EVENT_FIELDS and (
|
|
191
|
+
isinstance(value, int) or math.isfinite(value)
|
|
192
|
+
) else None
|
|
193
|
+
if not isinstance(value, str):
|
|
194
|
+
return None
|
|
195
|
+
if len(value) > 120 or CONTROL_RE.search(value):
|
|
196
|
+
return None
|
|
197
|
+
inspected = urllib.parse.unquote_plus(value)
|
|
198
|
+
if key in NUMERIC_EVENT_FIELDS:
|
|
199
|
+
return value if re.fullmatch(r"-?\d+(?:\.\d+)?", inspected) else None
|
|
200
|
+
if key in BOOLEAN_EVENT_FIELDS:
|
|
201
|
+
return value if inspected.lower() in {"true", "false"} else None
|
|
202
|
+
if (EMAIL_ANY_RE.search(inspected) or URL_RE.search(inspected)
|
|
203
|
+
or UUID_RE.search(inspected) or JWT_RE.search(inspected)
|
|
204
|
+
or LONG_NUMBER_RE.search(inspected)):
|
|
205
|
+
return None
|
|
206
|
+
for candidate in re.findall(r"[A-Za-z0-9_-]{20,}", inspected):
|
|
207
|
+
if looks_opaque_token(candidate):
|
|
208
|
+
return None
|
|
209
|
+
return value
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def parse_event_properties(properties, identity_salt):
|
|
213
|
+
"""Extract a strict safe allowlist and ephemeral identity references."""
|
|
214
|
+
safe, identity_refs = {}, {}
|
|
215
|
+
for prop in properties if isinstance(properties, list) else []:
|
|
216
|
+
if not isinstance(prop, dict):
|
|
217
|
+
continue
|
|
218
|
+
key = prop.get("dataKey")
|
|
219
|
+
value = event_property_value(prop)
|
|
220
|
+
if key in SAFE_EVENT_FIELDS:
|
|
221
|
+
safe_value = safe_event_value(key, value)
|
|
222
|
+
if safe_value is not None:
|
|
223
|
+
safe[key] = safe_value
|
|
224
|
+
elif key in IDENTITY_EVENT_FIELDS:
|
|
225
|
+
ref = ephemeral_ref(key, value, identity_salt)
|
|
226
|
+
if ref:
|
|
227
|
+
identity_refs[key] = ref
|
|
228
|
+
return safe, identity_refs
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def fetch_event_data(base, app_slug, headers, start, end):
|
|
232
|
+
"""Fetch bulk Umami event data, retaining only safe fields and hashed IDs."""
|
|
233
|
+
safe_by_event, identity_by_event, seen = {}, {}, set()
|
|
234
|
+
identity_salt = os.urandom(16)
|
|
235
|
+
page, expected_count, status, error_type = 1, 0, "available", None
|
|
236
|
+
while page <= MAX_EVENT_PAGES:
|
|
237
|
+
query = urllib.parse.urlencode({
|
|
238
|
+
"startAt": start, "endAt": end, "page": page, "pageSize": PAGE_SIZE,
|
|
239
|
+
})
|
|
240
|
+
try:
|
|
241
|
+
response = call(
|
|
242
|
+
f"{base}/analytics/{app_slug}/event-data?{query}", headers,
|
|
243
|
+
)
|
|
244
|
+
except Exception as error: # Keep base funnel usable when enrichment fails.
|
|
245
|
+
status = "partial" if seen else "unavailable"
|
|
246
|
+
error_type = type(error).__name__
|
|
247
|
+
break
|
|
248
|
+
if not isinstance(response, dict):
|
|
249
|
+
status, error_type = "partial" if seen else "unavailable", "invalid_response"
|
|
250
|
+
break
|
|
251
|
+
entries = response.get("data", [])
|
|
252
|
+
if not isinstance(entries, list):
|
|
253
|
+
status, error_type = "partial" if seen else "unavailable", "invalid_data"
|
|
254
|
+
break
|
|
255
|
+
try:
|
|
256
|
+
expected_count = int(response.get("count", expected_count or 0) or 0)
|
|
257
|
+
except (TypeError, ValueError):
|
|
258
|
+
status, error_type = "partial" if seen else "unavailable", "invalid_count"
|
|
259
|
+
break
|
|
260
|
+
for entry in entries:
|
|
261
|
+
if not isinstance(entry, dict) or not entry.get("eventId"):
|
|
262
|
+
continue
|
|
263
|
+
event_id = str(entry["eventId"])
|
|
264
|
+
safe, identity_refs = parse_event_properties(
|
|
265
|
+
entry.get("eventProperties", []), identity_salt,
|
|
266
|
+
)
|
|
267
|
+
safe_by_event[event_id] = safe
|
|
268
|
+
identity_by_event[event_id] = identity_refs
|
|
269
|
+
seen.add(event_id)
|
|
270
|
+
if not entries:
|
|
271
|
+
if len(seen) < expected_count:
|
|
272
|
+
status = "partial" if seen else "unavailable"
|
|
273
|
+
error_type = "empty_page"
|
|
274
|
+
break
|
|
275
|
+
if len(seen) >= expected_count:
|
|
276
|
+
break
|
|
277
|
+
if len(entries) < PAGE_SIZE:
|
|
278
|
+
status = "partial" if seen else "unavailable"
|
|
279
|
+
error_type = "short_page"
|
|
280
|
+
break
|
|
281
|
+
page += 1
|
|
282
|
+
if page > MAX_EVENT_PAGES and len(seen) < expected_count:
|
|
283
|
+
status, error_type = "partial", "page_limit"
|
|
284
|
+
return safe_by_event, identity_by_event, {
|
|
285
|
+
"status": status,
|
|
286
|
+
"reported_events": expected_count,
|
|
287
|
+
"fetched_events": len(seen),
|
|
288
|
+
"error_type": error_type,
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def attach_event_data(rows, base, app_slug, headers, start, end, skip=False):
|
|
293
|
+
marked = sum(bool(row.get("hasData")) for row in rows)
|
|
294
|
+
if skip:
|
|
295
|
+
return {
|
|
296
|
+
"status": "skipped", "reported_events": None, "fetched_events": 0,
|
|
297
|
+
"marked_rows": marked, "matched_rows": 0, "error_type": None,
|
|
298
|
+
}
|
|
299
|
+
safe_by_event, identity_by_event, status = fetch_event_data(
|
|
300
|
+
base, app_slug, headers, start, end,
|
|
301
|
+
)
|
|
302
|
+
matched = 0
|
|
303
|
+
for row in rows:
|
|
304
|
+
event_id = str(row.get("id") or "")
|
|
305
|
+
if event_id in safe_by_event:
|
|
306
|
+
row["eventData"] = safe_by_event[event_id]
|
|
307
|
+
row["eventIdentityRefs"] = identity_by_event.get(event_id, {})
|
|
308
|
+
row["_eventDataLoaded"] = True
|
|
309
|
+
matched += 1
|
|
310
|
+
elif row.get("hasData"):
|
|
311
|
+
row["_eventDataLoaded"] = False
|
|
312
|
+
return {**status, "marked_rows": marked, "matched_rows": matched}
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def fetch_events(args):
|
|
316
|
+
args.window = getattr(args, "window", None) or resolve_window(args)
|
|
317
|
+
start, end = args.window["startAt"], args.window["endAt"]
|
|
318
|
+
base, headers = os_connection()
|
|
319
|
+
sites = call(f"{base}/analytics", headers)
|
|
320
|
+
if not isinstance(sites, list) or any(not isinstance(s, dict) for s in sites):
|
|
321
|
+
sys.exit("invalid OS analytics site list")
|
|
322
|
+
if args.site:
|
|
323
|
+
query = args.site.casefold()
|
|
324
|
+
exact = [s for s in sites if query in {
|
|
325
|
+
str(s.get(key) or "").casefold() for key in ("websiteId", "appSlug", "domain", "name")
|
|
326
|
+
}]
|
|
327
|
+
sites = exact or [s for s in sites if any(
|
|
328
|
+
query in (s.get(key) or "").casefold() for key in ("domain", "name")
|
|
329
|
+
)]
|
|
330
|
+
if not sites:
|
|
331
|
+
sys.exit("no matching website; check --site or register the product's analytics site")
|
|
332
|
+
if len(sites) != 1:
|
|
333
|
+
sys.exit(f"expected one website, found {len(sites)}; use --site with an exact name/domain/id")
|
|
334
|
+
slug = sites[0].get("appSlug")
|
|
335
|
+
if not isinstance(slug, str) or not re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?", slug):
|
|
336
|
+
sys.exit("analytics site has no valid appSlug; check its OS registration")
|
|
337
|
+
args.website = {"id": sites[0].get("websiteId"), **{key: sites[0].get(key) for key in ("name", "domain")}}
|
|
338
|
+
print(f"# website: {sites[0].get('name')} ({sites[0].get('domain')})")
|
|
339
|
+
rows, page = [], 1
|
|
340
|
+
while True:
|
|
341
|
+
ev = call(f"{base}/analytics/{slug}/events?startAt={start}&endAt={end}&page={page}&pageSize={PAGE_SIZE}", headers)
|
|
342
|
+
if not isinstance(ev, dict) or not isinstance(ev.get("data"), list):
|
|
343
|
+
sys.exit("invalid event page; coverage is unknown")
|
|
344
|
+
data = ev.get("data", [])
|
|
345
|
+
rows.extend(data)
|
|
346
|
+
count = ev.get("count")
|
|
347
|
+
if type(count) is not int or count < 0:
|
|
348
|
+
sys.exit("event response has no valid total count; coverage is unknown")
|
|
349
|
+
if count > MAX_EVENT_PAGES * PAGE_SIZE:
|
|
350
|
+
sys.exit(f"event count {count} exceeds the {MAX_EVENT_PAGES * PAGE_SIZE} event cap; narrow the date window")
|
|
351
|
+
if len(rows) >= count:
|
|
352
|
+
break
|
|
353
|
+
if not data:
|
|
354
|
+
sys.exit(f"empty event page {page}: fetched {len(rows)} of {count}; count/data mismatch, retry or inspect the source")
|
|
355
|
+
if page >= MAX_EVENT_PAGES:
|
|
356
|
+
sys.exit(f"incomplete event download at page cap: {len(rows)} of {count}; inspect pagination or narrow the window")
|
|
357
|
+
page += 1
|
|
358
|
+
event_data_status = attach_event_data(
|
|
359
|
+
rows, base, slug, headers, start, end, skip=args.skip_event_data,
|
|
360
|
+
)
|
|
361
|
+
return rows, event_data_status
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def utc_date_ms(value):
|
|
365
|
+
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
|
|
366
|
+
raise ValueError("expected YYYY-MM-DD")
|
|
367
|
+
return int(dt.datetime.strptime(value, "%Y-%m-%d").replace(
|
|
368
|
+
tzinfo=dt.timezone.utc,
|
|
369
|
+
).timestamp() * 1000)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def resolve_window(args):
|
|
373
|
+
bounds = {}
|
|
374
|
+
for option in ("since", "until"):
|
|
375
|
+
value = getattr(args, option, None)
|
|
376
|
+
try:
|
|
377
|
+
bounds[option] = utc_date_ms(value) if value else None
|
|
378
|
+
except ValueError as error:
|
|
379
|
+
raise ValueError(f"--{option} must be a valid YYYY-MM-DD date (UTC)") from error
|
|
380
|
+
start = bounds["since"] if bounds["since"] is not None else 0
|
|
381
|
+
# Umami's endAt is inclusive; --until is exclusive.
|
|
382
|
+
end = bounds["until"] - 1 if bounds["until"] is not None else int(time.time() * 1000)
|
|
383
|
+
if end < start:
|
|
384
|
+
raise ValueError("--until must be after --since" if bounds["until"] is not None
|
|
385
|
+
else "--since must not be in the future when --until is omitted")
|
|
386
|
+
return {"startAt": start, "endAt": end, "timezone": "UTC",
|
|
387
|
+
"start_utc": dt.datetime.fromtimestamp(start / 1000, dt.timezone.utc).isoformat(),
|
|
388
|
+
"end_inclusive_utc": dt.datetime.fromtimestamp(end / 1000, dt.timezone.utc).isoformat()}
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def first_touch_source(evs):
|
|
392
|
+
q = urllib.parse.parse_qs(evs[0].get("urlQuery") or "")
|
|
393
|
+
utm = (q.get("utm_source") or [None])[0]
|
|
394
|
+
return utm or (evs[0].get("referrerDomain") or "direct")
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def internal_reason(evs, src, args):
|
|
398
|
+
# A smoke tag may first appear after an initial pageview; keep the whole
|
|
399
|
+
# dedicated audit session out even when emulating a mobile device.
|
|
400
|
+
sources = {src}
|
|
401
|
+
for event in evs:
|
|
402
|
+
query = urllib.parse.parse_qs(event.get("urlQuery") or "")
|
|
403
|
+
sources.update(query.get("utm_source", []))
|
|
404
|
+
matched = sources.intersection(args.internal_sources)
|
|
405
|
+
if matched:
|
|
406
|
+
return "source:" + sorted(matched)[0]
|
|
407
|
+
country = evs[0].get("country")
|
|
408
|
+
return f"verified_country:{country}" if country in args.internal_country else None
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def is_suspected_internal(evs, src, args):
|
|
412
|
+
"""Heuristics are sensitivity flags, never automatic exclusions."""
|
|
413
|
+
f0 = evs[0]
|
|
414
|
+
if (src == "direct" and f0.get("device") in ("laptop", "desktop")
|
|
415
|
+
and f0.get("country") in args.dc_country):
|
|
416
|
+
return True
|
|
417
|
+
for e in evs:
|
|
418
|
+
p = e.get("urlPath") or ""
|
|
419
|
+
if e.get("eventType") == 1 and ("/admin" in p or "/tmp/" in p or "localhost" in p):
|
|
420
|
+
return True
|
|
421
|
+
return False
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def safe_path(path):
|
|
425
|
+
"""Redact common identifier/token path segments before output or storage."""
|
|
426
|
+
if not isinstance(path, str):
|
|
427
|
+
return path
|
|
428
|
+
path = path.split("?", 1)[0].split("#", 1)[0]
|
|
429
|
+
parts, redacted, redact_next = path.split("/"), [], False
|
|
430
|
+
for raw in parts:
|
|
431
|
+
decoded = urllib.parse.unquote(raw)
|
|
432
|
+
lowered = decoded.lower()
|
|
433
|
+
sensitive = (
|
|
434
|
+
redact_next
|
|
435
|
+
or safe_event_value("surface", decoded) is None
|
|
436
|
+
or bool(UUID_RE.fullmatch(decoded))
|
|
437
|
+
or bool(JWT_RE.fullmatch(decoded))
|
|
438
|
+
or looks_opaque_token(decoded)
|
|
439
|
+
)
|
|
440
|
+
redacted.append(":redacted" if sensitive else raw)
|
|
441
|
+
redact_next = lowered in SENSITIVE_PATH_PARENTS
|
|
442
|
+
return "/".join(redacted)
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def safe_label(value):
|
|
446
|
+
"""Keep categorical labels, not arbitrary URLs/free-form text."""
|
|
447
|
+
if not isinstance(value, str) or not re.fullmatch(r"[\w .:/-]{1,120}", value):
|
|
448
|
+
return None
|
|
449
|
+
return safe_event_value("surface", value)
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def safe_route_path(path):
|
|
453
|
+
"""Preserve hash-router/anchor semantics for stage detection after redaction."""
|
|
454
|
+
before, marker, fragment = path.partition("#")
|
|
455
|
+
base = safe_path(before)
|
|
456
|
+
if not marker:
|
|
457
|
+
return base
|
|
458
|
+
safe_fragment = safe_path(fragment) if fragment.startswith("/") else safe_label(fragment)
|
|
459
|
+
return base + "#" + (safe_fragment or ":redacted")
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def snapshot_rows(rows):
|
|
463
|
+
"""Retain uncapped safe input rows; pseudonymous joins last for this snapshot."""
|
|
464
|
+
salt = os.urandom(16)
|
|
465
|
+
result = []
|
|
466
|
+
for row in rows:
|
|
467
|
+
dt.datetime.fromisoformat(row["createdAt"].replace("Z", "+00:00"))
|
|
468
|
+
if type(row["eventType"]) is not int or row["eventType"] < 0:
|
|
469
|
+
raise ValueError("Umami event type must be a non-negative integer")
|
|
470
|
+
session_ref = ephemeral_ref("session_id", row["sessionId"], salt)
|
|
471
|
+
path = row.get("urlPath") or ""
|
|
472
|
+
query = urllib.parse.parse_qs(row.get("urlQuery") or "")
|
|
473
|
+
attribution = {key: [safe_label(value) or "redacted" for value in query[key]]
|
|
474
|
+
for key in SAFE_UTM_FIELDS if key in query}
|
|
475
|
+
# Retain distinct-page counting even when different private paths redact alike.
|
|
476
|
+
page_ref = ephemeral_ref("page", path.split("#")[0].rstrip("/") or "/", salt)
|
|
477
|
+
item = {
|
|
478
|
+
"id": ephemeral_ref("event", row.get("id"), salt),
|
|
479
|
+
"sessionId": session_ref, "_sessionRef": session_ref,
|
|
480
|
+
"createdAt": row["createdAt"], "eventType": row["eventType"],
|
|
481
|
+
"urlPath": safe_route_path(path), "_pageRef": page_ref,
|
|
482
|
+
"urlQuery": urllib.parse.urlencode(attribution, doseq=True),
|
|
483
|
+
"referrerDomain": (safe_label(row["referrerDomain"]) or "redacted") if row.get("referrerDomain") else "",
|
|
484
|
+
"country": row.get("country") if re.fullmatch(r"[A-Z]{2}", row.get("country") or "") else None,
|
|
485
|
+
"device": row.get("device") if row.get("device") in {"mobile", "tablet", "laptop", "desktop"} else None,
|
|
486
|
+
}
|
|
487
|
+
if row.get("eventName"):
|
|
488
|
+
item["eventName"] = safe_label(row["eventName"]) or "redacted"
|
|
489
|
+
if row.get("hasData"):
|
|
490
|
+
item["hasData"] = True
|
|
491
|
+
if row.get("_eventDataLoaded") or "eventData" in row:
|
|
492
|
+
item["_eventDataLoaded"] = True
|
|
493
|
+
item["eventData"] = safe_event_props(row)
|
|
494
|
+
refs = event_identity_refs(row, salt)
|
|
495
|
+
# Even pre-enriched input refs are re-keyed together for this export.
|
|
496
|
+
item["eventIdentityRefs"] = {
|
|
497
|
+
key: ephemeral_ref(key, value, salt) for key, value in refs.items()
|
|
498
|
+
}
|
|
499
|
+
result.append(item)
|
|
500
|
+
return result
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def make_snapshot(rows, args, event_data_status):
|
|
504
|
+
return {
|
|
505
|
+
"format": SNAPSHOT_FORMAT,
|
|
506
|
+
"retrieved_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
507
|
+
"website": args.website, "window": args.window,
|
|
508
|
+
"analysis_options": {key: getattr(args, key) for key in ANALYSIS_OPTIONS},
|
|
509
|
+
"coverage": {"base_events": "complete", "downloaded_rows": len(rows),
|
|
510
|
+
"non_funnel_event_types": dict(collections.Counter(
|
|
511
|
+
str(row["eventType"]) for row in rows if row["eventType"] not in (1, 2))),
|
|
512
|
+
"event_data": event_data_status},
|
|
513
|
+
"redaction": "Safe categorical fields/properties and UTM source/medium/campaign/content only; "
|
|
514
|
+
"identities and distinct paths use per-snapshot pseudonyms. "
|
|
515
|
+
"Private path segments and unsupported properties cannot be reconstructed.",
|
|
516
|
+
"rows": snapshot_rows(rows),
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def read_snapshot(path, args):
|
|
521
|
+
with open(path, encoding="utf-8") as source:
|
|
522
|
+
snapshot = json.load(source)
|
|
523
|
+
if snapshot.get("format") != SNAPSHOT_FORMAT or not isinstance(snapshot.get("rows"), list):
|
|
524
|
+
raise ValueError("expected an egbe-umami-events.v1 snapshot, not a summary")
|
|
525
|
+
for key in ANALYSIS_OPTIONS:
|
|
526
|
+
setattr(args, key, snapshot["analysis_options"][key])
|
|
527
|
+
args.window, args.website = snapshot["window"], snapshot["website"]
|
|
528
|
+
return snapshot["rows"], snapshot["coverage"]["event_data"]
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def check_output_paths(*paths):
|
|
532
|
+
selected = [Path(path) for path in paths if path]
|
|
533
|
+
if len({path.resolve() for path in selected}) != len(selected):
|
|
534
|
+
raise ValueError("source and summary outputs must have different paths")
|
|
535
|
+
for path in selected:
|
|
536
|
+
if os.path.lexists(path):
|
|
537
|
+
raise ValueError(f"output already exists; preserve it and choose a new filename: {path}")
|
|
538
|
+
if not path.parent.is_dir():
|
|
539
|
+
raise ValueError(f"create the output directory first: {path.parent}")
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def write_json(path, value):
|
|
543
|
+
payload = json.dumps(value, indent=1, allow_nan=False)
|
|
544
|
+
with open(path, "x", encoding="utf-8") as output:
|
|
545
|
+
output.write(payload + "\n")
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def safe_event_props(event):
|
|
549
|
+
"""Return only explicitly allowlisted, short scalar event properties."""
|
|
550
|
+
value = event.get("eventData")
|
|
551
|
+
if isinstance(value, str) and value.startswith("{") and len(value) <= 5000:
|
|
552
|
+
try:
|
|
553
|
+
value = json.loads(value)
|
|
554
|
+
except json.JSONDecodeError:
|
|
555
|
+
value = None
|
|
556
|
+
if not isinstance(value, dict):
|
|
557
|
+
return {}
|
|
558
|
+
props = {}
|
|
559
|
+
for key in SAFE_EVENT_FIELDS:
|
|
560
|
+
prop_value = safe_event_value(key, value.get(key))
|
|
561
|
+
if prop_value is not None:
|
|
562
|
+
props[key] = prop_value
|
|
563
|
+
return props
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def event_identity_refs(event, salt):
|
|
567
|
+
existing = event.get("eventIdentityRefs")
|
|
568
|
+
if isinstance(existing, dict):
|
|
569
|
+
return {key: value for key, value in existing.items()
|
|
570
|
+
if key in IDENTITY_EVENT_FIELDS and isinstance(value, str)}
|
|
571
|
+
payload = event.get("eventData")
|
|
572
|
+
if not isinstance(payload, dict):
|
|
573
|
+
return {}
|
|
574
|
+
refs = {}
|
|
575
|
+
for key in IDENTITY_EVENT_FIELDS:
|
|
576
|
+
ref = ephemeral_ref(key, payload.get(key), salt)
|
|
577
|
+
if ref:
|
|
578
|
+
refs[key] = ref
|
|
579
|
+
return refs
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def build_timeline(evs, max_entries):
|
|
583
|
+
"""Build a redacted timeline, collapsing only adjacent identical pageviews."""
|
|
584
|
+
display, structured = [], []
|
|
585
|
+
eligible_count, last_pageview = 0, None
|
|
586
|
+
for event in evs:
|
|
587
|
+
if event["eventType"] == 1:
|
|
588
|
+
raw_path = event.get("urlPath")
|
|
589
|
+
if not raw_path or raw_path.startswith(ASSET_PATH_PREFIXES):
|
|
590
|
+
continue
|
|
591
|
+
token = safe_path(raw_path)
|
|
592
|
+
if token == last_pageview:
|
|
593
|
+
continue
|
|
594
|
+
last_pageview = token
|
|
595
|
+
item = {"at": event["createdAt"], "type": "pageview", "path": token}
|
|
596
|
+
elif event["eventType"] == 2:
|
|
597
|
+
last_pageview = None
|
|
598
|
+
name = event.get("eventName")
|
|
599
|
+
if not name:
|
|
600
|
+
continue
|
|
601
|
+
props = safe_event_props(event)
|
|
602
|
+
has_data = bool(event.get("hasData") or "eventData" in event)
|
|
603
|
+
data_loaded = bool(event.get("_eventDataLoaded") or "eventData" in event)
|
|
604
|
+
suffix = " " + json.dumps(props, sort_keys=True) if props else ""
|
|
605
|
+
if has_data and not data_loaded:
|
|
606
|
+
suffix += " {event_data:unavailable}"
|
|
607
|
+
token = f"[{name}]{suffix}"
|
|
608
|
+
item = {"at": event["createdAt"], "type": "event", "name": name}
|
|
609
|
+
if props:
|
|
610
|
+
item["properties"] = props
|
|
611
|
+
if has_data:
|
|
612
|
+
item["has_data"] = True
|
|
613
|
+
item["event_data_loaded"] = data_loaded
|
|
614
|
+
else:
|
|
615
|
+
continue
|
|
616
|
+
eligible_count += 1
|
|
617
|
+
if len(structured) >= max_entries:
|
|
618
|
+
continue
|
|
619
|
+
display.append(f"{event['createdAt'][11:19]} {token}")
|
|
620
|
+
structured.append(item)
|
|
621
|
+
return display, structured, eligible_count
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
def analyze(rows, args, event_data_status=None):
|
|
625
|
+
rows.sort(key=lambda r: r["createdAt"])
|
|
626
|
+
sessions = collections.defaultdict(list)
|
|
627
|
+
for r in rows:
|
|
628
|
+
sessions[r["sessionId"]].append(r)
|
|
629
|
+
|
|
630
|
+
included, internal, suspected = {}, {}, set()
|
|
631
|
+
exclusion_reasons = collections.Counter()
|
|
632
|
+
for sid, evs in sessions.items():
|
|
633
|
+
src = first_touch_source(evs)
|
|
634
|
+
reason = internal_reason(evs, src, args)
|
|
635
|
+
(internal if reason else included)[sid] = (src, evs)
|
|
636
|
+
if reason:
|
|
637
|
+
exclusion_reasons[reason] += 1
|
|
638
|
+
if sid in included and is_suspected_internal(evs, src, args):
|
|
639
|
+
suspected.add(sid)
|
|
640
|
+
|
|
641
|
+
funnel = collections.Counter()
|
|
642
|
+
suspect_funnel = collections.Counter()
|
|
643
|
+
mix = collections.Counter()
|
|
644
|
+
arrivals_by_source = collections.Counter()
|
|
645
|
+
depth = collections.Counter()
|
|
646
|
+
paths = collections.defaultdict(set)
|
|
647
|
+
events = collections.defaultdict(set)
|
|
648
|
+
event_occurrences = collections.Counter()
|
|
649
|
+
interesting = []
|
|
650
|
+
journey_rows = []
|
|
651
|
+
repeat_completions = {}
|
|
652
|
+
identity_salt = os.urandom(16)
|
|
653
|
+
start_names = TECHNICAL_START_EVENTS | set(args.act_start_event)
|
|
654
|
+
completion_names = TECHNICAL_DONE_EVENTS | set(args.act_event)
|
|
655
|
+
run = {
|
|
656
|
+
"window": getattr(args, "window", None), "website": getattr(args, "website", None),
|
|
657
|
+
"config": {"execution_start_events": sorted(start_names),
|
|
658
|
+
"execution_completion_events": sorted(completion_names),
|
|
659
|
+
"execution_start_paths": args.act_path,
|
|
660
|
+
"internal_sources": args.internal_sources,
|
|
661
|
+
"internal_countries": args.internal_country,
|
|
662
|
+
"suspect_countries": args.dc_country,
|
|
663
|
+
"ambient_events": sorted(ENGAGE_EVENTS | EXPOSURE_EVENTS),
|
|
664
|
+
"max_journey_events": args.max_journey_events},
|
|
665
|
+
}
|
|
666
|
+
print("# run: " + json.dumps(run, sort_keys=True))
|
|
667
|
+
if event_data_status is None:
|
|
668
|
+
marked = sum(bool(row.get("hasData")) for row in rows)
|
|
669
|
+
loaded = sum(bool(row.get("_eventDataLoaded") or "eventData" in row) for row in rows)
|
|
670
|
+
event_data_status = {
|
|
671
|
+
"status": "provided" if loaded else "not_fetched",
|
|
672
|
+
"reported_events": None,
|
|
673
|
+
"fetched_events": loaded,
|
|
674
|
+
"marked_rows": marked,
|
|
675
|
+
"matched_rows": loaded,
|
|
676
|
+
"error_type": None,
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
for sid, (src, evs) in included.items():
|
|
680
|
+
session_ref = evs[0].get("_sessionRef") or ephemeral_ref("session_id", sid, identity_salt)
|
|
681
|
+
mix[src] += 1
|
|
682
|
+
pv = [e for e in evs if e["eventType"] == 1 and e.get("urlPath")
|
|
683
|
+
and not e["urlPath"].startswith(ASSET_PATH_PREFIXES)]
|
|
684
|
+
event_names = [e.get("eventName") for e in evs
|
|
685
|
+
if e["eventType"] == 2 and e.get("eventName")]
|
|
686
|
+
evn_counts = collections.Counter(event_names)
|
|
687
|
+
evn = set(event_names)
|
|
688
|
+
for e in pv:
|
|
689
|
+
paths[safe_path(e["urlPath"])].add(sid)
|
|
690
|
+
for n, count in evn_counts.items():
|
|
691
|
+
events[n].add(sid)
|
|
692
|
+
event_occurrences[n] += count
|
|
693
|
+
npv = len(pv)
|
|
694
|
+
depth_bucket = f"{npv}pv" if npv <= 1 else ("2-3pv" if npv <= 3 else "4+pv")
|
|
695
|
+
depth[depth_bucket] += 1
|
|
696
|
+
pset = {p["urlPath"] for p in pv}
|
|
697
|
+
base_pages = {p.get("_pageRef") or p["urlPath"].split("#")[0].rstrip("/") or "/" for p in pv}
|
|
698
|
+
# Activity proxy, not proven attention: exposures may fire on load.
|
|
699
|
+
activity_events = evn - ENGAGE_EVENTS - EXPOSURE_EVENTS
|
|
700
|
+
scroll_dwell = {"engagement-scroll", "engagement-time"} <= evn
|
|
701
|
+
engaged = len(base_pages) > 1 or bool(activity_events) or scroll_dwell
|
|
702
|
+
completion_count = sum(evn_counts[name] for name in completion_names)
|
|
703
|
+
execution_started = bool(start_names & evn) or any(
|
|
704
|
+
p.startswith(tuple(args.act_path)) for p in pset if args.act_path)
|
|
705
|
+
execution_completed = completion_count > 0
|
|
706
|
+
f = collections.Counter()
|
|
707
|
+
if pv:
|
|
708
|
+
f["arrival"] += 1
|
|
709
|
+
arrivals_by_source[src] += 1
|
|
710
|
+
if not engaged: f["unengaged_arrivals"] += 1
|
|
711
|
+
if engaged: f["engaged"] += 1
|
|
712
|
+
if "cta_click" in evn: f["cta_click"] += 1
|
|
713
|
+
if "button_click" in evn: f["button_click"] += 1
|
|
714
|
+
if execution_started: f["execution_started"] += 1
|
|
715
|
+
if execution_completed: f["execution_completed*"] += 1
|
|
716
|
+
if completion_count > 1:
|
|
717
|
+
repeat_completions[session_ref] = completion_count
|
|
718
|
+
if any(p.startswith(SIGNUP_PATHS) for p in pset):
|
|
719
|
+
f["signup_page_seen"] += 1
|
|
720
|
+
if SIGNUP_START_EVENTS & evn:
|
|
721
|
+
f["signup_started"] += 1
|
|
722
|
+
if SIGNUP_DONE_EVENTS & evn:
|
|
723
|
+
f["signup_completed*"] += 1 # * reconcile vs ground truth
|
|
724
|
+
if any(p.startswith(POST_AUTH_PATHS) for p in pset):
|
|
725
|
+
f["post_auth_seen"] += 1
|
|
726
|
+
if PAYWALL_EVENTS & evn or any(p.startswith(PRICING_PATHS) for p in pset):
|
|
727
|
+
f["pricing_seen"] += 1
|
|
728
|
+
if CHECKOUT_EVENTS & evn or any(p.startswith(CHECKOUT_PATHS) for p in pset):
|
|
729
|
+
f["checkout_started"] += 1
|
|
730
|
+
funnel.update(f)
|
|
731
|
+
if sid in suspected:
|
|
732
|
+
suspect_funnel.update(f)
|
|
733
|
+
# Journey list stays focused on sessions with something to read:
|
|
734
|
+
# Keep exposure timelines for reach analysis even when not engaged.
|
|
735
|
+
# scroll+dwell-only sessions have no journey beyond beacons.
|
|
736
|
+
beyond = (len(base_pages) > 1 or execution_started or execution_completed
|
|
737
|
+
or evn - ENGAGE_EVENTS)
|
|
738
|
+
if beyond:
|
|
739
|
+
tl, structured_timeline, timeline_items_total = build_timeline(
|
|
740
|
+
evs, args.max_journey_events,
|
|
741
|
+
)
|
|
742
|
+
session_identity_refs = {"visitor": set(), "visit": set()}
|
|
743
|
+
for event in evs:
|
|
744
|
+
refs = event_identity_refs(event, identity_salt)
|
|
745
|
+
if refs.get("visitor_id"):
|
|
746
|
+
session_identity_refs["visitor"].add(refs["visitor_id"])
|
|
747
|
+
if refs.get("visit_id"):
|
|
748
|
+
session_identity_refs["visit"].add(refs["visit_id"])
|
|
749
|
+
anonymized_refs = {
|
|
750
|
+
key: sorted(value) for key, value in session_identity_refs.items() if value
|
|
751
|
+
}
|
|
752
|
+
f0 = evs[0]
|
|
753
|
+
identity_suffix = f" refs={anonymized_refs}" if anonymized_refs else ""
|
|
754
|
+
truncated = timeline_items_total > args.max_journey_events
|
|
755
|
+
truncation_suffix = (
|
|
756
|
+
f" | … {timeline_items_total - args.max_journey_events} more"
|
|
757
|
+
if truncated else ""
|
|
758
|
+
)
|
|
759
|
+
interesting.append(
|
|
760
|
+
f"{session_ref} {f0['createdAt'][:10]} {f0.get('country')}/{f0.get('device')} "
|
|
761
|
+
f"src={src}{' [suspect]' if sid in suspected else ''}{identity_suffix}\n "
|
|
762
|
+
+ " | ".join(tl) + truncation_suffix)
|
|
763
|
+
journey_rows.append({
|
|
764
|
+
"session_ref": session_ref,
|
|
765
|
+
"date": f0["createdAt"][:10],
|
|
766
|
+
"country": f0.get("country"),
|
|
767
|
+
"device": f0.get("device"),
|
|
768
|
+
"source": src,
|
|
769
|
+
"suspected_internal": sid in suspected,
|
|
770
|
+
"execution_completion_event_count": completion_count,
|
|
771
|
+
"events_total": len(evs),
|
|
772
|
+
"timeline_items_total": timeline_items_total,
|
|
773
|
+
"timeline_truncated": truncated,
|
|
774
|
+
"identity_refs": anonymized_refs,
|
|
775
|
+
"timeline": structured_timeline,
|
|
776
|
+
})
|
|
777
|
+
|
|
778
|
+
n = max(len(included), 1)
|
|
779
|
+
print(f"\n== internal filter: sources={args.internal_sources} "
|
|
780
|
+
f"verified-internal-country={args.internal_country or 'NONE'} "
|
|
781
|
+
f"suspect-dc-laptop-direct={args.dc_country}")
|
|
782
|
+
print(f"== sessions: included={len(included)} internal={len(internal)} "
|
|
783
|
+
f"suspected_included={len(suspected)} "
|
|
784
|
+
f"excluded_by_reason={dict(exclusion_reasons)}")
|
|
785
|
+
print(" included = not excluded by configured rules; not verified humans/customers")
|
|
786
|
+
print(f"event data: {event_data_status}")
|
|
787
|
+
print(f"traffic (first touch): {dict(mix.most_common(10))}")
|
|
788
|
+
print(f"observed arrivals by source (suspects included): {dict(arrivals_by_source)}")
|
|
789
|
+
arrivals = funnel["arrival"]
|
|
790
|
+
bounce = 100 * funnel["unengaged_arrivals"] / arrivals if arrivals else None
|
|
791
|
+
print(f"depth: {dict(depth)} | bounce(unengaged/arrivals)="
|
|
792
|
+
+ (f"{bounce:.1f}%" if bounce is not None else "unknown (no observed arrivals)"))
|
|
793
|
+
without_suspects = {key: value - suspect_funnel[key] for key, value in funnel.items()}
|
|
794
|
+
if suspected:
|
|
795
|
+
print(f"sensitivity excluding suspects: sessions={len(included) - len(suspected)} "
|
|
796
|
+
f"funnel={without_suspects}")
|
|
797
|
+
print(f"top paths: { {k: len(v) for k, v in sorted(paths.items(), key=lambda kv: -len(kv[1]))[:12]} }")
|
|
798
|
+
print(f"custom events (uniq sessions / raw events): "
|
|
799
|
+
f"{ {k: f'{len(v)}/{event_occurrences[k]}' for k, v in sorted(events.items(), key=lambda kv: -len(kv[1]))[:15]} }")
|
|
800
|
+
print("\n== stage observations (% of non-excluded sessions; not ordered conversions)")
|
|
801
|
+
order = ["arrival", "engaged", "unengaged_arrivals", "cta_click", "button_click", "execution_started", "execution_completed*",
|
|
802
|
+
"signup_page_seen", "signup_started", "signup_completed*", "post_auth_seen", "pricing_seen", "checkout_started"]
|
|
803
|
+
for k in order:
|
|
804
|
+
print(f" {k:22s} {funnel.get(k, 0):6d} ({funnel.get(k, 0) * 100 / n:.1f}%)")
|
|
805
|
+
print(" * execution/signup completion events are technical proxies; "
|
|
806
|
+
"reconcile with domain outcome and ground truth")
|
|
807
|
+
print(" engaged = another distinct page, non-ambient custom event, or scroll+time "
|
|
808
|
+
"beacons; activity proxy, not purchase intent")
|
|
809
|
+
print(f"repeat execution-completion sessions: {repeat_completions or '{}'}")
|
|
810
|
+
print("\n== activity/exposure journeys (included sessions):")
|
|
811
|
+
for line in interesting:
|
|
812
|
+
print(" " + line)
|
|
813
|
+
if args.json:
|
|
814
|
+
out = {**run, "included_sessions": len(included), "internal": len(internal), "mix": dict(mix),
|
|
815
|
+
"non_funnel_event_types": dict(collections.Counter(
|
|
816
|
+
str(row["eventType"]) for row in rows if row["eventType"] not in (1, 2))),
|
|
817
|
+
"exclusion_reasons": dict(exclusion_reasons),
|
|
818
|
+
"arrivals_by_source": dict(arrivals_by_source),
|
|
819
|
+
"suspected_internal": len(suspected),
|
|
820
|
+
"funnel_excluding_suspected": without_suspects,
|
|
821
|
+
"bounce_percent": bounce,
|
|
822
|
+
"depth": dict(depth), "funnel": dict(funnel),
|
|
823
|
+
"paths": {k: len(v) for k, v in paths.items()},
|
|
824
|
+
"events": {k: {"unique_sessions": len(v),
|
|
825
|
+
"raw_events": event_occurrences[k]}
|
|
826
|
+
for k, v in events.items()},
|
|
827
|
+
"event_data": event_data_status,
|
|
828
|
+
"metric_notes": {
|
|
829
|
+
"execution_completed*": {
|
|
830
|
+
"kind": "technical_proxy",
|
|
831
|
+
"reconcile_with": "domain_outcome",
|
|
832
|
+
},
|
|
833
|
+
"signup_completed*": {
|
|
834
|
+
"kind": "completion_event_proxy",
|
|
835
|
+
"reconcile_with": "signup_ground_truth",
|
|
836
|
+
},
|
|
837
|
+
"signup_started": {"kind": "intent_event_requires_verified_wiring"},
|
|
838
|
+
"cta_click": {"kind": "canonical_cta_event_only"},
|
|
839
|
+
"button_click": {"kind": "secondary_interaction_requires_property_filter"},
|
|
840
|
+
"signup_page_seen": {"kind": "page_visit_not_intent"},
|
|
841
|
+
"engaged": {
|
|
842
|
+
"kind": "arrival_with_activity_requires_verified_wiring",
|
|
843
|
+
"rule": "distinct_pages>1 OR non_ambient_event OR scroll_and_time_beacons",
|
|
844
|
+
"ambient_events": sorted(ENGAGE_EVENTS | EXPOSURE_EVENTS),
|
|
845
|
+
"commercial_intent_verified": False,
|
|
846
|
+
},
|
|
847
|
+
"post_auth_seen": {"kind": "page_visit_not_signup"},
|
|
848
|
+
"arrival": {"kind": "session_with_non_asset_pageview"},
|
|
849
|
+
"bounce_percent": {"denominator": "arrival", "numerator": "unengaged_arrivals"},
|
|
850
|
+
"included_sessions": {
|
|
851
|
+
"kind": "not_excluded_by_configured_rules",
|
|
852
|
+
"suspected_internal_included": True,
|
|
853
|
+
"human_or_customer_verified": False,
|
|
854
|
+
},
|
|
855
|
+
"journeys.timeline.path": {"redacted": True},
|
|
856
|
+
"journeys.session_ref": {"ephemeral_per_run": True},
|
|
857
|
+
"journeys.identity_refs": {"ephemeral_per_run": True},
|
|
858
|
+
},
|
|
859
|
+
"repeat_execution_completion_sessions": repeat_completions,
|
|
860
|
+
"journeys": journey_rows}
|
|
861
|
+
write_json(args.json, out)
|
|
862
|
+
print(f"\n(json written to {args.json})")
|
|
863
|
+
|
|
864
|
+
|
|
865
|
+
def main():
|
|
866
|
+
ap = argparse.ArgumentParser()
|
|
867
|
+
ap.add_argument("--site", help="exact website name/domain/id, otherwise a unique name/domain substring")
|
|
868
|
+
ap.add_argument("--since", help="inclusive YYYY-MM-DD in UTC (default: all time)")
|
|
869
|
+
ap.add_argument("--until", help="exclusive YYYY-MM-DD in UTC (default: now)")
|
|
870
|
+
ap.add_argument("--act-path", action="append", default=[],
|
|
871
|
+
help="path prefix that counts as execution start (repeatable)")
|
|
872
|
+
ap.add_argument("--act-start-event", action="append", default=[],
|
|
873
|
+
dest="act_start_event",
|
|
874
|
+
help="event name that counts as execution start (repeatable)")
|
|
875
|
+
ap.add_argument("--act-event", action="append", default=[],
|
|
876
|
+
help="event name that counts as technical execution completion (repeatable)")
|
|
877
|
+
ap.add_argument("--internal-country", action="append", default=[],
|
|
878
|
+
dest="internal_country",
|
|
879
|
+
help="verified internal geography to exclude; no default. "
|
|
880
|
+
"Do not infer internal traffic from country alone")
|
|
881
|
+
ap.add_argument("--dc-country", action="extend", nargs="*",
|
|
882
|
+
dest="dc_country", help="suspect countries (default DE FI); empty flag clears them")
|
|
883
|
+
ap.add_argument("--internal-sources", action="extend", nargs="*",
|
|
884
|
+
dest="internal_sources", help="synthetic sources (default smoke); empty flag clears them")
|
|
885
|
+
ap.add_argument("--skip-event-data", action="store_true",
|
|
886
|
+
help="skip Umami event-data enrichment and report it skipped")
|
|
887
|
+
ap.add_argument("--max-journey-events", type=int, default=100,
|
|
888
|
+
help="maximum filtered timeline entries printed/stored per journey (default: 100)")
|
|
889
|
+
ap.add_argument("--json", help="also write summary json here")
|
|
890
|
+
ap.add_argument("--events-json", help="save uncapped sanitized input rows for offline replay")
|
|
891
|
+
ap.add_argument("--from-events", help="replay a saved event snapshot/config without network access")
|
|
892
|
+
args = ap.parse_args()
|
|
893
|
+
args.dc_country = args.dc_country if args.dc_country is not None else ["DE", "FI"]
|
|
894
|
+
args.internal_sources = args.internal_sources if args.internal_sources is not None else ["smoke"]
|
|
895
|
+
if args.max_journey_events < 1:
|
|
896
|
+
ap.error("--max-journey-events must be at least 1")
|
|
897
|
+
try:
|
|
898
|
+
check_output_paths(args.json, args.events_json)
|
|
899
|
+
if args.from_events:
|
|
900
|
+
live_options = {
|
|
901
|
+
"--site", "--since", "--until", "--act-path", "--act-start-event",
|
|
902
|
+
"--act-event", "--internal-country", "--dc-country", "--internal-sources",
|
|
903
|
+
"--skip-event-data", "--max-journey-events", "--events-json",
|
|
904
|
+
}
|
|
905
|
+
if any(arg.split("=", 1)[0] in live_options for arg in sys.argv[1:]):
|
|
906
|
+
ap.error("--from-events uses the saved config; only --json can be added")
|
|
907
|
+
rows, event_data_status = read_snapshot(args.from_events, args)
|
|
908
|
+
analyze(rows, args, event_data_status)
|
|
909
|
+
return
|
|
910
|
+
args.window = resolve_window(args)
|
|
911
|
+
except (ValueError, OSError, KeyError) as error:
|
|
912
|
+
ap.error(str(error))
|
|
913
|
+
try:
|
|
914
|
+
rows, event_data_status = fetch_events(args)
|
|
915
|
+
except (ValueError, OSError, RuntimeError) as error:
|
|
916
|
+
ap.error(str(error))
|
|
917
|
+
print(f"# events fetched: {len(rows)}")
|
|
918
|
+
# Analyze exactly the safe dataset we retain; replay reproduces its joins too.
|
|
919
|
+
if args.events_json:
|
|
920
|
+
snapshot = make_snapshot(rows, args, event_data_status)
|
|
921
|
+
write_json(args.events_json, snapshot)
|
|
922
|
+
rows = snapshot["rows"]
|
|
923
|
+
analyze(rows, args, event_data_status)
|
|
924
|
+
|
|
925
|
+
|
|
926
|
+
if __name__ == "__main__":
|
|
927
|
+
main()
|