@egbe-ai/opencode 0.0.0-stage → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (479) hide show
  1. package/LICENSE +4 -0
  2. package/README.md +124 -2
  3. package/index.js +5 -0
  4. package/package.json +55 -4
  5. package/plugin.js +230 -0
  6. package/session-log.js +315 -0
  7. package/skills/.egbe-catalog/manifest.json +1 -0
  8. package/skills/egbe-os-analytics/SKILL.md +499 -0
  9. package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
  10. package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
  11. package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
  12. package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
  13. package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
  14. package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
  15. package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
  16. package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
  17. package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
  18. package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
  19. package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
  20. package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
  21. package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
  22. package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
  23. package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
  24. package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
  25. package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
  26. package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
  27. package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
  28. package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
  29. package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
  30. package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
  31. package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
  32. package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
  33. package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
  34. package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
  35. package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
  36. package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
  37. package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
  38. package/skills/egbe-os-db/SKILL.md +174 -0
  39. package/skills/egbe-os-deploy/SKILL.md +412 -0
  40. package/skills/egbe-os-design/SKILL.md +182 -0
  41. package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
  42. package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
  43. package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
  44. package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
  45. package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
  46. package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
  47. package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
  48. package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
  49. package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
  50. package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
  51. package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
  52. package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
  53. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
  54. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
  55. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
  56. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
  57. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
  58. package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
  59. package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
  60. package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
  61. package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
  62. package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
  63. package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
  64. package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
  65. package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
  66. package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
  67. package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
  68. package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
  69. package/skills/egbe-os-destination-readiness/references/report.md +116 -0
  70. package/skills/egbe-os-domains/SKILL.md +189 -0
  71. package/skills/egbe-os-errors/SKILL.md +259 -0
  72. package/skills/egbe-os-git/SKILL.md +275 -0
  73. package/skills/egbe-os-git/references/issues.md +33 -0
  74. package/skills/egbe-os-google-ads/SKILL.md +163 -0
  75. package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
  76. package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
  77. package/skills/egbe-os-google-ads/references/conversions.md +153 -0
  78. package/skills/egbe-os-google-ads/references/delivery.md +128 -0
  79. package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
  80. package/skills/egbe-os-google-ads/references/reporting.md +105 -0
  81. package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
  82. package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
  83. package/skills/egbe-os-google-auth/SKILL.md +285 -0
  84. package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
  85. package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
  86. package/skills/egbe-os-images/SKILL.md +163 -0
  87. package/skills/egbe-os-images/references/stock-images.md +21 -0
  88. package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
  89. package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
  90. package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
  91. package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
  92. package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
  93. package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
  94. package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
  95. package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
  96. package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
  97. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
  98. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
  99. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
  100. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
  101. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
  102. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
  103. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
  104. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
  105. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
  106. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
  107. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
  108. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
  109. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
  110. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
  111. package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
  112. package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
  113. package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
  114. package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
  115. package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
  116. package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
  117. package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
  118. package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
  119. package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
  120. package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
  121. package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
  122. package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
  123. package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
  124. package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
  125. package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
  126. package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
  127. package/skills/egbe-os-llm/SKILL.md +160 -0
  128. package/skills/egbe-os-mail/SKILL.md +180 -0
  129. package/skills/egbe-os-me/SKILL.md +119 -0
  130. package/skills/egbe-os-meta-ads/SKILL.md +114 -0
  131. package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
  132. package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
  133. package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
  134. package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
  135. package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
  136. package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
  137. package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
  138. package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
  139. package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
  140. package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
  141. package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
  142. package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
  143. package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
  144. package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
  145. package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
  146. package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
  147. package/skills/egbe-os-notifications/SKILL.md +78 -0
  148. package/skills/egbe-os-openai-ads/SKILL.md +149 -0
  149. package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
  150. package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
  151. package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
  152. package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
  153. package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
  154. package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
  155. package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
  156. package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
  157. package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
  158. package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
  159. package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
  160. package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
  161. package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
  162. package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
  163. package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
  164. package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
  165. package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
  166. package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
  167. package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
  168. package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
  169. package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
  170. package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
  171. package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
  172. package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
  173. package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
  174. package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
  175. package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
  176. package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
  177. package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
  178. package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
  179. package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
  180. package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
  181. package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
  182. package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
  183. package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
  184. package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
  185. package/skills/egbe-os-payments/SKILL.md +215 -0
  186. package/skills/egbe-os-payments/agents/openai.yaml +4 -0
  187. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
  188. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
  189. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
  190. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
  191. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
  192. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
  193. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
  194. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
  195. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
  196. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
  197. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
  198. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
  199. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
  200. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
  201. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
  202. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
  203. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
  204. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
  205. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
  206. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
  207. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
  208. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
  209. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
  210. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
  211. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
  212. package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
  213. package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
  214. package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
  215. package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
  216. package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
  217. package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
  218. package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
  219. package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
  220. package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
  221. package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
  222. package/skills/egbe-os-playbook/SKILL.md +79 -0
  223. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
  224. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
  225. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
  226. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
  227. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
  228. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
  229. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
  230. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
  231. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
  232. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
  233. package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
  234. package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
  235. package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
  236. package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
  237. package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
  238. package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
  239. package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
  240. package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
  241. package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
  242. package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
  243. package/skills/egbe-os-playbook/playbooks/product.md +67 -0
  244. package/skills/egbe-os-playbook/references/catalog.md +127 -0
  245. package/skills/egbe-os-playbook/references/changes.md +167 -0
  246. package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
  247. package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
  248. package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
  249. package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
  250. package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
  251. package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
  252. package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
  253. package/skills/egbe-os-product-auth/SKILL.md +151 -0
  254. package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
  255. package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
  256. package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
  257. package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
  258. package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
  259. package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
  260. package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
  261. package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
  262. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
  263. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
  264. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
  265. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
  266. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
  267. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
  268. package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
  269. package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
  270. package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
  271. package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
  272. package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
  273. package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
  274. package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
  275. package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
  276. package/skills/egbe-os-product-build/SKILL.md +111 -0
  277. package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
  278. package/skills/egbe-os-product-build/references/capabilities.md +25 -0
  279. package/skills/egbe-os-product-build/references/engineering.md +72 -0
  280. package/skills/egbe-os-product-build/references/execution.md +55 -0
  281. package/skills/egbe-os-product-build/references/foundations.md +93 -0
  282. package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
  283. package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
  284. package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
  285. package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
  286. package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
  287. package/skills/egbe-os-product-market-research/SKILL.md +159 -0
  288. package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
  289. package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
  290. package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
  291. package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
  292. package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
  293. package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
  294. package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
  295. package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
  296. package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
  297. package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
  298. package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
  299. package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
  300. package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
  301. package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
  302. package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
  303. package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
  304. package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
  305. package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
  306. package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
  307. package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
  308. package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
  309. package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
  310. package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
  311. package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
  312. package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
  313. package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
  314. package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
  315. package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
  316. package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
  317. package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
  318. package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
  319. package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
  320. package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
  321. package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
  322. package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
  323. package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
  324. package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
  325. package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
  326. package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
  327. package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
  328. package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
  329. package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
  330. package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
  331. package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
  332. package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
  333. package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
  334. package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
  335. package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
  336. package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
  337. package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
  338. package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
  339. package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
  340. package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
  341. package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
  342. package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
  343. package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
  344. package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
  345. package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
  346. package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
  347. package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
  348. package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
  349. package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
  350. package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
  351. package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
  352. package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
  353. package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
  354. package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
  355. package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
  356. package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
  357. package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
  358. package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
  359. package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
  360. package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
  361. package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
  362. package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
  363. package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
  364. package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
  365. package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
  366. package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
  367. package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
  368. package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
  369. package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
  370. package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
  371. package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
  372. package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
  373. package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
  374. package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
  375. package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
  376. package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
  377. package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
  378. package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
  379. package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
  380. package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
  381. package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
  382. package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
  383. package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
  384. package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
  385. package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
  386. package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
  387. package/skills/egbe-os-product-report/SKILL.md +150 -0
  388. package/skills/egbe-os-product-report/references/artifacts.md +84 -0
  389. package/skills/egbe-os-product-report/references/brief.md +34 -0
  390. package/skills/egbe-os-product-report/references/daily.md +76 -0
  391. package/skills/egbe-os-product-report/references/sources.md +68 -0
  392. package/skills/egbe-os-product-routines/SKILL.md +166 -0
  393. package/skills/egbe-os-product-routines/references/catalog.md +22 -0
  394. package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
  395. package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
  396. package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
  397. package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
  398. package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
  399. package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
  400. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
  401. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
  402. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
  403. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
  404. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
  405. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
  406. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
  407. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
  408. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
  409. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
  410. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
  411. package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
  412. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
  413. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
  414. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
  415. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
  416. package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
  417. package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
  418. package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
  419. package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
  420. package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
  421. package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
  422. package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
  423. package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
  424. package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
  425. package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
  426. package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
  427. package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
  428. package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
  429. package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
  430. package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
  431. package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
  432. package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
  433. package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
  434. package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
  435. package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
  436. package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
  437. package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
  438. package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
  439. package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
  440. package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
  441. package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
  442. package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
  443. package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
  444. package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
  445. package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
  446. package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
  447. package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
  448. package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
  449. package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
  450. package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
  451. package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
  452. package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
  453. package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
  454. package/skills/egbe-os-report-ui/SKILL.md +133 -0
  455. package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
  456. package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
  457. package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
  458. package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
  459. package/skills/egbe-os-report-ui/references/components.md +209 -0
  460. package/skills/egbe-os-research/SKILL.md +125 -0
  461. package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
  462. package/skills/egbe-os-scrape/SKILL.md +160 -0
  463. package/skills/egbe-os-search/SKILL.md +182 -0
  464. package/skills/egbe-os-search/references/google-places.md +81 -0
  465. package/skills/egbe-os-support/SKILL.md +149 -0
  466. package/skills/egbe-os-telemetry/SKILL.md +344 -0
  467. package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
  468. package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
  469. package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
  470. package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
  471. package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
  472. package/skills/egbe-os-tools/SKILL.md +180 -0
  473. package/skills/egbe-os-tools/references/theona.md +26 -0
  474. package/skills/egbe-os-wallet/SKILL.md +181 -0
  475. package/skills/egbe-setup/EGBE.md +484 -0
  476. package/skills/egbe-setup/SKILL.md +116 -0
  477. package/skills/egbe-setup/references/project.md +80 -0
  478. package/state.js +37 -0
  479. package/update.js +361 -0
@@ -0,0 +1,927 @@
1
+ #!/usr/bin/env python3
2
+ """Collect Umami evidence and report per-session stage observations, not conversions.
3
+
4
+ Stdlib-only. Auth: project-local .env.local (OS_URL / OS_API_KEY), or environment.
5
+ Run from the project root, with the complete Egbe catalog installed.
6
+
7
+ python3 umami_funnel.py [--site <name-domain-or-id>] [--since 2026-07-01]
8
+ [--until 2026-08-01]
9
+ [--act-path /color] [--act-start-event generation_started]
10
+ [--act-event activated]
11
+ [--events-json evidence/events.json] [--json analysis/funnel.json]
12
+ python3 umami_funnel.py --from-events evidence/events.json --json analysis/replay.json
13
+
14
+ Output: totals (included vs excluded), traffic mix, depth/bounce, top paths,
15
+ custom events, stage counts, per-source sessions/arrivals, and compact,
16
+ filtered journeys for sessions with activity or exposure events. Raise
17
+ `--max-journey-events` when a longer timeline is needed. Repeated completion
18
+ events and a small allowlist of non-PII event properties are preserved by
19
+ joining Umami's paged event-data endpoint to event rows. Paths and canonical
20
+ visitor/visit IDs are anonymized before output. Exclude confirmed internal
21
+ activity and disclose suspects; completion is a technical proxy, never proof
22
+ of usable value. Filtering is described in references/data-recipes.md.
23
+ """
24
+ import argparse
25
+ import collections
26
+ import datetime as dt
27
+ import hashlib
28
+ import json
29
+ import math
30
+ import os
31
+ from pathlib import Path
32
+ import re
33
+ import runpy
34
+ import sys
35
+ import time
36
+ import urllib.parse
37
+ import urllib.error
38
+ import urllib.request
39
+
40
+ PAGE_SIZE, MAX_EVENT_PAGES = 500, 200
41
+ SIGNUP_PATHS = ("/signup", "/register", "/ru/signup", "/#/join")
42
+ POST_AUTH_PATHS = ("/dashboard", "/#/dashboard", "/#/app", "/account", "/admin")
43
+ CHECKOUT_PATHS = ("/checkout",)
44
+ PRICING_PATHS = ("/pricing", "/#pricing", "/#plans")
45
+ ENGAGE_EVENTS = {"campaign-visit", "engagement-scroll", "engagement-time", "engagement-click"}
46
+ EXPOSURE_EVENTS = {"section_view", "scroll_depth"}
47
+ SIGNUP_START_EVENTS = {"signup_started"}
48
+ SIGNUP_DONE_EVENTS = {"signup_completed"}
49
+ CHECKOUT_EVENTS = {"checkout_started"}
50
+ PAYWALL_EVENTS = {"paywall_view"}
51
+ TECHNICAL_START_EVENTS = {
52
+ "main_action_submit", "quiz_start", "request_form_start", "activation_started",
53
+ }
54
+ TECHNICAL_DONE_EVENTS = {
55
+ "activated", "main_action_success", "quiz_complete", "activation_completed",
56
+ }
57
+ SAFE_EVENT_FIELDS = {
58
+ "button", "cta", "action", "plan", "surface", "status", "hq",
59
+ "activation_type", "style", "room_type", "op", "count", "product_slug",
60
+ "cta_id", "cta_role", "button_id", "button_role", "input_type",
61
+ "result_type", "duration_ms", "checkout_type", "currency", "amount_cents",
62
+ "price_cents", "plan_name", "section", "percent", "thresholdSeconds", "position",
63
+ }
64
+ NUMERIC_EVENT_FIELDS = {
65
+ "count", "duration_ms", "amount_cents", "price_cents",
66
+ "percent", "thresholdSeconds", "position",
67
+ }
68
+ BOOLEAN_EVENT_FIELDS = {"hq"}
69
+ IDENTITY_EVENT_FIELDS = {"visitor_id", "visit_id"}
70
+ ASSET_PATH_PREFIXES = ("/_next", "/favicon", "/assets", "/static/")
71
+ SENSITIVE_PATH_PARENTS = {
72
+ "u", "user", "users", "r", "share", "invite", "reset", "verify",
73
+ "magic", "token", "session", "sessions",
74
+ }
75
+ EMAIL_ANY_RE = re.compile(r"[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}", re.IGNORECASE)
76
+ UUID_RE = re.compile(
77
+ r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
78
+ re.IGNORECASE,
79
+ )
80
+ JWT_RE = re.compile(r"[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}")
81
+ URL_RE = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*://\S+|\bwww\.\S+", re.IGNORECASE)
82
+ LONG_HEX_RE = re.compile(r"^[0-9a-f]{20,}$", re.IGNORECASE)
83
+ BASE64URL_RE = re.compile(r"^[A-Za-z0-9_-]{24,}$")
84
+ CONTROL_RE = re.compile(r"[\x00-\x1f\x7f]")
85
+ LONG_NUMBER_RE = re.compile(r"(?<!\d)\+?\d[\d .()-]{6,}\d(?!\d)")
86
+ SNAPSHOT_FORMAT = "egbe-umami-events.v1"
87
+ SAFE_UTM_FIELDS = ("utm_source", "utm_medium", "utm_campaign", "utm_content")
88
+ ANALYSIS_OPTIONS = (
89
+ "act_event", "act_start_event", "act_path", "internal_sources",
90
+ "internal_country", "dc_country", "max_journey_events",
91
+ )
92
+
93
+
94
+ def os_connection():
95
+ root = Path(__file__).resolve().parents[1]
96
+ quickstart = "egbe-os-quickstart" if root.name.startswith("egbe-os-") else "quickstart"
97
+ helper_path = root.parent / quickstart / "scripts/egbe-curl.py"
98
+ if not helper_path.is_file():
99
+ sys.exit("Install the complete Egbe catalog through quickstart before collecting analytics")
100
+ helper = runpy.run_path(str(helper_path))
101
+ settings = Path.cwd() / ".env.local"
102
+ try:
103
+ env = helper["project_environment"](str(settings) if settings.exists() else None)
104
+ base, key = env.get("OS_URL", "").rstrip("/"), env.get("OS_API_KEY", "")
105
+ parsed = urllib.parse.urlsplit(base)
106
+ if not key or not parsed.hostname or parsed.scheme not in {"http", "https"} or parsed.query or parsed.fragment or parsed.username:
107
+ raise ValueError("OS_URL and OS_API_KEY are required")
108
+ args = helper["command"]([base + "/analytics"], env)
109
+ headers = dict(args[i + 1].split(": ", 1) for i, arg in enumerate(args) if arg == "-H")
110
+ headers["Authorization"] = "Bearer " + key
111
+ return base, headers
112
+ except (ValueError, OSError):
113
+ sys.exit("Check OS_URL / OS_API_KEY in .env.local and this agent's quickstart connection identity")
114
+
115
+
116
+ class NoRedirect(urllib.request.HTTPRedirectHandler):
117
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
118
+ return None # Never forward project credentials to a redirected host.
119
+
120
+
121
+ def call(url, headers):
122
+ req = urllib.request.Request(url, headers=headers)
123
+ try:
124
+ with urllib.request.build_opener(NoRedirect()).open(req, timeout=60) as response:
125
+ result = json.load(response)
126
+ except urllib.error.HTTPError as error:
127
+ raise RuntimeError(f"OS analytics returned HTTP {error.code}; check access, app registration and OS release") from None
128
+ if not isinstance(result, dict) or result.get("code") != 200 or "data" not in result:
129
+ raise ValueError("Invalid OS analytics response envelope")
130
+ return result["data"]
131
+
132
+
133
+ def event_property_value(prop):
134
+ for key in ("stringValue", "numberValue", "dateValue"):
135
+ if prop.get(key) is not None:
136
+ return prop[key]
137
+ return None
138
+
139
+
140
+ def ephemeral_ref(kind, value, salt):
141
+ if not isinstance(value, (str, int, float)) or not str(value):
142
+ return None
143
+ digest = hashlib.sha256(
144
+ salt + b"\0" + kind.encode("utf-8") + b"\0" + str(value).encode("utf-8")
145
+ ).hexdigest()
146
+ return digest[:12]
147
+
148
+
149
+ def looks_opaque_token(value):
150
+ """Distinguish likely secrets/IDs from ordinary long semantic slugs."""
151
+ if not isinstance(value, str):
152
+ return False
153
+ if LONG_HEX_RE.fullmatch(value) or JWT_RE.fullmatch(value):
154
+ return True
155
+ if not BASE64URL_RE.fullmatch(value):
156
+ return False
157
+ chunks = re.split(r"[-_]", value)
158
+ mixed_chunks = [
159
+ chunk for chunk in chunks
160
+ if any(char.isalpha() for char in chunk)
161
+ and any(char.isdigit() for char in chunk)
162
+ ]
163
+ secret_prefix = (
164
+ len(chunks) >= 2
165
+ and ((chunks[0] in {"sk", "pk", "rk"} and chunks[1] in {"live", "test", "prod"})
166
+ or chunks[:2] in (["api", "key"], ["access", "token"]))
167
+ )
168
+ if (secret_prefix
169
+ or any(len(chunk) >= 12 for chunk in mixed_chunks)
170
+ or (len(mixed_chunks) >= 2 and sum(map(len, mixed_chunks)) >= 12)):
171
+ return True
172
+ if len(chunks) > 1 and all(
173
+ re.fullmatch(r"[a-z0-9]{1,16}", chunk) for chunk in chunks):
174
+ # Preserve readable kebab/snake-case route and event labels.
175
+ return False
176
+ compact = value.replace("-", "").replace("_", "")
177
+ classes = sum((
178
+ any(char.islower() for char in compact),
179
+ any(char.isupper() for char in compact),
180
+ any(char.isdigit() for char in compact),
181
+ ))
182
+ return (classes >= 2 and len(set(compact)) >= 8) or len(set(compact)) >= 16
183
+
184
+
185
+ def safe_event_value(key, value):
186
+ """Return a short scalar only when its value has no obvious PII/secret."""
187
+ if isinstance(value, bool):
188
+ return value if key in BOOLEAN_EVENT_FIELDS else None
189
+ if isinstance(value, (int, float)):
190
+ return value if key in NUMERIC_EVENT_FIELDS and (
191
+ isinstance(value, int) or math.isfinite(value)
192
+ ) else None
193
+ if not isinstance(value, str):
194
+ return None
195
+ if len(value) > 120 or CONTROL_RE.search(value):
196
+ return None
197
+ inspected = urllib.parse.unquote_plus(value)
198
+ if key in NUMERIC_EVENT_FIELDS:
199
+ return value if re.fullmatch(r"-?\d+(?:\.\d+)?", inspected) else None
200
+ if key in BOOLEAN_EVENT_FIELDS:
201
+ return value if inspected.lower() in {"true", "false"} else None
202
+ if (EMAIL_ANY_RE.search(inspected) or URL_RE.search(inspected)
203
+ or UUID_RE.search(inspected) or JWT_RE.search(inspected)
204
+ or LONG_NUMBER_RE.search(inspected)):
205
+ return None
206
+ for candidate in re.findall(r"[A-Za-z0-9_-]{20,}", inspected):
207
+ if looks_opaque_token(candidate):
208
+ return None
209
+ return value
210
+
211
+
212
+ def parse_event_properties(properties, identity_salt):
213
+ """Extract a strict safe allowlist and ephemeral identity references."""
214
+ safe, identity_refs = {}, {}
215
+ for prop in properties if isinstance(properties, list) else []:
216
+ if not isinstance(prop, dict):
217
+ continue
218
+ key = prop.get("dataKey")
219
+ value = event_property_value(prop)
220
+ if key in SAFE_EVENT_FIELDS:
221
+ safe_value = safe_event_value(key, value)
222
+ if safe_value is not None:
223
+ safe[key] = safe_value
224
+ elif key in IDENTITY_EVENT_FIELDS:
225
+ ref = ephemeral_ref(key, value, identity_salt)
226
+ if ref:
227
+ identity_refs[key] = ref
228
+ return safe, identity_refs
229
+
230
+
231
+ def fetch_event_data(base, app_slug, headers, start, end):
232
+ """Fetch bulk Umami event data, retaining only safe fields and hashed IDs."""
233
+ safe_by_event, identity_by_event, seen = {}, {}, set()
234
+ identity_salt = os.urandom(16)
235
+ page, expected_count, status, error_type = 1, 0, "available", None
236
+ while page <= MAX_EVENT_PAGES:
237
+ query = urllib.parse.urlencode({
238
+ "startAt": start, "endAt": end, "page": page, "pageSize": PAGE_SIZE,
239
+ })
240
+ try:
241
+ response = call(
242
+ f"{base}/analytics/{app_slug}/event-data?{query}", headers,
243
+ )
244
+ except Exception as error: # Keep base funnel usable when enrichment fails.
245
+ status = "partial" if seen else "unavailable"
246
+ error_type = type(error).__name__
247
+ break
248
+ if not isinstance(response, dict):
249
+ status, error_type = "partial" if seen else "unavailable", "invalid_response"
250
+ break
251
+ entries = response.get("data", [])
252
+ if not isinstance(entries, list):
253
+ status, error_type = "partial" if seen else "unavailable", "invalid_data"
254
+ break
255
+ try:
256
+ expected_count = int(response.get("count", expected_count or 0) or 0)
257
+ except (TypeError, ValueError):
258
+ status, error_type = "partial" if seen else "unavailable", "invalid_count"
259
+ break
260
+ for entry in entries:
261
+ if not isinstance(entry, dict) or not entry.get("eventId"):
262
+ continue
263
+ event_id = str(entry["eventId"])
264
+ safe, identity_refs = parse_event_properties(
265
+ entry.get("eventProperties", []), identity_salt,
266
+ )
267
+ safe_by_event[event_id] = safe
268
+ identity_by_event[event_id] = identity_refs
269
+ seen.add(event_id)
270
+ if not entries:
271
+ if len(seen) < expected_count:
272
+ status = "partial" if seen else "unavailable"
273
+ error_type = "empty_page"
274
+ break
275
+ if len(seen) >= expected_count:
276
+ break
277
+ if len(entries) < PAGE_SIZE:
278
+ status = "partial" if seen else "unavailable"
279
+ error_type = "short_page"
280
+ break
281
+ page += 1
282
+ if page > MAX_EVENT_PAGES and len(seen) < expected_count:
283
+ status, error_type = "partial", "page_limit"
284
+ return safe_by_event, identity_by_event, {
285
+ "status": status,
286
+ "reported_events": expected_count,
287
+ "fetched_events": len(seen),
288
+ "error_type": error_type,
289
+ }
290
+
291
+
292
+ def attach_event_data(rows, base, app_slug, headers, start, end, skip=False):
293
+ marked = sum(bool(row.get("hasData")) for row in rows)
294
+ if skip:
295
+ return {
296
+ "status": "skipped", "reported_events": None, "fetched_events": 0,
297
+ "marked_rows": marked, "matched_rows": 0, "error_type": None,
298
+ }
299
+ safe_by_event, identity_by_event, status = fetch_event_data(
300
+ base, app_slug, headers, start, end,
301
+ )
302
+ matched = 0
303
+ for row in rows:
304
+ event_id = str(row.get("id") or "")
305
+ if event_id in safe_by_event:
306
+ row["eventData"] = safe_by_event[event_id]
307
+ row["eventIdentityRefs"] = identity_by_event.get(event_id, {})
308
+ row["_eventDataLoaded"] = True
309
+ matched += 1
310
+ elif row.get("hasData"):
311
+ row["_eventDataLoaded"] = False
312
+ return {**status, "marked_rows": marked, "matched_rows": matched}
313
+
314
+
315
+ def fetch_events(args):
316
+ args.window = getattr(args, "window", None) or resolve_window(args)
317
+ start, end = args.window["startAt"], args.window["endAt"]
318
+ base, headers = os_connection()
319
+ sites = call(f"{base}/analytics", headers)
320
+ if not isinstance(sites, list) or any(not isinstance(s, dict) for s in sites):
321
+ sys.exit("invalid OS analytics site list")
322
+ if args.site:
323
+ query = args.site.casefold()
324
+ exact = [s for s in sites if query in {
325
+ str(s.get(key) or "").casefold() for key in ("websiteId", "appSlug", "domain", "name")
326
+ }]
327
+ sites = exact or [s for s in sites if any(
328
+ query in (s.get(key) or "").casefold() for key in ("domain", "name")
329
+ )]
330
+ if not sites:
331
+ sys.exit("no matching website; check --site or register the product's analytics site")
332
+ if len(sites) != 1:
333
+ sys.exit(f"expected one website, found {len(sites)}; use --site with an exact name/domain/id")
334
+ slug = sites[0].get("appSlug")
335
+ if not isinstance(slug, str) or not re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,38}[a-z0-9])?", slug):
336
+ sys.exit("analytics site has no valid appSlug; check its OS registration")
337
+ args.website = {"id": sites[0].get("websiteId"), **{key: sites[0].get(key) for key in ("name", "domain")}}
338
+ print(f"# website: {sites[0].get('name')} ({sites[0].get('domain')})")
339
+ rows, page = [], 1
340
+ while True:
341
+ ev = call(f"{base}/analytics/{slug}/events?startAt={start}&endAt={end}&page={page}&pageSize={PAGE_SIZE}", headers)
342
+ if not isinstance(ev, dict) or not isinstance(ev.get("data"), list):
343
+ sys.exit("invalid event page; coverage is unknown")
344
+ data = ev.get("data", [])
345
+ rows.extend(data)
346
+ count = ev.get("count")
347
+ if type(count) is not int or count < 0:
348
+ sys.exit("event response has no valid total count; coverage is unknown")
349
+ if count > MAX_EVENT_PAGES * PAGE_SIZE:
350
+ sys.exit(f"event count {count} exceeds the {MAX_EVENT_PAGES * PAGE_SIZE} event cap; narrow the date window")
351
+ if len(rows) >= count:
352
+ break
353
+ if not data:
354
+ sys.exit(f"empty event page {page}: fetched {len(rows)} of {count}; count/data mismatch, retry or inspect the source")
355
+ if page >= MAX_EVENT_PAGES:
356
+ sys.exit(f"incomplete event download at page cap: {len(rows)} of {count}; inspect pagination or narrow the window")
357
+ page += 1
358
+ event_data_status = attach_event_data(
359
+ rows, base, slug, headers, start, end, skip=args.skip_event_data,
360
+ )
361
+ return rows, event_data_status
362
+
363
+
364
+ def utc_date_ms(value):
365
+ if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
366
+ raise ValueError("expected YYYY-MM-DD")
367
+ return int(dt.datetime.strptime(value, "%Y-%m-%d").replace(
368
+ tzinfo=dt.timezone.utc,
369
+ ).timestamp() * 1000)
370
+
371
+
372
+ def resolve_window(args):
373
+ bounds = {}
374
+ for option in ("since", "until"):
375
+ value = getattr(args, option, None)
376
+ try:
377
+ bounds[option] = utc_date_ms(value) if value else None
378
+ except ValueError as error:
379
+ raise ValueError(f"--{option} must be a valid YYYY-MM-DD date (UTC)") from error
380
+ start = bounds["since"] if bounds["since"] is not None else 0
381
+ # Umami's endAt is inclusive; --until is exclusive.
382
+ end = bounds["until"] - 1 if bounds["until"] is not None else int(time.time() * 1000)
383
+ if end < start:
384
+ raise ValueError("--until must be after --since" if bounds["until"] is not None
385
+ else "--since must not be in the future when --until is omitted")
386
+ return {"startAt": start, "endAt": end, "timezone": "UTC",
387
+ "start_utc": dt.datetime.fromtimestamp(start / 1000, dt.timezone.utc).isoformat(),
388
+ "end_inclusive_utc": dt.datetime.fromtimestamp(end / 1000, dt.timezone.utc).isoformat()}
389
+
390
+
391
+ def first_touch_source(evs):
392
+ q = urllib.parse.parse_qs(evs[0].get("urlQuery") or "")
393
+ utm = (q.get("utm_source") or [None])[0]
394
+ return utm or (evs[0].get("referrerDomain") or "direct")
395
+
396
+
397
+ def internal_reason(evs, src, args):
398
+ # A smoke tag may first appear after an initial pageview; keep the whole
399
+ # dedicated audit session out even when emulating a mobile device.
400
+ sources = {src}
401
+ for event in evs:
402
+ query = urllib.parse.parse_qs(event.get("urlQuery") or "")
403
+ sources.update(query.get("utm_source", []))
404
+ matched = sources.intersection(args.internal_sources)
405
+ if matched:
406
+ return "source:" + sorted(matched)[0]
407
+ country = evs[0].get("country")
408
+ return f"verified_country:{country}" if country in args.internal_country else None
409
+
410
+
411
+ def is_suspected_internal(evs, src, args):
412
+ """Heuristics are sensitivity flags, never automatic exclusions."""
413
+ f0 = evs[0]
414
+ if (src == "direct" and f0.get("device") in ("laptop", "desktop")
415
+ and f0.get("country") in args.dc_country):
416
+ return True
417
+ for e in evs:
418
+ p = e.get("urlPath") or ""
419
+ if e.get("eventType") == 1 and ("/admin" in p or "/tmp/" in p or "localhost" in p):
420
+ return True
421
+ return False
422
+
423
+
424
+ def safe_path(path):
425
+ """Redact common identifier/token path segments before output or storage."""
426
+ if not isinstance(path, str):
427
+ return path
428
+ path = path.split("?", 1)[0].split("#", 1)[0]
429
+ parts, redacted, redact_next = path.split("/"), [], False
430
+ for raw in parts:
431
+ decoded = urllib.parse.unquote(raw)
432
+ lowered = decoded.lower()
433
+ sensitive = (
434
+ redact_next
435
+ or safe_event_value("surface", decoded) is None
436
+ or bool(UUID_RE.fullmatch(decoded))
437
+ or bool(JWT_RE.fullmatch(decoded))
438
+ or looks_opaque_token(decoded)
439
+ )
440
+ redacted.append(":redacted" if sensitive else raw)
441
+ redact_next = lowered in SENSITIVE_PATH_PARENTS
442
+ return "/".join(redacted)
443
+
444
+
445
+ def safe_label(value):
446
+ """Keep categorical labels, not arbitrary URLs/free-form text."""
447
+ if not isinstance(value, str) or not re.fullmatch(r"[\w .:/-]{1,120}", value):
448
+ return None
449
+ return safe_event_value("surface", value)
450
+
451
+
452
+ def safe_route_path(path):
453
+ """Preserve hash-router/anchor semantics for stage detection after redaction."""
454
+ before, marker, fragment = path.partition("#")
455
+ base = safe_path(before)
456
+ if not marker:
457
+ return base
458
+ safe_fragment = safe_path(fragment) if fragment.startswith("/") else safe_label(fragment)
459
+ return base + "#" + (safe_fragment or ":redacted")
460
+
461
+
462
+ def snapshot_rows(rows):
463
+ """Retain uncapped safe input rows; pseudonymous joins last for this snapshot."""
464
+ salt = os.urandom(16)
465
+ result = []
466
+ for row in rows:
467
+ dt.datetime.fromisoformat(row["createdAt"].replace("Z", "+00:00"))
468
+ if type(row["eventType"]) is not int or row["eventType"] < 0:
469
+ raise ValueError("Umami event type must be a non-negative integer")
470
+ session_ref = ephemeral_ref("session_id", row["sessionId"], salt)
471
+ path = row.get("urlPath") or ""
472
+ query = urllib.parse.parse_qs(row.get("urlQuery") or "")
473
+ attribution = {key: [safe_label(value) or "redacted" for value in query[key]]
474
+ for key in SAFE_UTM_FIELDS if key in query}
475
+ # Retain distinct-page counting even when different private paths redact alike.
476
+ page_ref = ephemeral_ref("page", path.split("#")[0].rstrip("/") or "/", salt)
477
+ item = {
478
+ "id": ephemeral_ref("event", row.get("id"), salt),
479
+ "sessionId": session_ref, "_sessionRef": session_ref,
480
+ "createdAt": row["createdAt"], "eventType": row["eventType"],
481
+ "urlPath": safe_route_path(path), "_pageRef": page_ref,
482
+ "urlQuery": urllib.parse.urlencode(attribution, doseq=True),
483
+ "referrerDomain": (safe_label(row["referrerDomain"]) or "redacted") if row.get("referrerDomain") else "",
484
+ "country": row.get("country") if re.fullmatch(r"[A-Z]{2}", row.get("country") or "") else None,
485
+ "device": row.get("device") if row.get("device") in {"mobile", "tablet", "laptop", "desktop"} else None,
486
+ }
487
+ if row.get("eventName"):
488
+ item["eventName"] = safe_label(row["eventName"]) or "redacted"
489
+ if row.get("hasData"):
490
+ item["hasData"] = True
491
+ if row.get("_eventDataLoaded") or "eventData" in row:
492
+ item["_eventDataLoaded"] = True
493
+ item["eventData"] = safe_event_props(row)
494
+ refs = event_identity_refs(row, salt)
495
+ # Even pre-enriched input refs are re-keyed together for this export.
496
+ item["eventIdentityRefs"] = {
497
+ key: ephemeral_ref(key, value, salt) for key, value in refs.items()
498
+ }
499
+ result.append(item)
500
+ return result
501
+
502
+
503
+ def make_snapshot(rows, args, event_data_status):
504
+ return {
505
+ "format": SNAPSHOT_FORMAT,
506
+ "retrieved_at": dt.datetime.now(dt.timezone.utc).isoformat(),
507
+ "website": args.website, "window": args.window,
508
+ "analysis_options": {key: getattr(args, key) for key in ANALYSIS_OPTIONS},
509
+ "coverage": {"base_events": "complete", "downloaded_rows": len(rows),
510
+ "non_funnel_event_types": dict(collections.Counter(
511
+ str(row["eventType"]) for row in rows if row["eventType"] not in (1, 2))),
512
+ "event_data": event_data_status},
513
+ "redaction": "Safe categorical fields/properties and UTM source/medium/campaign/content only; "
514
+ "identities and distinct paths use per-snapshot pseudonyms. "
515
+ "Private path segments and unsupported properties cannot be reconstructed.",
516
+ "rows": snapshot_rows(rows),
517
+ }
518
+
519
+
520
+ def read_snapshot(path, args):
521
+ with open(path, encoding="utf-8") as source:
522
+ snapshot = json.load(source)
523
+ if snapshot.get("format") != SNAPSHOT_FORMAT or not isinstance(snapshot.get("rows"), list):
524
+ raise ValueError("expected an egbe-umami-events.v1 snapshot, not a summary")
525
+ for key in ANALYSIS_OPTIONS:
526
+ setattr(args, key, snapshot["analysis_options"][key])
527
+ args.window, args.website = snapshot["window"], snapshot["website"]
528
+ return snapshot["rows"], snapshot["coverage"]["event_data"]
529
+
530
+
531
+ def check_output_paths(*paths):
532
+ selected = [Path(path) for path in paths if path]
533
+ if len({path.resolve() for path in selected}) != len(selected):
534
+ raise ValueError("source and summary outputs must have different paths")
535
+ for path in selected:
536
+ if os.path.lexists(path):
537
+ raise ValueError(f"output already exists; preserve it and choose a new filename: {path}")
538
+ if not path.parent.is_dir():
539
+ raise ValueError(f"create the output directory first: {path.parent}")
540
+
541
+
542
+ def write_json(path, value):
543
+ payload = json.dumps(value, indent=1, allow_nan=False)
544
+ with open(path, "x", encoding="utf-8") as output:
545
+ output.write(payload + "\n")
546
+
547
+
548
+ def safe_event_props(event):
549
+ """Return only explicitly allowlisted, short scalar event properties."""
550
+ value = event.get("eventData")
551
+ if isinstance(value, str) and value.startswith("{") and len(value) <= 5000:
552
+ try:
553
+ value = json.loads(value)
554
+ except json.JSONDecodeError:
555
+ value = None
556
+ if not isinstance(value, dict):
557
+ return {}
558
+ props = {}
559
+ for key in SAFE_EVENT_FIELDS:
560
+ prop_value = safe_event_value(key, value.get(key))
561
+ if prop_value is not None:
562
+ props[key] = prop_value
563
+ return props
564
+
565
+
566
+ def event_identity_refs(event, salt):
567
+ existing = event.get("eventIdentityRefs")
568
+ if isinstance(existing, dict):
569
+ return {key: value for key, value in existing.items()
570
+ if key in IDENTITY_EVENT_FIELDS and isinstance(value, str)}
571
+ payload = event.get("eventData")
572
+ if not isinstance(payload, dict):
573
+ return {}
574
+ refs = {}
575
+ for key in IDENTITY_EVENT_FIELDS:
576
+ ref = ephemeral_ref(key, payload.get(key), salt)
577
+ if ref:
578
+ refs[key] = ref
579
+ return refs
580
+
581
+
582
+ def build_timeline(evs, max_entries):
583
+ """Build a redacted timeline, collapsing only adjacent identical pageviews."""
584
+ display, structured = [], []
585
+ eligible_count, last_pageview = 0, None
586
+ for event in evs:
587
+ if event["eventType"] == 1:
588
+ raw_path = event.get("urlPath")
589
+ if not raw_path or raw_path.startswith(ASSET_PATH_PREFIXES):
590
+ continue
591
+ token = safe_path(raw_path)
592
+ if token == last_pageview:
593
+ continue
594
+ last_pageview = token
595
+ item = {"at": event["createdAt"], "type": "pageview", "path": token}
596
+ elif event["eventType"] == 2:
597
+ last_pageview = None
598
+ name = event.get("eventName")
599
+ if not name:
600
+ continue
601
+ props = safe_event_props(event)
602
+ has_data = bool(event.get("hasData") or "eventData" in event)
603
+ data_loaded = bool(event.get("_eventDataLoaded") or "eventData" in event)
604
+ suffix = " " + json.dumps(props, sort_keys=True) if props else ""
605
+ if has_data and not data_loaded:
606
+ suffix += " {event_data:unavailable}"
607
+ token = f"[{name}]{suffix}"
608
+ item = {"at": event["createdAt"], "type": "event", "name": name}
609
+ if props:
610
+ item["properties"] = props
611
+ if has_data:
612
+ item["has_data"] = True
613
+ item["event_data_loaded"] = data_loaded
614
+ else:
615
+ continue
616
+ eligible_count += 1
617
+ if len(structured) >= max_entries:
618
+ continue
619
+ display.append(f"{event['createdAt'][11:19]} {token}")
620
+ structured.append(item)
621
+ return display, structured, eligible_count
622
+
623
+
624
+ def analyze(rows, args, event_data_status=None):
625
+ rows.sort(key=lambda r: r["createdAt"])
626
+ sessions = collections.defaultdict(list)
627
+ for r in rows:
628
+ sessions[r["sessionId"]].append(r)
629
+
630
+ included, internal, suspected = {}, {}, set()
631
+ exclusion_reasons = collections.Counter()
632
+ for sid, evs in sessions.items():
633
+ src = first_touch_source(evs)
634
+ reason = internal_reason(evs, src, args)
635
+ (internal if reason else included)[sid] = (src, evs)
636
+ if reason:
637
+ exclusion_reasons[reason] += 1
638
+ if sid in included and is_suspected_internal(evs, src, args):
639
+ suspected.add(sid)
640
+
641
+ funnel = collections.Counter()
642
+ suspect_funnel = collections.Counter()
643
+ mix = collections.Counter()
644
+ arrivals_by_source = collections.Counter()
645
+ depth = collections.Counter()
646
+ paths = collections.defaultdict(set)
647
+ events = collections.defaultdict(set)
648
+ event_occurrences = collections.Counter()
649
+ interesting = []
650
+ journey_rows = []
651
+ repeat_completions = {}
652
+ identity_salt = os.urandom(16)
653
+ start_names = TECHNICAL_START_EVENTS | set(args.act_start_event)
654
+ completion_names = TECHNICAL_DONE_EVENTS | set(args.act_event)
655
+ run = {
656
+ "window": getattr(args, "window", None), "website": getattr(args, "website", None),
657
+ "config": {"execution_start_events": sorted(start_names),
658
+ "execution_completion_events": sorted(completion_names),
659
+ "execution_start_paths": args.act_path,
660
+ "internal_sources": args.internal_sources,
661
+ "internal_countries": args.internal_country,
662
+ "suspect_countries": args.dc_country,
663
+ "ambient_events": sorted(ENGAGE_EVENTS | EXPOSURE_EVENTS),
664
+ "max_journey_events": args.max_journey_events},
665
+ }
666
+ print("# run: " + json.dumps(run, sort_keys=True))
667
+ if event_data_status is None:
668
+ marked = sum(bool(row.get("hasData")) for row in rows)
669
+ loaded = sum(bool(row.get("_eventDataLoaded") or "eventData" in row) for row in rows)
670
+ event_data_status = {
671
+ "status": "provided" if loaded else "not_fetched",
672
+ "reported_events": None,
673
+ "fetched_events": loaded,
674
+ "marked_rows": marked,
675
+ "matched_rows": loaded,
676
+ "error_type": None,
677
+ }
678
+
679
+ for sid, (src, evs) in included.items():
680
+ session_ref = evs[0].get("_sessionRef") or ephemeral_ref("session_id", sid, identity_salt)
681
+ mix[src] += 1
682
+ pv = [e for e in evs if e["eventType"] == 1 and e.get("urlPath")
683
+ and not e["urlPath"].startswith(ASSET_PATH_PREFIXES)]
684
+ event_names = [e.get("eventName") for e in evs
685
+ if e["eventType"] == 2 and e.get("eventName")]
686
+ evn_counts = collections.Counter(event_names)
687
+ evn = set(event_names)
688
+ for e in pv:
689
+ paths[safe_path(e["urlPath"])].add(sid)
690
+ for n, count in evn_counts.items():
691
+ events[n].add(sid)
692
+ event_occurrences[n] += count
693
+ npv = len(pv)
694
+ depth_bucket = f"{npv}pv" if npv <= 1 else ("2-3pv" if npv <= 3 else "4+pv")
695
+ depth[depth_bucket] += 1
696
+ pset = {p["urlPath"] for p in pv}
697
+ base_pages = {p.get("_pageRef") or p["urlPath"].split("#")[0].rstrip("/") or "/" for p in pv}
698
+ # Activity proxy, not proven attention: exposures may fire on load.
699
+ activity_events = evn - ENGAGE_EVENTS - EXPOSURE_EVENTS
700
+ scroll_dwell = {"engagement-scroll", "engagement-time"} <= evn
701
+ engaged = len(base_pages) > 1 or bool(activity_events) or scroll_dwell
702
+ completion_count = sum(evn_counts[name] for name in completion_names)
703
+ execution_started = bool(start_names & evn) or any(
704
+ p.startswith(tuple(args.act_path)) for p in pset if args.act_path)
705
+ execution_completed = completion_count > 0
706
+ f = collections.Counter()
707
+ if pv:
708
+ f["arrival"] += 1
709
+ arrivals_by_source[src] += 1
710
+ if not engaged: f["unengaged_arrivals"] += 1
711
+ if engaged: f["engaged"] += 1
712
+ if "cta_click" in evn: f["cta_click"] += 1
713
+ if "button_click" in evn: f["button_click"] += 1
714
+ if execution_started: f["execution_started"] += 1
715
+ if execution_completed: f["execution_completed*"] += 1
716
+ if completion_count > 1:
717
+ repeat_completions[session_ref] = completion_count
718
+ if any(p.startswith(SIGNUP_PATHS) for p in pset):
719
+ f["signup_page_seen"] += 1
720
+ if SIGNUP_START_EVENTS & evn:
721
+ f["signup_started"] += 1
722
+ if SIGNUP_DONE_EVENTS & evn:
723
+ f["signup_completed*"] += 1 # * reconcile vs ground truth
724
+ if any(p.startswith(POST_AUTH_PATHS) for p in pset):
725
+ f["post_auth_seen"] += 1
726
+ if PAYWALL_EVENTS & evn or any(p.startswith(PRICING_PATHS) for p in pset):
727
+ f["pricing_seen"] += 1
728
+ if CHECKOUT_EVENTS & evn or any(p.startswith(CHECKOUT_PATHS) for p in pset):
729
+ f["checkout_started"] += 1
730
+ funnel.update(f)
731
+ if sid in suspected:
732
+ suspect_funnel.update(f)
733
+ # Journey list stays focused on sessions with something to read:
734
+ # Keep exposure timelines for reach analysis even when not engaged.
735
+ # scroll+dwell-only sessions have no journey beyond beacons.
736
+ beyond = (len(base_pages) > 1 or execution_started or execution_completed
737
+ or evn - ENGAGE_EVENTS)
738
+ if beyond:
739
+ tl, structured_timeline, timeline_items_total = build_timeline(
740
+ evs, args.max_journey_events,
741
+ )
742
+ session_identity_refs = {"visitor": set(), "visit": set()}
743
+ for event in evs:
744
+ refs = event_identity_refs(event, identity_salt)
745
+ if refs.get("visitor_id"):
746
+ session_identity_refs["visitor"].add(refs["visitor_id"])
747
+ if refs.get("visit_id"):
748
+ session_identity_refs["visit"].add(refs["visit_id"])
749
+ anonymized_refs = {
750
+ key: sorted(value) for key, value in session_identity_refs.items() if value
751
+ }
752
+ f0 = evs[0]
753
+ identity_suffix = f" refs={anonymized_refs}" if anonymized_refs else ""
754
+ truncated = timeline_items_total > args.max_journey_events
755
+ truncation_suffix = (
756
+ f" | … {timeline_items_total - args.max_journey_events} more"
757
+ if truncated else ""
758
+ )
759
+ interesting.append(
760
+ f"{session_ref} {f0['createdAt'][:10]} {f0.get('country')}/{f0.get('device')} "
761
+ f"src={src}{' [suspect]' if sid in suspected else ''}{identity_suffix}\n "
762
+ + " | ".join(tl) + truncation_suffix)
763
+ journey_rows.append({
764
+ "session_ref": session_ref,
765
+ "date": f0["createdAt"][:10],
766
+ "country": f0.get("country"),
767
+ "device": f0.get("device"),
768
+ "source": src,
769
+ "suspected_internal": sid in suspected,
770
+ "execution_completion_event_count": completion_count,
771
+ "events_total": len(evs),
772
+ "timeline_items_total": timeline_items_total,
773
+ "timeline_truncated": truncated,
774
+ "identity_refs": anonymized_refs,
775
+ "timeline": structured_timeline,
776
+ })
777
+
778
+ n = max(len(included), 1)
779
+ print(f"\n== internal filter: sources={args.internal_sources} "
780
+ f"verified-internal-country={args.internal_country or 'NONE'} "
781
+ f"suspect-dc-laptop-direct={args.dc_country}")
782
+ print(f"== sessions: included={len(included)} internal={len(internal)} "
783
+ f"suspected_included={len(suspected)} "
784
+ f"excluded_by_reason={dict(exclusion_reasons)}")
785
+ print(" included = not excluded by configured rules; not verified humans/customers")
786
+ print(f"event data: {event_data_status}")
787
+ print(f"traffic (first touch): {dict(mix.most_common(10))}")
788
+ print(f"observed arrivals by source (suspects included): {dict(arrivals_by_source)}")
789
+ arrivals = funnel["arrival"]
790
+ bounce = 100 * funnel["unengaged_arrivals"] / arrivals if arrivals else None
791
+ print(f"depth: {dict(depth)} | bounce(unengaged/arrivals)="
792
+ + (f"{bounce:.1f}%" if bounce is not None else "unknown (no observed arrivals)"))
793
+ without_suspects = {key: value - suspect_funnel[key] for key, value in funnel.items()}
794
+ if suspected:
795
+ print(f"sensitivity excluding suspects: sessions={len(included) - len(suspected)} "
796
+ f"funnel={without_suspects}")
797
+ print(f"top paths: { {k: len(v) for k, v in sorted(paths.items(), key=lambda kv: -len(kv[1]))[:12]} }")
798
+ print(f"custom events (uniq sessions / raw events): "
799
+ f"{ {k: f'{len(v)}/{event_occurrences[k]}' for k, v in sorted(events.items(), key=lambda kv: -len(kv[1]))[:15]} }")
800
+ print("\n== stage observations (% of non-excluded sessions; not ordered conversions)")
801
+ order = ["arrival", "engaged", "unengaged_arrivals", "cta_click", "button_click", "execution_started", "execution_completed*",
802
+ "signup_page_seen", "signup_started", "signup_completed*", "post_auth_seen", "pricing_seen", "checkout_started"]
803
+ for k in order:
804
+ print(f" {k:22s} {funnel.get(k, 0):6d} ({funnel.get(k, 0) * 100 / n:.1f}%)")
805
+ print(" * execution/signup completion events are technical proxies; "
806
+ "reconcile with domain outcome and ground truth")
807
+ print(" engaged = another distinct page, non-ambient custom event, or scroll+time "
808
+ "beacons; activity proxy, not purchase intent")
809
+ print(f"repeat execution-completion sessions: {repeat_completions or '{}'}")
810
+ print("\n== activity/exposure journeys (included sessions):")
811
+ for line in interesting:
812
+ print(" " + line)
813
+ if args.json:
814
+ out = {**run, "included_sessions": len(included), "internal": len(internal), "mix": dict(mix),
815
+ "non_funnel_event_types": dict(collections.Counter(
816
+ str(row["eventType"]) for row in rows if row["eventType"] not in (1, 2))),
817
+ "exclusion_reasons": dict(exclusion_reasons),
818
+ "arrivals_by_source": dict(arrivals_by_source),
819
+ "suspected_internal": len(suspected),
820
+ "funnel_excluding_suspected": without_suspects,
821
+ "bounce_percent": bounce,
822
+ "depth": dict(depth), "funnel": dict(funnel),
823
+ "paths": {k: len(v) for k, v in paths.items()},
824
+ "events": {k: {"unique_sessions": len(v),
825
+ "raw_events": event_occurrences[k]}
826
+ for k, v in events.items()},
827
+ "event_data": event_data_status,
828
+ "metric_notes": {
829
+ "execution_completed*": {
830
+ "kind": "technical_proxy",
831
+ "reconcile_with": "domain_outcome",
832
+ },
833
+ "signup_completed*": {
834
+ "kind": "completion_event_proxy",
835
+ "reconcile_with": "signup_ground_truth",
836
+ },
837
+ "signup_started": {"kind": "intent_event_requires_verified_wiring"},
838
+ "cta_click": {"kind": "canonical_cta_event_only"},
839
+ "button_click": {"kind": "secondary_interaction_requires_property_filter"},
840
+ "signup_page_seen": {"kind": "page_visit_not_intent"},
841
+ "engaged": {
842
+ "kind": "arrival_with_activity_requires_verified_wiring",
843
+ "rule": "distinct_pages>1 OR non_ambient_event OR scroll_and_time_beacons",
844
+ "ambient_events": sorted(ENGAGE_EVENTS | EXPOSURE_EVENTS),
845
+ "commercial_intent_verified": False,
846
+ },
847
+ "post_auth_seen": {"kind": "page_visit_not_signup"},
848
+ "arrival": {"kind": "session_with_non_asset_pageview"},
849
+ "bounce_percent": {"denominator": "arrival", "numerator": "unengaged_arrivals"},
850
+ "included_sessions": {
851
+ "kind": "not_excluded_by_configured_rules",
852
+ "suspected_internal_included": True,
853
+ "human_or_customer_verified": False,
854
+ },
855
+ "journeys.timeline.path": {"redacted": True},
856
+ "journeys.session_ref": {"ephemeral_per_run": True},
857
+ "journeys.identity_refs": {"ephemeral_per_run": True},
858
+ },
859
+ "repeat_execution_completion_sessions": repeat_completions,
860
+ "journeys": journey_rows}
861
+ write_json(args.json, out)
862
+ print(f"\n(json written to {args.json})")
863
+
864
+
865
+ def main():
866
+ ap = argparse.ArgumentParser()
867
+ ap.add_argument("--site", help="exact website name/domain/id, otherwise a unique name/domain substring")
868
+ ap.add_argument("--since", help="inclusive YYYY-MM-DD in UTC (default: all time)")
869
+ ap.add_argument("--until", help="exclusive YYYY-MM-DD in UTC (default: now)")
870
+ ap.add_argument("--act-path", action="append", default=[],
871
+ help="path prefix that counts as execution start (repeatable)")
872
+ ap.add_argument("--act-start-event", action="append", default=[],
873
+ dest="act_start_event",
874
+ help="event name that counts as execution start (repeatable)")
875
+ ap.add_argument("--act-event", action="append", default=[],
876
+ help="event name that counts as technical execution completion (repeatable)")
877
+ ap.add_argument("--internal-country", action="append", default=[],
878
+ dest="internal_country",
879
+ help="verified internal geography to exclude; no default. "
880
+ "Do not infer internal traffic from country alone")
881
+ ap.add_argument("--dc-country", action="extend", nargs="*",
882
+ dest="dc_country", help="suspect countries (default DE FI); empty flag clears them")
883
+ ap.add_argument("--internal-sources", action="extend", nargs="*",
884
+ dest="internal_sources", help="synthetic sources (default smoke); empty flag clears them")
885
+ ap.add_argument("--skip-event-data", action="store_true",
886
+ help="skip Umami event-data enrichment and report it skipped")
887
+ ap.add_argument("--max-journey-events", type=int, default=100,
888
+ help="maximum filtered timeline entries printed/stored per journey (default: 100)")
889
+ ap.add_argument("--json", help="also write summary json here")
890
+ ap.add_argument("--events-json", help="save uncapped sanitized input rows for offline replay")
891
+ ap.add_argument("--from-events", help="replay a saved event snapshot/config without network access")
892
+ args = ap.parse_args()
893
+ args.dc_country = args.dc_country if args.dc_country is not None else ["DE", "FI"]
894
+ args.internal_sources = args.internal_sources if args.internal_sources is not None else ["smoke"]
895
+ if args.max_journey_events < 1:
896
+ ap.error("--max-journey-events must be at least 1")
897
+ try:
898
+ check_output_paths(args.json, args.events_json)
899
+ if args.from_events:
900
+ live_options = {
901
+ "--site", "--since", "--until", "--act-path", "--act-start-event",
902
+ "--act-event", "--internal-country", "--dc-country", "--internal-sources",
903
+ "--skip-event-data", "--max-journey-events", "--events-json",
904
+ }
905
+ if any(arg.split("=", 1)[0] in live_options for arg in sys.argv[1:]):
906
+ ap.error("--from-events uses the saved config; only --json can be added")
907
+ rows, event_data_status = read_snapshot(args.from_events, args)
908
+ analyze(rows, args, event_data_status)
909
+ return
910
+ args.window = resolve_window(args)
911
+ except (ValueError, OSError, KeyError) as error:
912
+ ap.error(str(error))
913
+ try:
914
+ rows, event_data_status = fetch_events(args)
915
+ except (ValueError, OSError, RuntimeError) as error:
916
+ ap.error(str(error))
917
+ print(f"# events fetched: {len(rows)}")
918
+ # Analyze exactly the safe dataset we retain; replay reproduces its joins too.
919
+ if args.events_json:
920
+ snapshot = make_snapshot(rows, args, event_data_status)
921
+ write_json(args.events_json, snapshot)
922
+ rows = snapshot["rows"]
923
+ analyze(rows, args, event_data_status)
924
+
925
+
926
+ if __name__ == "__main__":
927
+ main()