@egbe-ai/opencode 0.0.0-stage → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (479) hide show
  1. package/LICENSE +4 -0
  2. package/README.md +124 -2
  3. package/index.js +5 -0
  4. package/package.json +55 -4
  5. package/plugin.js +230 -0
  6. package/session-log.js +315 -0
  7. package/skills/.egbe-catalog/manifest.json +1 -0
  8. package/skills/egbe-os-analytics/SKILL.md +499 -0
  9. package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
  10. package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
  11. package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
  12. package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
  13. package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
  14. package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
  15. package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
  16. package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
  17. package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
  18. package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
  19. package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
  20. package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
  21. package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
  22. package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
  23. package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
  24. package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
  25. package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
  26. package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
  27. package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
  28. package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
  29. package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
  30. package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
  31. package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
  32. package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
  33. package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
  34. package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
  35. package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
  36. package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
  37. package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
  38. package/skills/egbe-os-db/SKILL.md +174 -0
  39. package/skills/egbe-os-deploy/SKILL.md +412 -0
  40. package/skills/egbe-os-design/SKILL.md +182 -0
  41. package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
  42. package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
  43. package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
  44. package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
  45. package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
  46. package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
  47. package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
  48. package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
  49. package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
  50. package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
  51. package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
  52. package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
  53. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
  54. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
  55. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
  56. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
  57. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
  58. package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
  59. package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
  60. package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
  61. package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
  62. package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
  63. package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
  64. package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
  65. package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
  66. package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
  67. package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
  68. package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
  69. package/skills/egbe-os-destination-readiness/references/report.md +116 -0
  70. package/skills/egbe-os-domains/SKILL.md +189 -0
  71. package/skills/egbe-os-errors/SKILL.md +259 -0
  72. package/skills/egbe-os-git/SKILL.md +275 -0
  73. package/skills/egbe-os-git/references/issues.md +33 -0
  74. package/skills/egbe-os-google-ads/SKILL.md +163 -0
  75. package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
  76. package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
  77. package/skills/egbe-os-google-ads/references/conversions.md +153 -0
  78. package/skills/egbe-os-google-ads/references/delivery.md +128 -0
  79. package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
  80. package/skills/egbe-os-google-ads/references/reporting.md +105 -0
  81. package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
  82. package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
  83. package/skills/egbe-os-google-auth/SKILL.md +285 -0
  84. package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
  85. package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
  86. package/skills/egbe-os-images/SKILL.md +163 -0
  87. package/skills/egbe-os-images/references/stock-images.md +21 -0
  88. package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
  89. package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
  90. package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
  91. package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
  92. package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
  93. package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
  94. package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
  95. package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
  96. package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
  97. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
  98. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
  99. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
  100. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
  101. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
  102. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
  103. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
  104. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
  105. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
  106. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
  107. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
  108. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
  109. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
  110. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
  111. package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
  112. package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
  113. package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
  114. package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
  115. package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
  116. package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
  117. package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
  118. package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
  119. package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
  120. package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
  121. package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
  122. package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
  123. package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
  124. package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
  125. package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
  126. package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
  127. package/skills/egbe-os-llm/SKILL.md +160 -0
  128. package/skills/egbe-os-mail/SKILL.md +180 -0
  129. package/skills/egbe-os-me/SKILL.md +119 -0
  130. package/skills/egbe-os-meta-ads/SKILL.md +114 -0
  131. package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
  132. package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
  133. package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
  134. package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
  135. package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
  136. package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
  137. package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
  138. package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
  139. package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
  140. package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
  141. package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
  142. package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
  143. package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
  144. package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
  145. package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
  146. package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
  147. package/skills/egbe-os-notifications/SKILL.md +78 -0
  148. package/skills/egbe-os-openai-ads/SKILL.md +149 -0
  149. package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
  150. package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
  151. package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
  152. package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
  153. package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
  154. package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
  155. package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
  156. package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
  157. package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
  158. package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
  159. package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
  160. package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
  161. package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
  162. package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
  163. package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
  164. package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
  165. package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
  166. package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
  167. package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
  168. package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
  169. package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
  170. package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
  171. package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
  172. package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
  173. package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
  174. package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
  175. package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
  176. package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
  177. package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
  178. package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
  179. package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
  180. package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
  181. package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
  182. package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
  183. package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
  184. package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
  185. package/skills/egbe-os-payments/SKILL.md +215 -0
  186. package/skills/egbe-os-payments/agents/openai.yaml +4 -0
  187. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
  188. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
  189. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
  190. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
  191. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
  192. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
  193. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
  194. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
  195. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
  196. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
  197. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
  198. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
  199. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
  200. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
  201. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
  202. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
  203. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
  204. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
  205. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
  206. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
  207. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
  208. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
  209. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
  210. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
  211. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
  212. package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
  213. package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
  214. package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
  215. package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
  216. package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
  217. package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
  218. package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
  219. package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
  220. package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
  221. package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
  222. package/skills/egbe-os-playbook/SKILL.md +79 -0
  223. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
  224. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
  225. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
  226. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
  227. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
  228. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
  229. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
  230. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
  231. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
  232. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
  233. package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
  234. package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
  235. package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
  236. package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
  237. package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
  238. package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
  239. package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
  240. package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
  241. package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
  242. package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
  243. package/skills/egbe-os-playbook/playbooks/product.md +67 -0
  244. package/skills/egbe-os-playbook/references/catalog.md +127 -0
  245. package/skills/egbe-os-playbook/references/changes.md +167 -0
  246. package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
  247. package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
  248. package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
  249. package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
  250. package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
  251. package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
  252. package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
  253. package/skills/egbe-os-product-auth/SKILL.md +151 -0
  254. package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
  255. package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
  256. package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
  257. package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
  258. package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
  259. package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
  260. package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
  261. package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
  262. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
  263. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
  264. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
  265. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
  266. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
  267. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
  268. package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
  269. package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
  270. package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
  271. package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
  272. package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
  273. package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
  274. package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
  275. package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
  276. package/skills/egbe-os-product-build/SKILL.md +111 -0
  277. package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
  278. package/skills/egbe-os-product-build/references/capabilities.md +25 -0
  279. package/skills/egbe-os-product-build/references/engineering.md +72 -0
  280. package/skills/egbe-os-product-build/references/execution.md +55 -0
  281. package/skills/egbe-os-product-build/references/foundations.md +93 -0
  282. package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
  283. package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
  284. package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
  285. package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
  286. package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
  287. package/skills/egbe-os-product-market-research/SKILL.md +159 -0
  288. package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
  289. package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
  290. package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
  291. package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
  292. package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
  293. package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
  294. package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
  295. package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
  296. package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
  297. package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
  298. package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
  299. package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
  300. package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
  301. package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
  302. package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
  303. package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
  304. package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
  305. package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
  306. package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
  307. package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
  308. package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
  309. package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
  310. package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
  311. package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
  312. package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
  313. package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
  314. package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
  315. package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
  316. package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
  317. package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
  318. package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
  319. package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
  320. package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
  321. package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
  322. package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
  323. package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
  324. package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
  325. package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
  326. package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
  327. package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
  328. package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
  329. package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
  330. package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
  331. package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
  332. package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
  333. package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
  334. package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
  335. package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
  336. package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
  337. package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
  338. package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
  339. package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
  340. package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
  341. package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
  342. package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
  343. package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
  344. package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
  345. package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
  346. package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
  347. package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
  348. package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
  349. package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
  350. package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
  351. package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
  352. package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
  353. package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
  354. package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
  355. package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
  356. package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
  357. package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
  358. package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
  359. package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
  360. package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
  361. package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
  362. package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
  363. package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
  364. package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
  365. package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
  366. package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
  367. package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
  368. package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
  369. package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
  370. package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
  371. package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
  372. package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
  373. package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
  374. package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
  375. package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
  376. package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
  377. package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
  378. package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
  379. package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
  380. package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
  381. package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
  382. package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
  383. package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
  384. package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
  385. package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
  386. package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
  387. package/skills/egbe-os-product-report/SKILL.md +150 -0
  388. package/skills/egbe-os-product-report/references/artifacts.md +84 -0
  389. package/skills/egbe-os-product-report/references/brief.md +34 -0
  390. package/skills/egbe-os-product-report/references/daily.md +76 -0
  391. package/skills/egbe-os-product-report/references/sources.md +68 -0
  392. package/skills/egbe-os-product-routines/SKILL.md +166 -0
  393. package/skills/egbe-os-product-routines/references/catalog.md +22 -0
  394. package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
  395. package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
  396. package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
  397. package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
  398. package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
  399. package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
  400. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
  401. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
  402. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
  403. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
  404. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
  405. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
  406. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
  407. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
  408. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
  409. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
  410. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
  411. package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
  412. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
  413. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
  414. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
  415. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
  416. package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
  417. package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
  418. package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
  419. package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
  420. package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
  421. package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
  422. package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
  423. package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
  424. package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
  425. package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
  426. package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
  427. package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
  428. package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
  429. package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
  430. package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
  431. package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
  432. package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
  433. package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
  434. package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
  435. package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
  436. package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
  437. package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
  438. package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
  439. package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
  440. package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
  441. package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
  442. package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
  443. package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
  444. package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
  445. package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
  446. package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
  447. package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
  448. package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
  449. package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
  450. package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
  451. package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
  452. package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
  453. package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
  454. package/skills/egbe-os-report-ui/SKILL.md +133 -0
  455. package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
  456. package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
  457. package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
  458. package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
  459. package/skills/egbe-os-report-ui/references/components.md +209 -0
  460. package/skills/egbe-os-research/SKILL.md +125 -0
  461. package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
  462. package/skills/egbe-os-scrape/SKILL.md +160 -0
  463. package/skills/egbe-os-search/SKILL.md +182 -0
  464. package/skills/egbe-os-search/references/google-places.md +81 -0
  465. package/skills/egbe-os-support/SKILL.md +149 -0
  466. package/skills/egbe-os-telemetry/SKILL.md +344 -0
  467. package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
  468. package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
  469. package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
  470. package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
  471. package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
  472. package/skills/egbe-os-tools/SKILL.md +180 -0
  473. package/skills/egbe-os-tools/references/theona.md +26 -0
  474. package/skills/egbe-os-wallet/SKILL.md +181 -0
  475. package/skills/egbe-setup/EGBE.md +484 -0
  476. package/skills/egbe-setup/SKILL.md +116 -0
  477. package/skills/egbe-setup/references/project.md +80 -0
  478. package/state.js +37 -0
  479. package/update.js +361 -0
@@ -0,0 +1,966 @@
1
+ """Browser evidence, approval journals and scoped funnel projections (stdlib only).
2
+
3
+ This module never opens a page, submits a form, invents a screen or grants
4
+ approval. Private contact data live in runtime; public rows contain references.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import copy
9
+ import hashlib
10
+ import importlib.util
11
+ import json
12
+ import re
13
+ from collections import Counter, defaultdict
14
+ from datetime import date, datetime, timedelta, timezone
15
+ from pathlib import Path
16
+ from urllib.parse import parse_qsl, quote, quote_plus, unquote, unquote_plus, urlencode, urlsplit, urlunsplit
17
+
18
+ _features_spec = importlib.util.spec_from_file_location(__name__ + "_features", Path(__file__).with_name("funnel_features.py"))
19
+ FEATURES = importlib.util.module_from_spec(_features_spec)
20
+ _features_spec.loader.exec_module(FEATURES)
21
+
22
+ MECHANICS = json.loads((Path(__file__).resolve().parent.parent /
23
+ "references/contracts/walk-mechanics.v2.json").read_text())
24
+ SCHEMA_VERSION = 2
25
+ CONTRACT = MECHANICS["contract"]
26
+ SCREEN_KINDS = tuple(MECHANICS["screen_kinds"])
27
+ ADVANCE_KINDS = tuple(MECHANICS["advance_types"])
28
+ STATUSES = tuple(MECHANICS["research_statuses"])
29
+ STOP_REASONS = tuple(MECHANICS["stop_reasons"])
30
+ VIEWPORT = "390x844"
31
+ MAX_SCREENS_PER_ATTEMPT = MECHANICS["max_screens_per_attempt"]
32
+ PENDING_STATUSES = {"queued", "in_progress", "needs_approval", "needs_email"}
33
+ PERMITTED_SUBMISSIONS = set(MECHANICS["submission_kinds"])
34
+ TRACKING_NAMES = {"fbclid", "gclid", "gclsrc", "gbraid", "wbraid", "dclid", "msclkid", "ttclid", "li_fat_id"}
35
+
36
+
37
+ class FunnelError(ValueError):
38
+ pass
39
+
40
+
41
+ def canonical_hash(value) -> str:
42
+ return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"),
43
+ ensure_ascii=False).encode()).hexdigest()
44
+
45
+
46
+ def identity(prefix: str, value) -> str:
47
+ return prefix + "_" + canonical_hash(value)[:20]
48
+
49
+
50
+ def url_identity(address: str, confirmed_tracking=()) -> str:
51
+ """Preserve meaningful query/fragment/order. Even UTM removal is opt-in.
52
+
53
+ Tracking may change an offer. Equivalence needs browser observations, not
54
+ just a familiar parameter name. Userinfo and non-web destinations fail.
55
+ """
56
+ if not isinstance(address, str):
57
+ raise FunnelError("destination must be a public HTTP(S) URL")
58
+ parts = urlsplit(address.strip())
59
+ scheme = parts.scheme.lower()
60
+ if scheme not in {"https", "http"} or not parts.hostname or parts.username or parts.password or re.search(r"[\x00-\x20]", address.strip()):
61
+ raise FunnelError("destination must be a public HTTP(S) URL without credentials")
62
+ try:
63
+ port = parts.port
64
+ except ValueError as exc:
65
+ raise FunnelError("invalid destination port") from exc
66
+ host = parts.hostname.lower()
67
+ host = "[" + host + "]" if ":" in host else host
68
+ netloc = host + (f":{port}" if port and port != (443 if scheme == "https" else 80) else "")
69
+ remove = set(confirmed_tracking)
70
+ if any(not (name.lower().startswith("utm_") or name.lower() in TRACKING_NAMES) for name in remove):
71
+ raise FunnelError("only confirmed tracking parameters can be removed")
72
+ query = parts.query
73
+ if remove:
74
+ query = urlencode([(k, v) for k, v in parse_qsl(query, keep_blank_values=True) if k not in remove])
75
+ return urlunsplit((scheme, netloc, parts.path or "/", query, parts.fragment))
76
+
77
+
78
+ def duration_proven(row: dict, policy: dict) -> bool:
79
+ return (type(row.get("served_days")) is int and row["served_days"] >= 0
80
+ and row.get("age_basis") == "platform_dates" and row.get("duration_scope") in ("country", "global")
81
+ and bool(row.get("country")))
82
+
83
+
84
+ def campaign_proof(members: list[dict], policy: dict) -> dict:
85
+ """Preserve dated and current observations for this exact campaign.
86
+
87
+ Native intervals and activity remain separate; neither establishes success.
88
+ """
89
+ proving = [copy.deepcopy(m) for m in members if duration_proven(m, policy)]
90
+ def country_current(member):
91
+ if not member.get("country"):
92
+ return False
93
+ if member.get("current_in_country") is not None:
94
+ return member["current_in_country"] is True
95
+ if member.get("running_at_capture") is not True:
96
+ return False
97
+ observations = member.get("source_observations") or []
98
+ if observations:
99
+ return native_current(observations, member["country"], policy)
100
+ if member.get("activity_scope") in ("country", "global") and member.get("activity_country") == member["country"]:
101
+ return True
102
+ # Compatibility with native-dated 04 records; a root consumer also
103
+ # verifies the pinned 03 activity observations before replication.
104
+ return member.get("duration_scope") in ("country", "global") and member.get("age_basis") == "platform_dates"
105
+ running = [copy.deepcopy(m) for m in members if country_current(m)]
106
+ status = "active" if running else "stopped" if any(
107
+ m.get("running_at_capture") is False for m in members) else "unknown"
108
+ return {"observation_status": status, "dated_refs": proving,
109
+ "running_refs": running, "source_members": copy.deepcopy(members),
110
+ "policy_sha256": canonical_hash(policy)}
111
+
112
+
113
+ def native_current(observations: list[dict], country: str, policy: dict) -> bool:
114
+ matched = [o for o in observations if o.get("country") == country]
115
+ statuses = {o.get("activity_status", o.get("status")) for o in matched}
116
+ if statuses & {"stopped", "inactive", "ended", "archived"} and statuses & {"active", "recent"}:
117
+ return False
118
+ for observation in matched:
119
+ # Activity seen in this country's query (global provider flag) or native
120
+ # country activity both count as current use in the market.
121
+ if observation.get("activity_scope") not in ("country", "global") or observation.get("activity_country") != country:
122
+ continue
123
+ status = observation.get("activity_status", observation.get("status"))
124
+ if status == "active":
125
+ return True
126
+ return False
127
+
128
+
129
+ def bind_activity_provenance(campaigns: list[dict], footprint: dict, policy: dict) -> None:
130
+ """Current activity in the campaign's country is independent of date availability."""
131
+ ads = {r.get("ad_id"): r for r in footprint.get("rows", []) if r.get("type") == "channel_ad"}
132
+ if not ads:
133
+ return
134
+ for campaign in campaigns:
135
+ for member in campaign["source_members"]:
136
+ ad = ads.get(member.get("footprint_ad_id"))
137
+ observations = []
138
+ if ad and ad.get("domain") == campaign["domain"]:
139
+ for observation in ad.get("observations", []):
140
+ placement = observation.get("placement")
141
+ channel = ad.get("channel") or ad.get("platform")
142
+ if placement in {"youtube", "youtube_ads"}:
143
+ channel = "youtube_ads"
144
+ elif placement in {"display", "display_ads", "display_native"}:
145
+ channel = "display_native"
146
+ if channel == campaign["channel"] and observation.get("country") == campaign.get("country"):
147
+ observations.append(copy.deepcopy(observation))
148
+ member["source_observations"] = observations
149
+ member["current_in_country"] = native_current(observations, campaign.get("country"), policy)
150
+ campaign.update(campaign_proof(campaign["source_members"], policy))
151
+
152
+
153
+ def destination_enrichments(document: dict, creatives: dict[str, dict]) -> dict[str, str]:
154
+ """A candidate URL is not a verified destination until tied to the ad."""
155
+ if not document:
156
+ return {}
157
+ if document.get("schema_version") != 1:
158
+ raise FunnelError("destination enrichment requires schema_version 1")
159
+ result = {}
160
+ for item in document.get("destinations", []):
161
+ cid = item.get("creative_id")
162
+ if cid not in creatives:
163
+ raise FunnelError("destination enrichment names an unselected creative")
164
+ if item.get("status") != "verified":
165
+ continue
166
+ if not item.get("source_ref") or not item.get("verified_on") or item.get("relation") not in {
167
+ "native_ad_destination", "observed_ad_cta", "provider_ad_history"}:
168
+ raise FunnelError("verified destination needs an explicit ad link and source")
169
+ try:
170
+ verified_on = date.fromisoformat(item["verified_on"])
171
+ except (TypeError, ValueError) as exc:
172
+ raise FunnelError("destination verified_on must be an observation date") from exc
173
+ if verified_on > date.today():
174
+ raise FunnelError("destination verification cannot be in the future")
175
+ result[cid] = item.get("url")
176
+ return result
177
+
178
+
179
+ def prepare_campaigns(selection: dict, creative_document: dict, policy: dict,
180
+ campaign_as_of: str, enrichment: dict | None = None) -> tuple[list[dict], list[dict]]:
181
+ if selection.get("artifact") != "creative_selection.v1" or selection.get("schema_version") != 1:
182
+ raise FunnelError("05 requires the canonical creative_selection.v1 manifest from 04")
183
+ if creative_document.get("schema_version") != 2:
184
+ raise FunnelError("legacy Creative Patterns requires a refresh; no automatic funnel proof migration")
185
+ creatives = {r["creative_id"]: r for r in creative_document.get("rows", []) if r.get("type") == "creative"}
186
+ selected = selection.get("selected_creatives")
187
+ if not isinstance(selected, list):
188
+ raise FunnelError("selection.selected_creatives must be an array")
189
+ chosen_ids = {r.get("creative_id") for r in selected}
190
+ enriched = destination_enrichments(enrichment or {}, {k: v for k, v in creatives.items() if k in chosen_ids})
191
+ groups: dict[tuple, dict] = {}
192
+ gaps = []
193
+ normalized_urls = {}
194
+
195
+ def normalized_destination(creative, address):
196
+ cid = creative["creative_id"]
197
+ if cid not in normalized_urls:
198
+ try:
199
+ normalized_urls[cid] = (url_identity(address), None)
200
+ except (FunnelError, ValueError) as exc:
201
+ normalized_urls[cid] = (None, str(exc))
202
+ return normalized_urls[cid]
203
+
204
+ for rank, ref in enumerate(selected, 1):
205
+ cid = ref.get("creative_id")
206
+ creative = creatives.get(cid)
207
+ if not creative:
208
+ raise FunnelError(f"selected creative {cid!r} is absent from the pinned 04 corpus")
209
+ for field in ("domain", "country", "channel"):
210
+ if ref.get(field) != creative.get(field):
211
+ raise FunnelError(f"selection scope differs from corpus for {cid}: {field}")
212
+ if creative.get("relevance") != "relevant" or creative.get("language_status") != "allowed" or creative.get("language_allowed") is not True:
213
+ raise FunnelError(f"selected creative {cid} is not an eligible decided 04 creative")
214
+ address = enriched.get(cid) or creative.get("landing_url") or ref.get("landing_url")
215
+ if not address:
216
+ attempted = next((r for r in (enrichment or {}).get("destinations", []) if r.get("creative_id") == cid), {})
217
+ gaps.append({"type": "coverage", "status": "destination_unknown", "detail": "Selected creative has no verified ad destination",
218
+ "creative_ids": [cid], "domain": ref["domain"], "country": ref.get("country"),
219
+ "channel": ref["channel"], "source": "creative_selection", "tier": "C"})
220
+ if attempted:
221
+ gaps[-1].update(detail=attempted.get("detail") or attempted.get("reason") or gaps[-1]["detail"],
222
+ lookup_status=attempted.get("status"), source_ref=attempted.get("source_ref"))
223
+ continue
224
+ normalized, error = normalized_destination(creative, address)
225
+ if error:
226
+ gaps.append({"type": "coverage", "status": "destination_unknown", "detail": "Selected creative destination is invalid: " + error,
227
+ "creative_ids": [cid], "domain": ref["domain"], "country": ref.get("country"), "channel": ref["channel"],
228
+ "source": "creative_selection", "lookup_status": "invalid_destination", "tier": "C"})
229
+ continue
230
+ key = (ref["domain"], ref.get("country"), ref["channel"], normalized)
231
+ if key not in groups:
232
+ campaign_id = identity("campaign", key)
233
+ # Conservative default: a cross-channel walk is separate until
234
+ # equivalence is explicitly established in browser observations.
235
+ groups[key] = {"type": "campaign", "campaign_id": campaign_id,
236
+ "funnel_id": identity("funnel", key), "rank": rank,
237
+ "domain": ref["domain"], "competitor_tier": ref.get("competitor_tier", "unknown"),
238
+ "role": "competitor", "country": ref.get("country"), "channel": ref["channel"],
239
+ "address": normalized, "original_urls": [], "creative_ids": [], "creatives": [],
240
+ "campaign_as_of": campaign_as_of, "members": []}
241
+ group = groups[key]
242
+ if cid in group["creative_ids"]:
243
+ continue
244
+ group["creative_ids"].append(cid)
245
+ group["original_urls"].append(address)
246
+ group["creatives"].append(copy.deepcopy(creative))
247
+ campaigns = list(groups.values())
248
+ # Include non-top ads only when they have the same verified campaign
249
+ # destination/scope. They may support longevity, never reorder selection.
250
+ for group in campaigns:
251
+ scope = (group["domain"], group.get("country"), group["channel"])
252
+ members = []
253
+ for creative in creatives.values():
254
+ if (creative.get("domain"), creative.get("country"), creative.get("channel")) != scope:
255
+ continue
256
+ if creative.get("relevance") != "relevant" or creative.get("language_status") != "allowed" or creative.get("language_allowed") is not True:
257
+ continue
258
+ address = enriched.get(creative["creative_id"]) or creative.get("landing_url")
259
+ if not address:
260
+ continue
261
+ normalized, error = normalized_destination(creative, address)
262
+ if error or normalized != group["address"]:
263
+ continue
264
+ members.append({k: copy.deepcopy(creative.get(k)) for k in (
265
+ "creative_id", "native_ad_id", "footprint_ad_id", "source_members", "domain", "country", "countries",
266
+ "channel", "competitor_tier", "relevance", "language_status", "language_allowed", "duration_scope", "age_basis", "served_days",
267
+ "running_at_capture", "first_seen", "last_seen", "landing_url", "source", "verified_by",
268
+ "activity_scope", "activity_country", "source_observations", "current_in_country")})
269
+ group.pop("members")
270
+ group.update(campaign_proof(members, policy))
271
+ group["destination_refs"] = []
272
+ for creative in group["creatives"]:
273
+ cid = creative["creative_id"]
274
+ ref = next((r for r in (enrichment or {}).get("destinations", []) if r.get("creative_id") == cid and r.get("status") == "verified"), None)
275
+ group["destination_refs"].append(copy.deepcopy(ref) if ref else {
276
+ "creative_id": cid, "url": creative.get("landing_url"), "status": "verified",
277
+ "relation": "corpus_destination", "source_ref": creative.get("source") or "creative_patterns"})
278
+ for cell in selection.get("cells", []):
279
+ if cell.get("selected_count", len(cell.get("creative_ids", []))) == 0:
280
+ gaps.append({"type": "coverage", "status": "no_selected_campaign", "detail": "04 has no eligible selected creative in this cell",
281
+ "domain": cell.get("domain"), "channel": cell.get("channel"), "country": cell.get("country"),
282
+ "creative_ids": [], "source": "creative_selection", "tier": "C"})
283
+ # Preserve missing-source/channel coverage rather than implying that an
284
+ # omitted selection cell means this channel has no advertising.
285
+ selected_scopes = {(r.get("domain"), r.get("country"), r.get("channel")) for r in selected}
286
+ for coverage in creative_document.get("rows", []):
287
+ scope = (coverage.get("domain"), coverage.get("country"), coverage.get("channel"))
288
+ if coverage.get("type") == "coverage" and scope not in selected_scopes and coverage.get("channel"):
289
+ gaps.append({"type": "coverage", "status": "no_available_campaign", "domain": scope[0], "country": scope[1],
290
+ "channel": scope[2], "detail": coverage.get("detail") or coverage.get("status") or "04 source unavailable",
291
+ "upstream_status": coverage.get("status"), "source": "creative_patterns_coverage", "tier": "C"})
292
+ return campaigns, gaps
293
+
294
+
295
+ def allocate_identities(registry: dict | None, mailbox: str, campaigns: list[dict]) -> dict:
296
+ registry = copy.deepcopy(registry or {"schema_version": 1, "mailbox": mailbox, "identities": []})
297
+ if registry.get("schema_version") != 1 or registry.get("mailbox") != mailbox:
298
+ raise FunnelError("identity registry mailbox changed; preserve existing aliases before resuming")
299
+ if not re.fullmatch(r"[^\s@+]+@[^\s@]+", mailbox):
300
+ raise FunnelError("research mailbox must be a base email address without an alias")
301
+ identities = registry.get("identities")
302
+ if not isinstance(identities, list):
303
+ raise FunnelError("identity registry identities must be an array")
304
+ local, host = mailbox.split("@", 1)
305
+ reserved = {row.get("alias") for row in identities}
306
+ by_funnel = {row.get("funnel_id"): row for row in identities if row.get("funnel_id")}
307
+ for campaign in campaigns:
308
+ fid = campaign["funnel_id"]
309
+ if fid in by_funnel:
310
+ continue
311
+ n = 1
312
+ while f"{local}+{n:02d}@{host}" in reserved:
313
+ n += 1
314
+ alias = f"{local}+{n:02d}@{host}"
315
+ row = {"identity_id": identity("identity", fid), "domain": campaign["domain"], "funnel_id": fid,
316
+ "alias": alias, "kind": "funnel"}
317
+ identities.append(row)
318
+ reserved.add(alias)
319
+ by_funnel[fid] = row
320
+ return registry
321
+
322
+
323
+ def apply_copy_policy(campaigns: list[dict], footprint: dict) -> None:
324
+ """Describe selected research evidence, not permission to copy or launch."""
325
+ for campaign in campaigns:
326
+ campaign["research_comparable"] = bool(campaign.get("source_members"))
327
+
328
+
329
+ def apply_equivalences(campaigns: list[dict], document: dict, workspace: Path) -> list[dict]:
330
+ """Optional reuse only after actual same-scope browser equivalence captures.
331
+
332
+ Campaign IDs/URLs/proof remain separate. The first stable physical ID is
333
+ reused, so prior alias reservations remain reserved. Cross-channel/market
334
+ evidence is kept separate rather than projected by an equivalence flag.
335
+ """
336
+ if not document:
337
+ return []
338
+ if document.get("schema_version") != 1:
339
+ raise FunnelError("equivalences requires schema_version 1")
340
+ by_id = {c["campaign_id"]: c for c in campaigns}
341
+ used, assets = set(), []
342
+ for equivalence in document.get("equivalences", []):
343
+ ids = equivalence.get("campaign_ids", [])
344
+ if not equivalence.get("equivalence_id") or len(ids) < 2 or len(set(ids)) != len(ids) or any(cid not in by_id or cid in used for cid in ids):
345
+ raise FunnelError("equivalence requires unique selected campaign IDs, used only once")
346
+ group = [by_id[cid] for cid in ids]
347
+ if len({(c["domain"], c.get("country"), c["channel"]) for c in group}) != 1:
348
+ raise FunnelError("equivalence cannot merge different competitors, countries or channels")
349
+ if set(equivalence.get("addresses", [])) != {c["address"] for c in group}:
350
+ raise FunnelError("equivalence addresses must match the concrete campaign destinations")
351
+ verification = equivalence.get("verification", {})
352
+ if any(verification.get(key) is not True for key in ("country", "content", "route")) or not equivalence.get("source_ref") or not equivalence.get("verified_on"):
353
+ raise FunnelError("equivalence needs observed country/content/route verification and source")
354
+ shots = [resolve_asset(workspace, a) for a in equivalence.get("screenshots", [])]
355
+ if len(shots) < len(group):
356
+ raise FunnelError("equivalence requires browser captures for each original destination")
357
+ if any(shot.get("contains_private_data") is True for shot in shots):
358
+ raise FunnelError("equivalence captures must be safe public browser observations without private contact pixels")
359
+ assets.extend(shots)
360
+ primary = min(group, key=lambda c: c["rank"])
361
+ ref = {"equivalence_id": equivalence["equivalence_id"], "campaign_ids": ids,
362
+ "addresses": equivalence["addresses"], "source_ref": equivalence["source_ref"],
363
+ "verified_on": equivalence["verified_on"], "screenshots": shots}
364
+ for c in group:
365
+ c["funnel_id"] = primary["funnel_id"]
366
+ c["equivalence_ref"] = ref
367
+ used.update(ids)
368
+ return assets
369
+
370
+
371
+ def resolve_asset(workspace: Path, screenshot: dict) -> dict:
372
+ if not isinstance(screenshot, dict) or not screenshot.get("path"):
373
+ raise FunnelError("every observed screen requires a screenshot object with path and sha256")
374
+ path = Path(screenshot["path"])
375
+ path = path if path.is_absolute() else workspace / path
376
+ try:
377
+ path = path.resolve(strict=True)
378
+ relative = path.relative_to(workspace.resolve())
379
+ except (OSError, ValueError) as exc:
380
+ raise FunnelError("screenshot must be an existing file inside the research workspace") from exc
381
+ data = path.read_bytes()
382
+ if not (data.startswith(b"\x89PNG\r\n\x1a\n") or data.startswith(b"\xff\xd8\xff")
383
+ or data.startswith((b"GIF87a", b"GIF89a")) or (data[:4] == b"RIFF" and data[8:12] == b"WEBP")):
384
+ raise FunnelError("screenshot is not a supported raster image")
385
+ digest = hashlib.sha256(data).hexdigest()
386
+ if screenshot.get("sha256") != digest:
387
+ raise FunnelError("screenshot hash mismatch; refresh the observation")
388
+ return {**screenshot, "path": relative.as_posix(), "sha256": digest}
389
+
390
+
391
+ def form_fingerprint(form: dict) -> str:
392
+ return canonical_hash({k: form.get(k) for k in (
393
+ "form_id", "revision", "funnel_id", "path_id", "campaign_ids", "screen_id", "url", "purpose", "fields", "alias")})
394
+
395
+
396
+ def validate_append_only(previous: dict, plans: dict, approvals: dict, journal: dict) -> dict:
397
+ current = {"schema_version": 2, "forms": plans.get("forms", []),
398
+ "approvals": approvals.get("approvals", []), "events": journal.get("events", [])}
399
+ for key in ("forms", "approvals", "events"):
400
+ prior = previous.get(key, [])
401
+ if current[key][:len(prior)] != prior:
402
+ raise FunnelError("approval/submission history is append-only; preserve revisions and resolve outcomes, never erase attempts")
403
+ return current
404
+
405
+
406
+ def _timestamp(value, name: str) -> str:
407
+ try:
408
+ parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
409
+ except (TypeError, ValueError) as exc:
410
+ raise FunnelError(f"{name} must be an ISO timestamp with timezone") from exc
411
+ if parsed.tzinfo is None or parsed > datetime.now(timezone.utc) + timedelta(minutes=5):
412
+ raise FunnelError(f"{name} must have a timezone and cannot be in the future")
413
+ return str(value)
414
+
415
+
416
+ def validate_journals(plans: dict, approvals: dict, journal: dict, candidates: list[dict], registry: dict | None) -> tuple[dict, dict]:
417
+ """Validate exact form revisions and intent-before-outcome event sequences."""
418
+ for label, doc in (("approval-plan", plans), ("approvals", approvals), ("submissions", journal)):
419
+ if doc.get("schema_version") != 2:
420
+ raise FunnelError(f"{label} requires schema_version 2")
421
+ by_funnel = {c["funnel_id"]: c for c in candidates}
422
+ alias_by_funnel = {r.get("funnel_id"): r.get("alias") for r in (registry or {}).get("identities", [])}
423
+ forms = {}
424
+ for form in plans.get("forms", []):
425
+ key = (form.get("form_id"), form.get("revision"))
426
+ if not key[0] or type(key[1]) is not int or key[1] < 1 or key in forms:
427
+ raise FunnelError("form_id/revision must be unique and revision a positive integer")
428
+ candidate = by_funnel.get(form.get("funnel_id"))
429
+ if not candidate or candidate.get("role") == "own":
430
+ raise FunnelError("planned submission must belong to a selected competitor funnel")
431
+ if form.get("purpose") not in PERMITTED_SUBMISSIONS or not isinstance(form.get("fields"), dict) or not form["fields"]:
432
+ raise FunnelError("only exact lead_form/quiz/webinar_registration fields can be approved")
433
+ if not form.get("path_id") or not form.get("screen_id") or not form.get("campaign_ids"):
434
+ raise FunnelError("planned form requires path, screen and campaign references")
435
+ if not set(form["campaign_ids"]) <= set(candidate.get("campaign_ids", [])):
436
+ raise FunnelError("planned form campaign scope mismatch")
437
+ url_identity(form.get("url"))
438
+ alias = form.get("alias")
439
+ if not alias or alias != alias_by_funnel.get(form["funnel_id"]):
440
+ raise FunnelError("planned form must use the funnel's stable research alias")
441
+ for field, value in form["fields"].items():
442
+ normalized = field.casefold()
443
+ if any(word in normalized for word in ("password", "card_number", "credit_card", "cvv", "payment", "captcha", "sms_code")):
444
+ raise FunnelError("account and card/payment fields cannot be approved")
445
+ if "email" in normalized and value != alias:
446
+ raise FunnelError("email must match the stable funnel alias")
447
+ if "phone" in normalized or "mobile" in normalized or normalized in {"tel", "telephone"}:
448
+ digits = re.sub(r"\D", "", str(value))
449
+ if not (len(digits) in (10, 11) and (len(digits) != 11 or digits.startswith("1"))
450
+ and re.fullmatch(r"55501\d\d", digits[-7:])):
451
+ raise FunnelError("phone must be a full US number in fiction range 555-0100..0199")
452
+ if form.get("form_sha256") not in (None, form_fingerprint(form)):
453
+ raise FunnelError("planned form fingerprint mismatch")
454
+ forms[key] = {**form, "form_sha256": form_fingerprint(form)}
455
+ approved = {}
456
+ for approval in approvals.get("approvals", []):
457
+ aid = approval.get("approval_id")
458
+ if not aid or aid in approved or approval.get("confirmed_by") != "human_operator" or not str(approval.get("source_ref") or "").startswith("chat:"):
459
+ raise FunnelError("approval needs a unique ID and explicit human chat confirmation reference")
460
+ _timestamp(approval.get("approved_at"), "approved_at")
461
+ matches = set()
462
+ for ref in approval.get("forms", []):
463
+ key = (ref.get("form_id"), ref.get("revision"))
464
+ if key not in forms or ref.get("form_sha256") != forms[key]["form_sha256"]:
465
+ raise FunnelError("approval does not match the exact form revision/data")
466
+ matches.add(key)
467
+ if not matches:
468
+ raise FunnelError("approval has no exact forms")
469
+ approved[aid] = {"forms": matches, "approved_at": approval["approved_at"]}
470
+ intents = {}
471
+ results = {}
472
+ fingerprints = {}
473
+ events = set()
474
+ for event in journal.get("events", []):
475
+ eid = event.get("event_id")
476
+ if not eid or eid in events:
477
+ raise FunnelError("submission event IDs must be unique")
478
+ events.add(eid)
479
+ timestamp = _timestamp(event.get("occurred_at"), "submission occurred_at")
480
+ kind = event.get("type")
481
+ if kind == "intent":
482
+ key = (event.get("form_id"), event.get("revision"))
483
+ aid = event.get("approval_id")
484
+ if key not in forms or aid not in approved or key not in approved[aid]["forms"]:
485
+ raise FunnelError("submission intent has no exact human approval")
486
+ form = forms[key]
487
+ if event.get("form_sha256") != form["form_sha256"] or event.get("fields") != form["fields"]:
488
+ raise FunnelError("submission data differ from the approved revision")
489
+ if datetime.fromisoformat(timestamp.replace("Z", "+00:00")) < datetime.fromisoformat(approved[aid]["approved_at"].replace("Z", "+00:00")):
490
+ raise FunnelError("submission intent predates human approval")
491
+ fingerprint = form["form_sha256"]
492
+ previous = fingerprints.get(fingerprint)
493
+ if previous and results.get(previous, {}).get("outcome") != "not_submitted":
494
+ raise FunnelError("duplicate or uncertain submission: inspect outcome before retrying")
495
+ if previous and event.get("retry_authorization_ref") != aid:
496
+ raise FunnelError("retry after confirmed non-submission requires explicit approval reference")
497
+ if previous and aid == intents[previous]["approval_id"]:
498
+ raise FunnelError("retry requires a new human approval, not the original form list confirmation")
499
+ fingerprints[fingerprint] = eid
500
+ intents[eid] = {**event, "form": form}
501
+ elif kind in {"outcome", "resolution"}:
502
+ ref = event.get("submission_ref")
503
+ if ref not in intents or not event.get("source_ref") or event.get("outcome") not in {"submitted", "not_submitted", "unknown"}:
504
+ raise FunnelError("submission outcome needs an earlier intent and observed source")
505
+ if ref in results and kind != "resolution":
506
+ raise FunnelError("duplicate outcome; use an explicit resolution event")
507
+ if datetime.fromisoformat(timestamp.replace("Z", "+00:00")) < datetime.fromisoformat(intents[ref]["occurred_at"].replace("Z", "+00:00")):
508
+ raise FunnelError("submission outcome predates its intent")
509
+ results[ref] = event
510
+ else:
511
+ raise FunnelError("submission event type must be intent, outcome or resolution")
512
+ return intents, results
513
+
514
+
515
+ def validate_walks(document: dict, candidates: list[dict], workspace: Path, inputs_sha256: str,
516
+ intents: dict | None = None, outcomes: dict | None = None) -> list[dict]:
517
+ if document.get("schema_version") != 2 or document.get("contract") != CONTRACT:
518
+ raise FunnelError("legacy walks require fresh browser evidence; expected funnel-walk.v2")
519
+ if document.get("inputs_sha256") != inputs_sha256:
520
+ raise FunnelError("walk inputs changed; inspect and rebind observations without repeating submissions")
521
+ by_funnel = {c["funnel_id"]: c for c in candidates}
522
+ seen = set()
523
+ valid = []
524
+ intents, outcomes = intents or {}, outcomes or {}
525
+ for source in document.get("walks", []):
526
+ walk = copy.deepcopy(source)
527
+ fid, pid = walk.get("funnel_id"), walk.get("path_id")
528
+ if fid not in by_funnel or not pid or (fid, pid) in seen:
529
+ raise FunnelError("every path needs a unique selected funnel_id/path_id")
530
+ seen.add((fid, pid))
531
+ candidate = by_funnel[fid]
532
+ if set(walk.get("campaign_ids", [])) != set(candidate.get("campaign_ids", [])):
533
+ raise FunnelError("walk campaign references differ from its selected funnel")
534
+ if walk.get("role", candidate["role"]) != candidate["role"]:
535
+ raise FunnelError("own/competitor walk role mismatch")
536
+ walk["role"] = candidate["role"]
537
+ walk["domain"] = candidate["domain"]
538
+ walk["country"] = candidate.get("country")
539
+ walk["channel"] = candidate.get("channel")
540
+ if candidate.get("equivalence_ref"):
541
+ walk["equivalence_ref"] = copy.deepcopy(candidate["equivalence_ref"])
542
+ if url_identity(walk.get("address")) != url_identity(candidate["address"]):
543
+ raise FunnelError("walk entry URL differs from the chosen campaign destination")
544
+ if walk.get("viewport") != VIEWPORT or walk.get("status") not in STATUSES:
545
+ raise FunnelError("walk requires the mobile viewport and a closed research status")
546
+ if walk.get("stop_reason") not in STOP_REASONS:
547
+ raise FunnelError("walk requires an explicit permitted stop reason")
548
+ if walk["status"] == "completed" and walk["stop_reason"] != "completed":
549
+ raise FunnelError("completed path needs a reached outcome, not a blocked boundary")
550
+ if walk["status"] in PENDING_STATUSES and walk["stop_reason"] == "completed":
551
+ raise FunnelError("pending path cannot claim completion")
552
+ if walk.get("observed_outcome") not in {"lead_submitted", "webinar_registered", "quiz_completed", "result_observed", "public_content_end", "not_observed"}:
553
+ raise FunnelError("observed_outcome must be a permitted reached outcome or not_observed")
554
+ if walk["status"] == "completed" and walk["observed_outcome"] == "not_observed":
555
+ raise FunnelError("completed path needs a actually reached observable outcome")
556
+ if not walk.get("persona_id") or not isinstance(walk.get("observed_market"), dict):
557
+ raise FunnelError("walk requires persona_id and observed_market (unknown values permitted)")
558
+ walked_on = walk.get("walked_on")
559
+ try:
560
+ observed_date = date.fromisoformat(walked_on)
561
+ except (TypeError, ValueError) as exc:
562
+ raise FunnelError("walked_on must be an observation date") from exc
563
+ if observed_date > date.today():
564
+ raise FunnelError("browser observation cannot be in the future")
565
+ if not isinstance(walk.get("unknown"), list):
566
+ raise FunnelError("walk requires an explicit unknown list")
567
+ screens = walk.get("screens")
568
+ if not isinstance(screens, list) or not screens:
569
+ raise FunnelError("an observed walk needs at least one real screen")
570
+ attempts = Counter()
571
+ screen_ids = set()
572
+ for n, screen in enumerate(screens, 1):
573
+ sid = screen.get("screen_id")
574
+ if not sid or sid in screen_ids:
575
+ raise FunnelError("screen_id must be unique within the path")
576
+ screen_ids.add(sid)
577
+ screen["n"] = n
578
+ kind = str(screen.get("kind") or "").split(":", 1)[0].strip()
579
+ if kind not in SCREEN_KINDS or screen.get("advance") not in ADVANCE_KINDS:
580
+ raise FunnelError("screen kind/advance must follow funnel-walk.v2")
581
+ if screen.get("branch_role") not in MECHANICS["branch_roles"] or not screen.get("attempt_id"):
582
+ raise FunnelError("screen requires main/auxiliary branch_role and attempt_id")
583
+ attempts[screen["attempt_id"]] += 1
584
+ if attempts[screen["attempt_id"]] > MAX_SCREENS_PER_ATTEMPT:
585
+ raise FunnelError("40 screens per attempt; save a budget stop and resume in a new attempt")
586
+ screen["screenshot"] = resolve_asset(workspace, screen.get("screenshot"))
587
+ if screen.get("display_screenshot"):
588
+ display = resolve_asset(workspace, screen["display_screenshot"])
589
+ if display.get("source_sha256") != screen["screenshot"]["sha256"] or display.get("derivation") != "privacy_redaction":
590
+ raise FunnelError("display screenshot must explicitly reference the raw observation and privacy redaction")
591
+ screen["source_screenshot"] = screen["screenshot"]
592
+ screen["screenshot"] = display
593
+ elif screen["screenshot"].get("contains_private_data") is True:
594
+ raise FunnelError("private contact pixels need a separately declared privacy-redacted display asset")
595
+ if "screenshots" in screen:
596
+ screen["screenshots"] = [resolve_asset(workspace, asset) for asset in screen["screenshots"]]
597
+ if any(asset.get("contains_private_data") is True for asset in screen["screenshots"]):
598
+ raise FunnelError("additional public screenshots cannot contain private contact pixels")
599
+ url_identity(screen.get("url"))
600
+ if type(screen.get("headline_words")) is not int or screen["headline_words"] < 0:
601
+ raise FunnelError("headline_words must be a nonnegative integer")
602
+ if not isinstance(screen.get("headline"), str) or not isinstance(screen.get("notes", ""), str):
603
+ raise FunnelError("screen content must preserve observed text")
604
+ if type(screen.get("options")) is not int and screen.get("options") != "none":
605
+ raise FunnelError("options must be an integer or none")
606
+ if type(screen.get("options")) is int and screen["options"] < 0:
607
+ raise FunnelError("options cannot be negative")
608
+ if screen.get("result_type") not in {None, "teaser", "final", "not_observed", "absent", "not_applicable"}:
609
+ raise FunnelError("result_type distinguishes teaser and final result")
610
+ if kind == "teaser" and screen.get("result_type") != "teaser":
611
+ raise FunnelError("teaser must not be encoded as a final result")
612
+ if kind == "result" and screen.get("result_type") != "final":
613
+ raise FunnelError("result screen needs a visually observed final result")
614
+ if screen.get("source", "browser") != "browser":
615
+ raise FunnelError("website screen must be a browser observation; page config/email is a separate source")
616
+ for number in screen.get("amounts", []):
617
+ if number.get("kind") not in {"price", "discount", "financing", "salary_claim", "other"} or not number.get("context"):
618
+ raise FunnelError("observed amounts need a semantic kind and context")
619
+ if not number.get("text"):
620
+ raise FunnelError("amounts preserve the actual displayed browser text")
621
+ action = screen.get("action")
622
+ if action is None:
623
+ action = {}
624
+ if not isinstance(action, dict):
625
+ raise FunnelError("screen action must be an object or null")
626
+ if action.get("type") is not None and not isinstance(action["type"], str):
627
+ raise FunnelError("screen action type must be a string")
628
+ screen["action"] = action
629
+ if action.get("type") in {"pay", "enter_card", "create_account", "book_live_call"}:
630
+ raise FunnelError("forbidden browser action")
631
+ if action.get("type") == "submit":
632
+ ref = action.get("submission_ref")
633
+ if ref not in intents:
634
+ raise FunnelError("submitted screen must reference an approved journal intent")
635
+ form = intents[ref]["form"]
636
+ if (form["funnel_id"], form["path_id"], form["screen_id"]) != (fid, pid, sid):
637
+ raise FunnelError("screen submission reference belongs to another path/form")
638
+ if candidate["role"] == "own":
639
+ raise FunnelError("own funnel cannot submit forms")
640
+ if screen.get("captured_before_fill") is not True:
641
+ raise FunnelError("submitted form screenshot must be captured before contact data were filled")
642
+ if kind == "contact request" and screen.get("submission_required") is True and any(
643
+ following.get("branch_role") == "main" for following in screens[n:]):
644
+ ref = action.get("submission_ref")
645
+ if action.get("type") != "submit" or outcomes.get(ref, {}).get("outcome") != "submitted":
646
+ raise FunnelError("post-contact continuation requires an approved observed submission")
647
+ if kind == "account gate" and any(s.get("branch_role") == "main" for s in screens[n:]) and not str(screen.get("operator_access_ref") or "").startswith("chat:"):
648
+ raise FunnelError("account-gated continuation requires access provided by an authorized account holder")
649
+ if kind in {"payment gate", "booking"} and any(s.get("branch_role") == "main" for s in screens[n:]):
650
+ if kind == "payment gate" or screen.get("booking_kind") == "live_sales_call":
651
+ raise FunnelError("payment or live sales booking ends the path before submission")
652
+ if not any(s["branch_role"] == "main" for s in screens):
653
+ raise FunnelError("auxiliary popup cannot replace the main funnel path")
654
+ opened = next(s["url"] for s in screens if s["branch_role"] == "main")
655
+ if url_identity(opened) != url_identity(walk["address"]):
656
+ redirects = walk.get("redirect_chain", [])
657
+ previous = url_identity(walk["address"])
658
+ for redirect in redirects:
659
+ if url_identity(redirect.get("from")) != previous or not redirect.get("source_ref"):
660
+ raise FunnelError("observed redirect chain must preserve the original ad entry URL")
661
+ previous = url_identity(redirect.get("to"))
662
+ if not redirects or previous != url_identity(opened):
663
+ raise FunnelError("different opened URL needs the actual observed redirect chain")
664
+ walk["opened_address"] = opened
665
+ partial = walk["status"] != "completed"
666
+ if partial and not walk["unknown"]:
667
+ raise FunnelError("partial path must identify its unseen downstream")
668
+ if candidate["role"] == "own" and walk["stop_reason"] not in {"own_no_submit", "form_not_submitted", "technical", "captcha", "completed", "budget"}:
669
+ raise FunnelError("own funnel uses an explicit no-submit boundary")
670
+ if walk["observed_outcome"] in {"lead_submitted", "webinar_registered"} and not any(
671
+ outcomes.get(s.get("action", {}).get("submission_ref"), {}).get("outcome") == "submitted" for s in screens):
672
+ raise FunnelError("claimed registration outcome requires an observed approved submission")
673
+ main_screens = [s for s in screens if s["branch_role"] == "main"]
674
+ last = main_screens[-1]
675
+ last_kind = last["kind"].split(":", 1)[0]
676
+ if walk["status"] == "completed":
677
+ if last_kind in {"account gate", "payment gate", "booking", "verification", "loader", "contact request"}:
678
+ raise FunnelError("a terminal gate or request cannot masquerade as a completed full path")
679
+ final_result = any(s.get("result_type") == "final" for s in main_screens)
680
+ final_submission = any(outcomes.get(s.get("action", {}).get("submission_ref"), {}).get("outcome") == "submitted" for s in main_screens)
681
+ if walk["observed_outcome"] in {"quiz_completed", "result_observed"} and not final_result:
682
+ if not (walk["observed_outcome"] == "quiz_completed" and final_submission and last_kind == "confirmation" and last.get("completion_observed") is True):
683
+ raise FunnelError("quiz completion needs an actual final result or observed approved completion response")
684
+ if walk["observed_outcome"] in {"lead_submitted", "webinar_registered"} and last_kind not in {"confirmation", "result"}:
685
+ raise FunnelError("registration completion needs an observed confirmation screen")
686
+ if walk["observed_outcome"] == "public_content_end" and last.get("completion_observed") is not True:
687
+ raise FunnelError("public content end needs an explicitly observed end, not a shallow landing stop")
688
+ walk["partial"] = partial
689
+ walk["observed_screens_count"] = len([s for s in screens if s["branch_role"] == "main"])
690
+ walk["screens_count"] = walk["observed_screens_count"]
691
+ walk["full_screens_count"] = None if partial else walk["observed_screens_count"]
692
+ walk["last_observed_screen"] = screens[-1]["screen_id"]
693
+ market = walk["observed_market"]
694
+ walk["market_mismatch"] = bool(market.get("country") and candidate.get("country") and market["country"] != candidate["country"])
695
+ walk["target_market"] = candidate.get("country") or "unknown"
696
+ walk["type"] = "funnel_walk"
697
+ valid.append(walk)
698
+ return valid
699
+
700
+
701
+ def observation_features(walk: dict) -> dict:
702
+ try:
703
+ return FEATURES.observation_features(walk)
704
+ except FEATURES.FunnelFeaturesError as exc:
705
+ raise FunnelError(str(exc)) from exc
706
+
707
+
708
+ def attach_campaigns(walks: list[dict], campaigns: list[dict]) -> list[dict]:
709
+ by_id = {c["campaign_id"]: c for c in campaigns}
710
+ for walk in walks:
711
+ linked = [by_id[cid] for cid in walk["campaign_ids"]]
712
+ walk.update(observation_features(walk))
713
+ walk["creative_ids"] = list(dict.fromkeys(cid for c in linked for cid in c["creative_ids"]))
714
+ walk["dated_refs"] = [r for c in linked if c.get("research_comparable") for r in c["dated_refs"]]
715
+ walk["running_refs"] = [r for c in linked if c.get("research_comparable") for r in c["running_refs"]]
716
+ walk["source_members"] = [r for c in linked for r in c["source_members"]]
717
+ walk["advertising_observation_status"] = "active" if any(c["observation_status"] == "active" for c in linked) else "observed" if linked else "unknown"
718
+ walk["observation_status"] = walk["advertising_observation_status"]
719
+ if walk.get("market_mismatch"):
720
+ walk["observation_status"] = "unknown"
721
+ walk["dated_refs"] = []
722
+ walk["running_refs"] = []
723
+ walk["policy_sha256"] = linked[0]["policy_sha256"] if linked else None
724
+ walk["competitor_tier"] = linked[0]["competitor_tier"] if linked else "own"
725
+ walk["research_comparable"] = any(c.get("research_comparable") for c in linked)
726
+ walk["walk_id"] = identity("walk", [walk["funnel_id"], walk["path_id"]])
727
+ walk["walk_ref"] = walk["walk_id"]
728
+ walk["creative_links"] = []
729
+ main = [s for s in walk["screens"] if s["branch_role"] == "main"]
730
+ for campaign in linked:
731
+ for creative in campaign["creatives"]:
732
+ walk["creative_links"].append({
733
+ "creative_id": creative["creative_id"], "campaign_id": campaign["campaign_id"],
734
+ "category_angle_ids": creative.get("category_angle_ids", []),
735
+ "message_family_id": creative.get("message_family_id"),
736
+ "ad_hook": creative.get("headline"), "ad_promise": creative.get("body"),
737
+ "ad_offer": creative.get("offer_kind"),
738
+ "landing_headline": main[0].get("headline"),
739
+ "landing_offer": next((s["offer"] for s in main if s.get("offer")), None),
740
+ "observed_ending": walk.get("observed_outcome"),
741
+ "ending_unknown": walk["partial"], "funnel_types": walk["funnel_types"],
742
+ "source": "04 angle dictionary + observed browser content",
743
+ })
744
+ walk["tier"] = "A"
745
+ return walks
746
+
747
+
748
+ def aggregate_patterns(campaigns: list[dict], walks: list[dict]) -> list[dict]:
749
+ """One vote per campaign; unknown excluded from denominator but shown.
750
+
751
+ Extra persona paths are additional observations, not votes/competitors.
752
+ A feature is known true if any relevant path shows it; false only if a
753
+ completed path determines absence. Unobserved downstream stays unknown.
754
+ """
755
+ groups = defaultdict(list)
756
+ by_campaign = defaultdict(list)
757
+ for walk in walks:
758
+ if walk["role"] == "own" or walk.get("market_mismatch"):
759
+ continue
760
+ for cid in walk["campaign_ids"]:
761
+ by_campaign[cid].append(walk)
762
+ for campaign in campaigns:
763
+ groups[(campaign.get("country"), campaign["channel"])].append(campaign)
764
+ out = []
765
+ for (country, channel), members in groups.items():
766
+ names = sorted({name for c in members for w in by_campaign[c["campaign_id"]] for name in w["features"]})
767
+ for name in names:
768
+ yes, known = [], []
769
+ for c in members:
770
+ values = [w["features"].get(name) for w in by_campaign[c["campaign_id"]]]
771
+ value = True if True in values else False if False in values else None
772
+ if value is not None:
773
+ known.append(c)
774
+ if value is True:
775
+ yes.append(c)
776
+ if not yes:
777
+ continue
778
+ domains = sorted({c["domain"] for c in yes})
779
+ dated = [c for c in yes if bool(c.get("dated_refs")) and c.get("research_comparable")]
780
+ dated_domains = sorted({c["domain"] for c in dated})
781
+ supporting = [w for c in yes for w in by_campaign[c["campaign_id"]] if w["features"].get(name) is True]
782
+ row = {"type": "funnel_pattern", "pattern_id": identity("pattern", [country, channel, name]),
783
+ "pattern": name, "pattern_group": "funnel_type" if name.startswith("type:") else "structure",
784
+ "country": country, "channel": channel, "role": "competitor", "selected_campaigns_total": len(members),
785
+ "campaigns_with": len(yes), "known_campaigns_total": len(known), "unknown_campaigns": len(members) - len(known),
786
+ "share": round(len(yes) / len(known), 4) if known else None,
787
+ "competitors": domains, "convergence": len(domains), "observed_convergence": len(domains),
788
+ "dated_convergence": len(dated_domains), "dated_competitors": dated_domains,
789
+ "direct_convergence": len({c["domain"] for c in dated if c["competitor_tier"] == "direct"}),
790
+ "indirect_convergence": len({c["domain"] for c in dated if c["competitor_tier"] == "indirect"}),
791
+ "observation_status": "active" if any(c["observation_status"] == "active" for c in yes) else "observed",
792
+ "dated_refs": [r for c in dated for r in c["dated_refs"]],
793
+ "running_refs": [r for c in yes if c.get("research_comparable") for r in c["running_refs"]],
794
+ "source_members": [r for c in yes for r in c["source_members"]],
795
+ "campaign_ids": [c["campaign_id"] for c in yes], "creative_ids": list(dict.fromkeys(cid for c in yes for cid in c["creative_ids"])),
796
+ "funnel_ids": sorted({c["funnel_id"] for c in yes}), "walk_refs": sorted({w["walk_ref"] for w in supporting}),
797
+ "policy_sha256": members[0]["policy_sha256"], "recurring": len(domains) >= 2,
798
+ "relevance": "relevant", "tier": "B"}
799
+ row["research_comparable_campaign_ids"] = [c["campaign_id"] for c in yes if c.get("research_comparable")]
800
+ row["research_comparable"] = bool(row["research_comparable_campaign_ids"])
801
+ out.append(row)
802
+ # Angles are paired with observed types, rather than measured by lexical
803
+ # overlap. One campaign gets one vote regardless of its ad/path count.
804
+ for (country, channel), members in groups.items():
805
+ link_keys = set()
806
+ known_members = []
807
+ campaign_links = {}
808
+ for c in members:
809
+ links = [link for w in by_campaign[c["campaign_id"]] for link in w.get("creative_links", [])]
810
+ keys = {(angle, link.get("message_family_id"), kind)
811
+ for link in links for angle in link.get("category_angle_ids", []) for kind in link.get("funnel_types", [])}
812
+ campaign_links[c["campaign_id"]] = keys
813
+ link_keys.update(keys)
814
+ if any(link.get("category_angle_ids") and link.get("funnel_types") for link in links):
815
+ known_members.append(c)
816
+ for key in sorted(link_keys, key=str):
817
+ yes = [c for c in known_members if key in campaign_links[c["campaign_id"]]]
818
+ dated = [c for c in yes if bool(c.get("dated_refs")) and c.get("research_comparable")]
819
+ domains, dated_domains = sorted({c["domain"] for c in yes}), sorted({c["domain"] for c in dated})
820
+ out.append({"type": "funnel_pattern", "pattern_id": identity("link", [country, channel, key]),
821
+ "pattern": "creative_link:" + key[0] + "→" + key[2], "pattern_group": "creative_link",
822
+ "country": country, "channel": channel, "role": "competitor", "category_angle_ids": [key[0]],
823
+ "message_family_id": key[1], "funnel_type": key[2], "selected_campaigns_total": len(members),
824
+ "campaigns_with": len(yes), "known_campaigns_total": len(known_members),
825
+ "unknown_campaigns": len(members) - len(known_members), "share": round(len(yes) / len(known_members), 4),
826
+ "competitors": domains, "convergence": len(domains), "observed_convergence": len(domains),
827
+ "dated_convergence": len(dated_domains), "dated_competitors": dated_domains,
828
+ "direct_convergence": len({c["domain"] for c in dated if c["competitor_tier"] == "direct"}),
829
+ "indirect_convergence": len({c["domain"] for c in dated if c["competitor_tier"] == "indirect"}),
830
+ "observation_status": "active" if any(c["observation_status"] == "active" for c in yes) else "observed",
831
+ "dated_refs": [r for c in dated for r in c["dated_refs"]],
832
+ "running_refs": [r for c in yes if c.get("research_comparable") for r in c["running_refs"]],
833
+ "source_members": [r for c in yes for r in c["source_members"]], "campaign_ids": [c["campaign_id"] for c in yes],
834
+ "creative_ids": list(dict.fromkeys(cid for c in yes for cid in c["creative_ids"])),
835
+ "funnel_ids": sorted({c["funnel_id"] for c in yes}),
836
+ "walk_refs": sorted({w["walk_id"] for c in yes for w in by_campaign[c["campaign_id"]]
837
+ if any(key[0] in link.get("category_angle_ids", []) and link.get("message_family_id") == key[1]
838
+ and key[2] in link.get("funnel_types", []) for link in w.get("creative_links", []))}),
839
+ "policy_sha256": members[0]["policy_sha256"], "recurring": len(domains) >= 2,
840
+ "relevance": "relevant", "research_comparable_campaign_ids": [c["campaign_id"] for c in yes if c.get("research_comparable")],
841
+ "research_comparable": any(c.get("research_comparable") for c in yes), "tier": "B"})
842
+ return out
843
+
844
+
845
+ _PUBLIC_REFERENCE_FIELDS = {
846
+ "address", "opened_address", "landing_url", "entry_url", "original_urls",
847
+ "destination_refs", "source_members", "dated_refs", "running_refs", "assets",
848
+ "screenshot", "screenshots", "display_screenshot", "media_assets", "path", "sha256",
849
+ }
850
+ _PUBLIC_URL_FIELDS = {"url", "address", "opened_address", "landing_url", "entry_url", "original_urls", "from", "to"}
851
+ _PRIVATE_ROW_FIELDS = {"fields", "alias", "mailbox", "message_ref", "raw_body", "source_screenshot"}
852
+ _PUBLIC_URL_PATTERN = re.compile(r"https?://[^\s<>\"']+", re.I)
853
+ _PUBLIC_METADATA_FIELDS = {
854
+ "id", "ids", "ref", "refs", "type", "kind", "source", "source_ref", "status",
855
+ "verification_status", "pattern", "advance", "role", "country", "channel",
856
+ "result_type", "observed_outcome", "stop_reason", "currency", "language",
857
+ "schema_version", "domain", "format", "tier", "competitor_tier", "ad_offer",
858
+ "relevance", "age_basis", "duration_scope", "viewport",
859
+ }
860
+ _PUBLIC_METADATA_SUFFIXES = (
861
+ "_id", "_ids", "_ref", "_refs", "_sha256", "_type", "_types", "_kind", "_kinds",
862
+ "_status", "_statuses", "_at", "_on", "_role", "_roles", "_group",
863
+ )
864
+
865
+
866
+ def _contact_pattern(private):
867
+ return r"(?<!\w)" + re.escape(private) + r"(?!\w)"
868
+
869
+
870
+ def _phone_contact_pattern(private):
871
+ digits = re.sub(r"\D", "", str(private))
872
+ if re.fullmatch(r"1?\d{3}55501\d\d", digits):
873
+ return r"(?<!\d)(?:\+?1[\s().-]*)?" + r"[\s().-]*".join(re.escape(d) for d in digits[-10:]) + r"(?!\d)"
874
+ return None
875
+
876
+
877
+ def private_identifier_in_reference(value, private_values):
878
+ decoded_forms = {unquote(unquote(value)), unquote_plus(value), unquote_plus(unquote(value))}
879
+ for decoded in decoded_forms:
880
+ for private in private_values:
881
+ if not private:
882
+ continue
883
+ if "@" in str(private) and re.search(_contact_pattern(str(private)), decoded, re.I):
884
+ return True
885
+ phone = _phone_contact_pattern(private)
886
+ if phone and re.search(phone, decoded):
887
+ return True
888
+ return False
889
+
890
+
891
+ def _private_mail_url(url, private_values, mail_context):
892
+ """Names in a hostname or business URL are not contact echoes."""
893
+ # Exact known email/phone identifiers must not appear in a public URL.
894
+ if private_identifier_in_reference(url, private_values):
895
+ return True
896
+ if not mail_context:
897
+ return False
898
+ try:
899
+ parts = urlsplit(url)
900
+ except ValueError:
901
+ # Malformed links inside private email are not safe public references.
902
+ return True
903
+ private_keys = {"token", "verification", "auth", "signature", "magic_link", "access_token", "email", "code", "uid", "key"}
904
+ return (any(k.lower() in private_keys for k, _ in parse_qsl(parts.query))
905
+ or bool(re.search(r"/(?:verify|confirm|activate|magic-link)(?:/|$)", parts.path, re.I)))
906
+
907
+
908
+ def _redact_text(text, private_values, mail_context):
909
+ variants = {variant for private in private_values if private
910
+ for variant in (str(private), quote(str(private), safe=""), quote_plus(str(private), safe=""))}
911
+
912
+ def scrub(fragment):
913
+ for private in sorted(variants, key=len, reverse=True):
914
+ fragment = re.sub(_contact_pattern(private), "[research contact]", fragment, flags=re.I)
915
+ for private in private_values:
916
+ phone = _phone_contact_pattern(private)
917
+ if phone:
918
+ fragment = re.sub(phone, "[research contact]", fragment)
919
+ return fragment
920
+
921
+ pieces, start = [], 0
922
+ for match in _PUBLIC_URL_PATTERN.finditer(text):
923
+ pieces.append(scrub(text[start:match.start()]))
924
+ url = match.group(0)
925
+ pieces.append("[private email URL]" if _private_mail_url(url, private_values, mail_context) else url)
926
+ start = match.end()
927
+ pieces.append(scrub(text[start:]))
928
+ return "".join(pieces)
929
+
930
+
931
+ def redact(value, private_values: list[str], mail_context: bool = False, *, _text=False, _field=None, _reference=False):
932
+ """Scrub public prose while preserving identity and pinned graph references.
933
+
934
+ Tokenized email links may be hidden in mail events and free text. A private
935
+ contact URL used as a bound destination/asset is rejected instead of silently
936
+ publishing a different reference that no longer matches the source hashes.
937
+ """
938
+ if isinstance(value, str):
939
+ if _reference and private_identifier_in_reference(value, private_values):
940
+ raise FunnelError("private contact identifier in a pinned public reference; use a public destination or safely named display asset")
941
+ if _field in _PUBLIC_URL_FIELDS and _private_mail_url(value, private_values, mail_context):
942
+ if _reference:
943
+ raise FunnelError("private contact URL in a pinned public reference; provide a public destination or display asset")
944
+ return "[private email URL]"
945
+ return _redact_text(value, private_values, mail_context) if _text or _field is None else value
946
+ if isinstance(value, dict):
947
+ mail_context = mail_context or value.get("type") in {"email_confirmation", "email_result"} or str(value.get("source_ref") or "").lower().startswith("mail:")
948
+ public = {}
949
+ for key, item in value.items():
950
+ if key in _PRIVATE_ROW_FIELDS:
951
+ continue
952
+ if key == "source_ref" and isinstance(item, str) and item.lower().startswith("mail:"):
953
+ public[key] = "[private mail source]"
954
+ continue
955
+ reference = _reference or key in _PUBLIC_REFERENCE_FIELDS or key in {"id", "ids", "ref", "refs", "source_ref"} or key.endswith(("_id", "_ids", "_ref", "_refs", "_sha256"))
956
+ structural = reference or key in _PUBLIC_URL_FIELDS or key in _PUBLIC_METADATA_FIELDS or key.endswith(_PUBLIC_METADATA_SUFFIXES)
957
+ public[key] = redact(item, private_values, mail_context,
958
+ _text=not structural, _field=key, _reference=reference)
959
+ return public
960
+ if isinstance(value, list):
961
+ return [redact(v, private_values, mail_context, _text=_text, _field=_field, _reference=_reference) for v in value]
962
+ if _text and type(value) is int and any(
963
+ pattern and re.fullmatch(pattern, str(value))
964
+ for pattern in (_phone_contact_pattern(private) for private in private_values)):
965
+ return "[research contact]"
966
+ return value