@egbe-ai/opencode 0.0.0-stage → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (479) hide show
  1. package/LICENSE +4 -0
  2. package/README.md +124 -2
  3. package/index.js +5 -0
  4. package/package.json +55 -4
  5. package/plugin.js +230 -0
  6. package/session-log.js +315 -0
  7. package/skills/.egbe-catalog/manifest.json +1 -0
  8. package/skills/egbe-os-analytics/SKILL.md +499 -0
  9. package/skills/egbe-os-analytics/canonical-events.v1.json +256 -0
  10. package/skills/egbe-os-analytics/references/session-funnel.md +130 -0
  11. package/skills/egbe-os-analytics/references/workflows/activation-tracking.md +147 -0
  12. package/skills/egbe-os-analytics/references/workflows/landing-click-tracking.md +209 -0
  13. package/skills/egbe-os-analytics/references/workflows/payment-lifecycle-tracking.md +122 -0
  14. package/skills/egbe-os-analytics/references/workflows/product-funnel-tracking.md +77 -0
  15. package/skills/egbe-os-analytics/references/workflows/signup-lifecycle-tracking.md +208 -0
  16. package/skills/egbe-os-analytics/scripts/umami_funnel.py +927 -0
  17. package/skills/egbe-os-apollo-lead-finder/SKILL.md +135 -0
  18. package/skills/egbe-os-apollo-lead-finder/references/operations.md +240 -0
  19. package/skills/egbe-os-apollo-lead-finder/scripts/enrich.js +6 -0
  20. package/skills/egbe-os-apollo-lead-finder/scripts/enrich.sh +4 -0
  21. package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.js +6 -0
  22. package/skills/egbe-os-apollo-lead-finder/scripts/find-leads.sh +4 -0
  23. package/skills/egbe-os-apollo-lead-finder/scripts/lib/apollo-client.js +343 -0
  24. package/skills/egbe-os-apollo-lead-finder/scripts/lib/connection-identity.js +17 -0
  25. package/skills/egbe-os-apollo-lead-finder/scripts/lib/helpers.js +127 -0
  26. package/skills/egbe-os-apollo-lead-finder/scripts/lib/places-filters.js +217 -0
  27. package/skills/egbe-os-apollo-lead-finder/scripts/lib/sourcing.js +153 -0
  28. package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.js +653 -0
  29. package/skills/egbe-os-apollo-lead-finder/scripts/places-leads.sh +4 -0
  30. package/skills/egbe-os-apollo-lead-finder/scripts/search.js +6 -0
  31. package/skills/egbe-os-apollo-lead-finder/scripts/search.sh +4 -0
  32. package/skills/egbe-os-campaign-landing-page/SKILL.md +95 -0
  33. package/skills/egbe-os-campaign-landing-page/agents/openai.yaml +4 -0
  34. package/skills/egbe-os-campaign-landing-page/references/leads-calendar-and-tracking.md +66 -0
  35. package/skills/egbe-os-campaign-landing-page/references/os-wiring.md +58 -0
  36. package/skills/egbe-os-campaign-landing-page/references/qa.md +40 -0
  37. package/skills/egbe-os-campaign-landing-page/references/routing.md +32 -0
  38. package/skills/egbe-os-db/SKILL.md +174 -0
  39. package/skills/egbe-os-deploy/SKILL.md +412 -0
  40. package/skills/egbe-os-design/SKILL.md +182 -0
  41. package/skills/egbe-os-design-landing-page/SKILL.md +117 -0
  42. package/skills/egbe-os-design-landing-page/agents/openai.yaml +4 -0
  43. package/skills/egbe-os-design-landing-page/assets/scaffold/.dockerignore +11 -0
  44. package/skills/egbe-os-design-landing-page/assets/scaffold/AGENTS.md +31 -0
  45. package/skills/egbe-os-design-landing-page/assets/scaffold/Dockerfile +21 -0
  46. package/skills/egbe-os-design-landing-page/assets/scaffold/eslint.config.mjs +18 -0
  47. package/skills/egbe-os-design-landing-page/assets/scaffold/next.config.ts +11 -0
  48. package/skills/egbe-os-design-landing-page/assets/scaffold/package.json +36 -0
  49. package/skills/egbe-os-design-landing-page/assets/scaffold/postcss.config.mjs +7 -0
  50. package/skills/egbe-os-design-landing-page/assets/scaffold/public/.gitkeep +1 -0
  51. package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.edge.config.ts +13 -0
  52. package/skills/egbe-os-design-landing-page/assets/scaffold/sentry.server.config.ts +13 -0
  53. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/global-error.tsx +19 -0
  54. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/globals.css +22 -0
  55. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/layout.tsx +21 -0
  56. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/page.tsx +3 -0
  57. package/skills/egbe-os-design-landing-page/assets/scaffold/src/app/privacy/page.tsx +28 -0
  58. package/skills/egbe-os-design-landing-page/assets/scaffold/src/components/egbe-analytics.tsx +39 -0
  59. package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation-client.ts +15 -0
  60. package/skills/egbe-os-design-landing-page/assets/scaffold/src/instrumentation.ts +13 -0
  61. package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/egbe-observability.generated.ts +20 -0
  62. package/skills/egbe-os-design-landing-page/assets/scaffold/src/lib/utils.ts +6 -0
  63. package/skills/egbe-os-design-landing-page/assets/scaffold/tsconfig.json +34 -0
  64. package/skills/egbe-os-design-landing-page/references/application-preparation.md +44 -0
  65. package/skills/egbe-os-design-landing-page/references/page-inputs.md +22 -0
  66. package/skills/egbe-os-destination-readiness/SKILL.md +132 -0
  67. package/skills/egbe-os-destination-readiness/references/checks.md +130 -0
  68. package/skills/egbe-os-destination-readiness/references/network-requirements.md +34 -0
  69. package/skills/egbe-os-destination-readiness/references/report.md +116 -0
  70. package/skills/egbe-os-domains/SKILL.md +189 -0
  71. package/skills/egbe-os-errors/SKILL.md +259 -0
  72. package/skills/egbe-os-git/SKILL.md +275 -0
  73. package/skills/egbe-os-git/references/issues.md +33 -0
  74. package/skills/egbe-os-google-ads/SKILL.md +163 -0
  75. package/skills/egbe-os-google-ads/references/artifacts.md +133 -0
  76. package/skills/egbe-os-google-ads/references/campaigns.md +157 -0
  77. package/skills/egbe-os-google-ads/references/conversions.md +153 -0
  78. package/skills/egbe-os-google-ads/references/delivery.md +128 -0
  79. package/skills/egbe-os-google-ads/references/keyword-ideas.md +40 -0
  80. package/skills/egbe-os-google-ads/references/reporting.md +105 -0
  81. package/skills/egbe-os-google-ads/references/search-assets.md +219 -0
  82. package/skills/egbe-os-google-ads/references/server-conversions.md +119 -0
  83. package/skills/egbe-os-google-auth/SKILL.md +285 -0
  84. package/skills/egbe-os-google-auth/agents/openai.yaml +4 -0
  85. package/skills/egbe-os-gtm-knowledge/SKILL.md +151 -0
  86. package/skills/egbe-os-images/SKILL.md +163 -0
  87. package/skills/egbe-os-images/references/stock-images.md +21 -0
  88. package/skills/egbe-os-instantly-campaign-launcher/SKILL.md +127 -0
  89. package/skills/egbe-os-instantly-campaign-launcher/references/instantly-ops.md +260 -0
  90. package/skills/egbe-os-instantly-campaign-launcher/references/observation-and-scheduling.md +188 -0
  91. package/skills/egbe-os-instantly-campaign-launcher/references/reply-handling.md +116 -0
  92. package/skills/egbe-os-instantly-campaign-launcher/references/run-spec-template.json +65 -0
  93. package/skills/egbe-os-instantly-campaign-launcher/references/run-state.md +145 -0
  94. package/skills/egbe-os-instantly-campaign-launcher/scripts/conclude-campaign.js +158 -0
  95. package/skills/egbe-os-instantly-campaign-launcher/scripts/install-crons.js +956 -0
  96. package/skills/egbe-os-instantly-campaign-launcher/scripts/launch-campaign.js +2008 -0
  97. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/campaign-metrics.js +124 -0
  98. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/confirmed-opt-outs.js +208 -0
  99. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/connection-identity.js +17 -0
  100. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/create-once.js +23 -0
  101. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/gtm-identity.js +125 -0
  102. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/helpers.js +545 -0
  103. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/html-body.js +266 -0
  104. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/instantly-client.js +328 -0
  105. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/outreach-db.js +370 -0
  106. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/recipient-exclusion/index.js +86 -0
  107. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/replenish-test-harness.js +53 -0
  108. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/reply-queue.js +67 -0
  109. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/run-config.js +103 -0
  110. package/skills/egbe-os-instantly-campaign-launcher/scripts/lib/sender-selection.js +105 -0
  111. package/skills/egbe-os-instantly-campaign-launcher/scripts/monitor-campaign.js +607 -0
  112. package/skills/egbe-os-instantly-campaign-launcher/scripts/outreach-report.js +147 -0
  113. package/skills/egbe-os-instantly-campaign-launcher/scripts/replenish-leads.js +619 -0
  114. package/skills/egbe-os-instantly-campaign-launcher/scripts/reply-worker.js +258 -0
  115. package/skills/egbe-os-instantly-campaign-launcher/scripts/run-cron-job.js +32 -0
  116. package/skills/egbe-os-instantly-campaign-launcher/scripts/send-reply.js +260 -0
  117. package/skills/egbe-os-linkedin-ads/SKILL.md +114 -0
  118. package/skills/egbe-os-linkedin-ads/references/destination-requirements.md +30 -0
  119. package/skills/egbe-os-linkedin-ads/references/runtime/api-reference.md +154 -0
  120. package/skills/egbe-os-linkedin-ads/references/runtime/artifacts.md +63 -0
  121. package/skills/egbe-os-linkedin-ads/references/workflows/build-paused.md +44 -0
  122. package/skills/egbe-os-linkedin-ads/references/workflows/conversions.md +88 -0
  123. package/skills/egbe-os-linkedin-ads/references/workflows/launch-and-pause.md +49 -0
  124. package/skills/egbe-os-linkedin-ads/references/workflows/monitoring.md +35 -0
  125. package/skills/egbe-os-linkedin-ads/references/workflows/readiness.md +63 -0
  126. package/skills/egbe-os-linkedin-ads/references/workflows/targeting-and-creative.md +52 -0
  127. package/skills/egbe-os-llm/SKILL.md +160 -0
  128. package/skills/egbe-os-mail/SKILL.md +180 -0
  129. package/skills/egbe-os-me/SKILL.md +119 -0
  130. package/skills/egbe-os-meta-ads/SKILL.md +114 -0
  131. package/skills/egbe-os-meta-ads/references/creative/creative-specs.md +15 -0
  132. package/skills/egbe-os-meta-ads/references/creative/talking-head.md +35 -0
  133. package/skills/egbe-os-meta-ads/references/destination-requirements.md +30 -0
  134. package/skills/egbe-os-meta-ads/references/runtime/api-reference.md +436 -0
  135. package/skills/egbe-os-meta-ads/references/runtime/artifacts.md +51 -0
  136. package/skills/egbe-os-meta-ads/references/runtime/insights-reference.md +187 -0
  137. package/skills/egbe-os-meta-ads/references/runtime/pixel-install.md +196 -0
  138. package/skills/egbe-os-meta-ads/references/runtime/shared-analytics-attribution.md +16 -0
  139. package/skills/egbe-os-meta-ads/references/workflows/build-paused.md +36 -0
  140. package/skills/egbe-os-meta-ads/references/workflows/launch-ready.md +16 -0
  141. package/skills/egbe-os-meta-ads/references/workflows/launch.md +23 -0
  142. package/skills/egbe-os-meta-ads/references/workflows/monitoring.md +72 -0
  143. package/skills/egbe-os-meta-ads/references/workflows/pause.md +17 -0
  144. package/skills/egbe-os-meta-ads/references/workflows/recovery.md +23 -0
  145. package/skills/egbe-os-meta-ads/scripts/check-pixel.py +229 -0
  146. package/skills/egbe-os-meta-ads/scripts/validate-meta-artifacts.py +187 -0
  147. package/skills/egbe-os-notifications/SKILL.md +78 -0
  148. package/skills/egbe-os-openai-ads/SKILL.md +149 -0
  149. package/skills/egbe-os-openai-ads/references/creative/chat-card.md +51 -0
  150. package/skills/egbe-os-openai-ads/references/destination-requirements.md +29 -0
  151. package/skills/egbe-os-openai-ads/references/runtime/api-reference.md +219 -0
  152. package/skills/egbe-os-openai-ads/references/runtime/artifacts.md +169 -0
  153. package/skills/egbe-os-openai-ads/references/workflows/build-paused.md +40 -0
  154. package/skills/egbe-os-openai-ads/references/workflows/conversions.md +40 -0
  155. package/skills/egbe-os-openai-ads/references/workflows/creative-assembly.md +24 -0
  156. package/skills/egbe-os-openai-ads/references/workflows/launch.md +27 -0
  157. package/skills/egbe-os-openai-ads/references/workflows/monitoring.md +54 -0
  158. package/skills/egbe-os-openai-ads/references/workflows/pause.md +22 -0
  159. package/skills/egbe-os-openai-ads/references/workflows/readiness.md +37 -0
  160. package/skills/egbe-os-openai-ads/references/workflows/tracking.md +32 -0
  161. package/skills/egbe-os-openai-ads/scripts/validate-openai-artifacts.py +271 -0
  162. package/skills/egbe-os-organic-trend-watch/SKILL.md +139 -0
  163. package/skills/egbe-os-organic-trend-watch/agents/openai.yaml +4 -0
  164. package/skills/egbe-os-organic-trend-watch/references/analysis-and-adaptation.md +54 -0
  165. package/skills/egbe-os-organic-trend-watch/references/collection-and-analysis.md +117 -0
  166. package/skills/egbe-os-organic-trend-watch/references/output-contract.md +133 -0
  167. package/skills/egbe-os-organic-trend-watch/references/provider-contract.md +169 -0
  168. package/skills/egbe-os-organic-trend-watch/references/runtime-contract.md +204 -0
  169. package/skills/egbe-os-organic-trend-watch/references/visual-analysis-prompt.md +17 -0
  170. package/skills/egbe-os-organic-trend-watch/scripts/_analysis.py +612 -0
  171. package/skills/egbe-os-organic-trend-watch/scripts/_contract.py +1133 -0
  172. package/skills/egbe-os-organic-trend-watch/scripts/_provider_inputs.py +165 -0
  173. package/skills/egbe-os-organic-trend-watch/scripts/analyze-visual-evidence.py +315 -0
  174. package/skills/egbe-os-organic-trend-watch/scripts/build-provider-inputs.py +88 -0
  175. package/skills/egbe-os-organic-trend-watch/scripts/connection_identity.py +22 -0
  176. package/skills/egbe-os-organic-trend-watch/scripts/egbe_artifacts.py +24 -0
  177. package/skills/egbe-os-organic-trend-watch/scripts/finalize-receipt.py +106 -0
  178. package/skills/egbe-os-organic-trend-watch/scripts/normalize-videos.py +647 -0
  179. package/skills/egbe-os-organic-trend-watch/scripts/prepare-visual-evidence.py +324 -0
  180. package/skills/egbe-os-organic-trend-watch/scripts/render-result.py +30 -0
  181. package/skills/egbe-os-organic-trend-watch/scripts/run-apify.py +723 -0
  182. package/skills/egbe-os-organic-trend-watch/scripts/run-state.py +192 -0
  183. package/skills/egbe-os-organic-trend-watch/scripts/validate-context.py +27 -0
  184. package/skills/egbe-os-organic-trend-watch/scripts/validate-runtime.py +137 -0
  185. package/skills/egbe-os-payments/SKILL.md +215 -0
  186. package/skills/egbe-os-payments/agents/openai.yaml +4 -0
  187. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/.egbe/scaffolds/billing-nextjs-subscription.v1.json +5 -0
  188. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/AGENTS.append.md +13 -0
  189. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/db/migrations/1000_egbe_billing.sql +66 -0
  190. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/package.fragment.json +12 -0
  191. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/scripts/billing-sync.mjs +162 -0
  192. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/error.tsx +11 -0
  193. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/loading.tsx +8 -0
  194. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/account/billing/page.tsx +121 -0
  195. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/checkout/route.ts +102 -0
  196. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/portal/route.ts +54 -0
  197. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/billing/status/route.ts +22 -0
  198. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/app/api/payments/webhook/route.ts +65 -0
  199. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/components/egbe-billing/plan-badge.tsx +16 -0
  200. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/billing-sync.test.mjs +58 -0
  201. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/catalog.ts +82 -0
  202. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/checkout-payload.ts +47 -0
  203. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/foundation-contract.test.mjs +59 -0
  204. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.d.ts +16 -0
  205. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.mjs +82 -0
  206. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/lifecycle.test.mjs +94 -0
  207. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/os-client.ts +96 -0
  208. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/plans.json +17 -0
  209. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/repository.ts +325 -0
  210. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.d.ts +6 -0
  211. package/skills/egbe-os-payments/assets/scaffolds/billing-nextjs-subscription/src/lib/egbe-billing/signature.mjs +22 -0
  212. package/skills/egbe-os-payments/references/contracts/plan-catalog.md +46 -0
  213. package/skills/egbe-os-payments/references/security/durable-verification.md +64 -0
  214. package/skills/egbe-os-payments/references/security/server-integration.md +26 -0
  215. package/skills/egbe-os-payments/references/security/webhook-and-entitlements.md +152 -0
  216. package/skills/egbe-os-payments/references/ux/billing-experience.md +47 -0
  217. package/skills/egbe-os-payments/references/workflows/account-billing.md +64 -0
  218. package/skills/egbe-os-payments/references/workflows/discounts-and-promotion-codes.md +265 -0
  219. package/skills/egbe-os-payments/references/workflows/one-time-purchase.md +75 -0
  220. package/skills/egbe-os-payments/references/workflows/product-checkout.md +355 -0
  221. package/skills/egbe-os-payments/references/workflows/subscription-lifecycle.md +55 -0
  222. package/skills/egbe-os-playbook/SKILL.md +79 -0
  223. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/email-outreach.md +91 -0
  224. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/google-search.md +40 -0
  225. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/linkedin.md +24 -0
  226. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/meta.md +23 -0
  227. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/openai.md +15 -0
  228. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/channels/reddit.md +56 -0
  229. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/choose-test.md +73 -0
  230. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/evaluate.md +91 -0
  231. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/market.md +96 -0
  232. package/skills/egbe-os-playbook/playbooks/market-and-acquisition/run-test.md +111 -0
  233. package/skills/egbe-os-playbook/playbooks/market-and-acquisition.md +56 -0
  234. package/skills/egbe-os-playbook/playbooks/product/business-loop.md +141 -0
  235. package/skills/egbe-os-playbook/playbooks/product/collaboration.md +26 -0
  236. package/skills/egbe-os-playbook/playbooks/product/deliver.md +75 -0
  237. package/skills/egbe-os-playbook/playbooks/product/diagnose.md +80 -0
  238. package/skills/egbe-os-playbook/playbooks/product/evaluate.md +67 -0
  239. package/skills/egbe-os-playbook/playbooks/product/experience.md +132 -0
  240. package/skills/egbe-os-playbook/playbooks/product/loop-examples.md +67 -0
  241. package/skills/egbe-os-playbook/playbooks/product/offer.md +65 -0
  242. package/skills/egbe-os-playbook/playbooks/product/value.md +91 -0
  243. package/skills/egbe-os-playbook/playbooks/product.md +67 -0
  244. package/skills/egbe-os-playbook/references/catalog.md +127 -0
  245. package/skills/egbe-os-playbook/references/changes.md +167 -0
  246. package/skills/egbe-os-playbook/scripts/playbook-changes.py +6 -0
  247. package/skills/egbe-os-playbook/scripts/playbook_changes/__init__.py +1 -0
  248. package/skills/egbe-os-playbook/scripts/playbook_changes/cli.py +57 -0
  249. package/skills/egbe-os-playbook/scripts/playbook_changes/operations.py +137 -0
  250. package/skills/egbe-os-playbook/scripts/playbook_changes/protocol.py +132 -0
  251. package/skills/egbe-os-playbook/scripts/playbook_changes/storage.py +180 -0
  252. package/skills/egbe-os-playbook/scripts/playbook_changes/transport.py +113 -0
  253. package/skills/egbe-os-product-auth/SKILL.md +151 -0
  254. package/skills/egbe-os-product-auth/agents/openai.yaml +4 -0
  255. package/skills/egbe-os-product-auth/assets/db/README.md +102 -0
  256. package/skills/egbe-os-product-auth/assets/db/create-validation-session.mjs +223 -0
  257. package/skills/egbe-os-product-auth/assets/db/migrate.mjs +64 -0
  258. package/skills/egbe-os-product-auth/assets/db/migrations/0001_auth.sql +33 -0
  259. package/skills/egbe-os-product-auth/assets/db/migrations/0002_auth_identities.sql +16 -0
  260. package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/AGENTS.append.md +17 -0
  261. package/skills/egbe-os-product-auth/assets/scaffolds/product-auth-database/package.fragment.json +10 -0
  262. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/AGENTS.append.md +9 -0
  263. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/package.fragment.json +13 -0
  264. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/db.ts +31 -0
  265. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/identity.ts +94 -0
  266. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/index.ts +3 -0
  267. package/skills/egbe-os-product-auth/assets/scaffolds/product-nextjs-postgres/src/lib/egbe-product/session.ts +91 -0
  268. package/skills/egbe-os-product-auth/references/contracts/live-auth-interface.md +108 -0
  269. package/skills/egbe-os-product-auth/references/frameworks/nextjs.md +34 -0
  270. package/skills/egbe-os-product-auth/references/security/session-and-credentials.md +73 -0
  271. package/skills/egbe-os-product-auth/references/ux/auth-experience.md +58 -0
  272. package/skills/egbe-os-product-auth/references/workflows/account-recovery.md +22 -0
  273. package/skills/egbe-os-product-auth/references/workflows/build-auth.md +212 -0
  274. package/skills/egbe-os-product-auth/references/workflows/google.md +34 -0
  275. package/skills/egbe-os-product-auth/references/workflows/passwordless-email.md +65 -0
  276. package/skills/egbe-os-product-build/SKILL.md +111 -0
  277. package/skills/egbe-os-product-build/agents/openai.yaml +4 -0
  278. package/skills/egbe-os-product-build/references/capabilities.md +25 -0
  279. package/skills/egbe-os-product-build/references/engineering.md +72 -0
  280. package/skills/egbe-os-product-build/references/execution.md +55 -0
  281. package/skills/egbe-os-product-build/references/foundations.md +93 -0
  282. package/skills/egbe-os-product-build/references/interface-quality.md +32 -0
  283. package/skills/egbe-os-product-build/references/workflows/private-review-handoff.md +86 -0
  284. package/skills/egbe-os-product-build/scripts/scaffold.py +296 -0
  285. package/skills/egbe-os-product-demo-video/SKILL.md +118 -0
  286. package/skills/egbe-os-product-demo-video/references/recording.md +32 -0
  287. package/skills/egbe-os-product-market-research/SKILL.md +159 -0
  288. package/skills/egbe-os-product-market-research/agents/openai.yaml +4 -0
  289. package/skills/egbe-os-product-market-research/references/catalog.v1.json +177 -0
  290. package/skills/egbe-os-product-market-research/references/config/channel-footprint-sources.v1.json +99 -0
  291. package/skills/egbe-os-product-market-research/references/config/creative-taxonomy.v1.json +367 -0
  292. package/skills/egbe-os-product-market-research/references/contracts/creative-video.md +75 -0
  293. package/skills/egbe-os-product-market-research/references/contracts/funnel-teardown.md +235 -0
  294. package/skills/egbe-os-product-market-research/references/contracts/geography-comparison.md +71 -0
  295. package/skills/egbe-os-product-market-research/references/contracts/lifecycle-capture.md +119 -0
  296. package/skills/egbe-os-product-market-research/references/contracts/research-artifacts.md +520 -0
  297. package/skills/egbe-os-product-market-research/references/contracts/walk-mechanics.v2.json +18 -0
  298. package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.md +137 -0
  299. package/skills/egbe-os-product-market-research/references/prompts/acquisition-channels.user.md +4 -0
  300. package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.md +118 -0
  301. package/skills/egbe-os-product-market-research/references/prompts/competitive-landscape.user.md +4 -0
  302. package/skills/egbe-os-product-market-research/references/prompts/demand-validation.md +154 -0
  303. package/skills/egbe-os-product-market-research/references/prompts/demand-validation.user.md +4 -0
  304. package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.md +100 -0
  305. package/skills/egbe-os-product-market-research/references/prompts/market-size-and-dynamics.user.md +4 -0
  306. package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.md +126 -0
  307. package/skills/egbe-os-product-market-research/references/prompts/pricing-and-monetization.user.md +4 -0
  308. package/skills/egbe-os-product-market-research/references/prompts/target-audience.md +98 -0
  309. package/skills/egbe-os-product-market-research/references/prompts/target-audience.user.md +4 -0
  310. package/skills/egbe-os-product-market-research/references/prompts/unit-economics.md +217 -0
  311. package/skills/egbe-os-product-market-research/references/prompts/unit-economics.user.md +4 -0
  312. package/skills/egbe-os-product-market-research/references/prompts/value-proposition.md +113 -0
  313. package/skills/egbe-os-product-market-research/references/prompts/value-proposition.user.md +4 -0
  314. package/skills/egbe-os-product-market-research/references/workflows/acquisition-channels.md +27 -0
  315. package/skills/egbe-os-product-market-research/references/workflows/channel-footprint.md +163 -0
  316. package/skills/egbe-os-product-market-research/references/workflows/community-demand.md +100 -0
  317. package/skills/egbe-os-product-market-research/references/workflows/competitive-landscape.md +27 -0
  318. package/skills/egbe-os-product-market-research/references/workflows/competitor-ads.md +82 -0
  319. package/skills/egbe-os-product-market-research/references/workflows/competitor-set.md +300 -0
  320. package/skills/egbe-os-product-market-research/references/workflows/creative-patterns.md +228 -0
  321. package/skills/egbe-os-product-market-research/references/workflows/demand-validation.md +27 -0
  322. package/skills/egbe-os-product-market-research/references/workflows/fleet-prior.md +80 -0
  323. package/skills/egbe-os-product-market-research/references/workflows/funnel-teardown.md +184 -0
  324. package/skills/egbe-os-product-market-research/references/workflows/geography-comparison.md +84 -0
  325. package/skills/egbe-os-product-market-research/references/workflows/keyword-economics.md +122 -0
  326. package/skills/egbe-os-product-market-research/references/workflows/lifecycle-capture.md +218 -0
  327. package/skills/egbe-os-product-market-research/references/workflows/local-research.md +55 -0
  328. package/skills/egbe-os-product-market-research/references/workflows/market-size-and-dynamics.md +27 -0
  329. package/skills/egbe-os-product-market-research/references/workflows/preview.md +175 -0
  330. package/skills/egbe-os-product-market-research/references/workflows/pricing-and-monetization.md +27 -0
  331. package/skills/egbe-os-product-market-research/references/workflows/recommendations.md +35 -0
  332. package/skills/egbe-os-product-market-research/references/workflows/review-mining.md +76 -0
  333. package/skills/egbe-os-product-market-research/references/workflows/target-audience.md +27 -0
  334. package/skills/egbe-os-product-market-research/references/workflows/tiktok-reels-market-trends.md +46 -0
  335. package/skills/egbe-os-product-market-research/references/workflows/unit-economics.md +27 -0
  336. package/skills/egbe-os-product-market-research/references/workflows/value-proposition.md +27 -0
  337. package/skills/egbe-os-product-market-research/scripts/build-organic-trend-context.py +167 -0
  338. package/skills/egbe-os-product-market-research/scripts/build-preview-context.py +91 -0
  339. package/skills/egbe-os-product-market-research/scripts/build-research-query.py +225 -0
  340. package/skills/egbe-os-product-market-research/scripts/channel-footprint-model.py +294 -0
  341. package/skills/egbe-os-product-market-research/scripts/connection_identity.py +22 -0
  342. package/skills/egbe-os-product-market-research/scripts/creative-model.py +364 -0
  343. package/skills/egbe-os-product-market-research/scripts/creative-selection.py +113 -0
  344. package/skills/egbe-os-product-market-research/scripts/creative-semantics.py +365 -0
  345. package/skills/egbe-os-product-market-research/scripts/creative-video.py +542 -0
  346. package/skills/egbe-os-product-market-research/scripts/egbe_artifacts.py +24 -0
  347. package/skills/egbe-os-product-market-research/scripts/finalize-organic-trend-watch.py +304 -0
  348. package/skills/egbe-os-product-market-research/scripts/funnel-email-review.py +97 -0
  349. package/skills/egbe-os-product-market-research/scripts/funnel-model.py +966 -0
  350. package/skills/egbe-os-product-market-research/scripts/funnel_features.py +50 -0
  351. package/skills/egbe-os-product-market-research/scripts/geography-comparison-contract.py +172 -0
  352. package/skills/egbe-os-product-market-research/scripts/geography-comparison.py +169 -0
  353. package/skills/egbe-os-product-market-research/scripts/keyword-forecast.py +61 -0
  354. package/skills/egbe-os-product-market-research/scripts/keyword-locale-publication.py +136 -0
  355. package/skills/egbe-os-product-market-research/scripts/lifecycle-start.py +190 -0
  356. package/skills/egbe-os-product-market-research/scripts/persist-research-report.py +63 -0
  357. package/skills/egbe-os-product-market-research/scripts/research-artifact-io.py +281 -0
  358. package/skills/egbe-os-product-market-research/scripts/research-evidence.py +496 -0
  359. package/skills/egbe-os-product-market-research/scripts/research-policy.py +33 -0
  360. package/skills/egbe-os-product-market-research/scripts/research-section-runtime.py +343 -0
  361. package/skills/egbe-os-product-market-research/scripts/research-section-workflow.py +288 -0
  362. package/skills/egbe-os-product-market-research/scripts/research_integrity.py +122 -0
  363. package/skills/egbe-os-product-market-research/scripts/review-research-completion.py +306 -0
  364. package/skills/egbe-os-product-market-research/scripts/run-acquisition-channels.py +72 -0
  365. package/skills/egbe-os-product-market-research/scripts/run-channel-footprint.py +2258 -0
  366. package/skills/egbe-os-product-market-research/scripts/run-community-demand.py +1603 -0
  367. package/skills/egbe-os-product-market-research/scripts/run-competitive-landscape.py +66 -0
  368. package/skills/egbe-os-product-market-research/scripts/run-competitor-ads.py +1156 -0
  369. package/skills/egbe-os-product-market-research/scripts/run-competitor-set.py +2514 -0
  370. package/skills/egbe-os-product-market-research/scripts/run-creative-patterns.py +2042 -0
  371. package/skills/egbe-os-product-market-research/scripts/run-demand-validation.py +72 -0
  372. package/skills/egbe-os-product-market-research/scripts/run-fleet-prior.py +652 -0
  373. package/skills/egbe-os-product-market-research/scripts/run-funnel-teardown.py +493 -0
  374. package/skills/egbe-os-product-market-research/scripts/run-keyword-economics.py +1443 -0
  375. package/skills/egbe-os-product-market-research/scripts/run-lifecycle-capture.py +1543 -0
  376. package/skills/egbe-os-product-market-research/scripts/run-market-size-and-dynamics.py +66 -0
  377. package/skills/egbe-os-product-market-research/scripts/run-os-research.py +311 -0
  378. package/skills/egbe-os-product-market-research/scripts/run-pricing-and-monetization.py +71 -0
  379. package/skills/egbe-os-product-market-research/scripts/run-review-mining.py +1051 -0
  380. package/skills/egbe-os-product-market-research/scripts/run-target-audience.py +66 -0
  381. package/skills/egbe-os-product-market-research/scripts/run-unit-economics.py +71 -0
  382. package/skills/egbe-os-product-market-research/scripts/run-value-proposition.py +67 -0
  383. package/skills/egbe-os-product-market-research/scripts/upsert-markdown-section.py +156 -0
  384. package/skills/egbe-os-product-market-research/scripts/walk-funnels.py +249 -0
  385. package/skills/egbe-os-product-market-research/scripts/write-preview.py +710 -0
  386. package/skills/egbe-os-product-market-research/scripts/write-recommendations.py +272 -0
  387. package/skills/egbe-os-product-report/SKILL.md +150 -0
  388. package/skills/egbe-os-product-report/references/artifacts.md +84 -0
  389. package/skills/egbe-os-product-report/references/brief.md +34 -0
  390. package/skills/egbe-os-product-report/references/daily.md +76 -0
  391. package/skills/egbe-os-product-report/references/sources.md +68 -0
  392. package/skills/egbe-os-product-routines/SKILL.md +166 -0
  393. package/skills/egbe-os-product-routines/references/catalog.md +22 -0
  394. package/skills/egbe-os-product-routines/references/continuation-check.md +41 -0
  395. package/skills/egbe-os-product-routines/references/product-reporting.md +163 -0
  396. package/skills/egbe-os-product-routines/references/run-preset.md +32 -0
  397. package/skills/egbe-os-quickstart/scripts/egbe-curl.py +357 -0
  398. package/skills/egbe-os-quiz-funnel/SKILL.md +152 -0
  399. package/skills/egbe-os-quiz-funnel/agents/openai.yaml +4 -0
  400. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.d.mts +224 -0
  401. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.mjs +652 -0
  402. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-engine.test.mjs +764 -0
  403. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.d.mts +31 -0
  404. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.mjs +87 -0
  405. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-history.test.mjs +265 -0
  406. package/skills/egbe-os-quiz-funnel/assets/engine/quiz-reference.test.mjs +254 -0
  407. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/QUIZ.md +195 -0
  408. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.json +421 -0
  409. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/quiz.public.json +369 -0
  410. package/skills/egbe-os-quiz-funnel/assets/examples/egbe-docs/product/quizzes/books-checkup/validation.json +21 -0
  411. package/skills/egbe-os-quiz-funnel/assets/reference/index.html +26 -0
  412. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-bootstrap.js +31 -0
  413. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-early-actions.mjs +11 -0
  414. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-first-screen.mjs +32 -0
  415. package/skills/egbe-os-quiz-funnel/assets/reference/quiz-page.mjs +480 -0
  416. package/skills/egbe-os-quiz-funnel/assets/reference/quiz.css +264 -0
  417. package/skills/egbe-os-quiz-funnel/references/competitor-teardown.md +159 -0
  418. package/skills/egbe-os-quiz-funnel/references/conversion-handoff.md +156 -0
  419. package/skills/egbe-os-quiz-funnel/references/implementation.md +207 -0
  420. package/skills/egbe-os-quiz-funnel/references/inputs.md +37 -0
  421. package/skills/egbe-os-quiz-funnel/references/measurement.md +122 -0
  422. package/skills/egbe-os-quiz-funnel/references/spec-contract.md +335 -0
  423. package/skills/egbe-os-quiz-funnel/references/verification.md +101 -0
  424. package/skills/egbe-os-quiz-funnel/references/walk-mechanics.v2.json +18 -0
  425. package/skills/egbe-os-quiz-funnel/scripts/compile-quiz-spec.py +1869 -0
  426. package/skills/egbe-os-quiz-funnel/scripts/prepare-research-identity.py +66 -0
  427. package/skills/egbe-os-reddit-ads/SKILL.md +121 -0
  428. package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/schema.json +41 -0
  429. package/skills/egbe-os-reddit-ads/assets/static-banners/bold-statement/template.html +106 -0
  430. package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/schema.json +61 -0
  431. package/skills/egbe-os-reddit-ads/assets/static-banners/data-proof/template.html +151 -0
  432. package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/schema.json +91 -0
  433. package/skills/egbe-os-reddit-ads/assets/static-banners/honest-breakdown/template.html +185 -0
  434. package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/schema.json +51 -0
  435. package/skills/egbe-os-reddit-ads/assets/static-banners/question-hook/template.html +121 -0
  436. package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/schema.json +87 -0
  437. package/skills/egbe-os-reddit-ads/assets/static-banners/reddit-post-style/template.html +212 -0
  438. package/skills/egbe-os-reddit-ads/references/creative/html-to-png-pipeline.md +123 -0
  439. package/skills/egbe-os-reddit-ads/references/creative/static-banner-qa.md +12 -0
  440. package/skills/egbe-os-reddit-ads/references/creative/static-banner-system.md +16 -0
  441. package/skills/egbe-os-reddit-ads/references/destination-requirements.md +36 -0
  442. package/skills/egbe-os-reddit-ads/references/runtime/api-reference.md +435 -0
  443. package/skills/egbe-os-reddit-ads/references/runtime/artifacts.md +57 -0
  444. package/skills/egbe-os-reddit-ads/references/runtime/monitoring.md +64 -0
  445. package/skills/egbe-os-reddit-ads/references/runtime/pixel-and-tracking.md +49 -0
  446. package/skills/egbe-os-reddit-ads/references/runtime/policies.md +19 -0
  447. package/skills/egbe-os-reddit-ads/references/runtime/shared-analytics-attribution.md +14 -0
  448. package/skills/egbe-os-reddit-ads/references/workflows/build-paused.md +35 -0
  449. package/skills/egbe-os-reddit-ads/references/workflows/launch-ready.md +19 -0
  450. package/skills/egbe-os-reddit-ads/references/workflows/launch.md +32 -0
  451. package/skills/egbe-os-reddit-ads/references/workflows/readiness.md +9 -0
  452. package/skills/egbe-os-reddit-ads/scripts/capture-ad.sh +178 -0
  453. package/skills/egbe-os-reddit-ads/scripts/validate-reddit-artifacts.py +187 -0
  454. package/skills/egbe-os-report-ui/SKILL.md +133 -0
  455. package/skills/egbe-os-report-ui/assets/egbe-wordmark.svg +9 -0
  456. package/skills/egbe-os-report-ui/assets/examples/components.html +417 -0
  457. package/skills/egbe-os-report-ui/assets/examples/editorial.html +209 -0
  458. package/skills/egbe-os-report-ui/assets/scaffold.html +179 -0
  459. package/skills/egbe-os-report-ui/references/components.md +209 -0
  460. package/skills/egbe-os-research/SKILL.md +125 -0
  461. package/skills/egbe-os-saas-unit-economics/SKILL.md +188 -0
  462. package/skills/egbe-os-scrape/SKILL.md +160 -0
  463. package/skills/egbe-os-search/SKILL.md +182 -0
  464. package/skills/egbe-os-search/references/google-places.md +81 -0
  465. package/skills/egbe-os-support/SKILL.md +149 -0
  466. package/skills/egbe-os-telemetry/SKILL.md +344 -0
  467. package/skills/egbe-os-telemetry/references/harnesses/claude-code.md +97 -0
  468. package/skills/egbe-os-telemetry/references/harnesses/codex.md +22 -0
  469. package/skills/egbe-os-telemetry/references/harnesses/hermes.md +39 -0
  470. package/skills/egbe-os-telemetry/references/harnesses/openclaw.md +139 -0
  471. package/skills/egbe-os-telemetry/references/harnesses/opencode.md +28 -0
  472. package/skills/egbe-os-tools/SKILL.md +180 -0
  473. package/skills/egbe-os-tools/references/theona.md +26 -0
  474. package/skills/egbe-os-wallet/SKILL.md +181 -0
  475. package/skills/egbe-setup/EGBE.md +484 -0
  476. package/skills/egbe-setup/SKILL.md +116 -0
  477. package/skills/egbe-setup/references/project.md +80 -0
  478. package/state.js +37 -0
  479. package/update.js +361 -0
@@ -0,0 +1,1543 @@
1
+ #!/usr/bin/env python3
2
+ """Lifecycle Capture runner (internal section of the competitor-led process).
3
+
4
+ What the competitors send after someone signs up: the welcome sequence, its
5
+ cadence, offers, discounts and deadlines. The claw's own mailbox is the
6
+ secret shopper: one ordinary plus-addressed alias per competitor, plus stable
7
+ per-funnel aliases imported from section 05
8
+ (`<local>+NN@<domain>`, NN from 10–99), so every message is attributed by the
9
+ address it was sent to.
10
+
11
+ Two phases, one command:
12
+
13
+ - Start: the runner plans one alias per competitor and names the entry
14
+ points from an agent's reviewed discovery; the assigned human subscribes (newsletter, free account, trial
15
+ or waitlist that needs no payment) in the browser and the agent records each
16
+ subscription in `subscriptions.json`. The runner validates the file and
17
+ publishes the subscriptions. Until every competitor is subscribed or
18
+ skipped with a reason, the receipt stays `pending`.
19
+ - Harvest: the runner reads the inbox by alias through OS (`GET /mail/inbox`
20
+ with `to` and `since`), reads new messages once each, extracts cadence,
21
+ offers, discounts and deadlines, and republishes the section. Run it
22
+ again as mail accumulates; a later harvest replaces the earlier one.
23
+
24
+ Policy, approved for this section only: research sign-ups on competitor
25
+ sites with the claw's plus-addressed mailbox alias; no payments and no card
26
+ details; no agent-created accounts or required phone; no CAPTCHA bypass;
27
+ no real third-party identities; unsubscribe when
28
+ the capture ends. Message bodies stay in the runtime cache; the evidence
29
+ carries subjects and extracted facts, never bodies or excerpts.
30
+
31
+ Usage:
32
+ python3 run-lifecycle-capture.py --workspace . --product-domain product.com # start, or harvest once subscribed
33
+ python3 run-lifecycle-capture.py --workspace . --product-domain product.com --harvest # harvest explicitly
34
+ python3 run-lifecycle-capture.py --workspace . --funnel-mail --refresh # same-day 05 mail
35
+ """
36
+ from __future__ import annotations
37
+
38
+ import sys as _artifact_sys
39
+ from pathlib import Path as _ArtifactPath
40
+ _artifact_sys.path.insert(0, str(_ArtifactPath(__file__).resolve().parent))
41
+ from egbe_artifacts import artifact_path as _artifact_path
42
+
43
+ import runpy as _egbe_runpy
44
+ from pathlib import Path as _EgbePath
45
+ _egbe_adapter = _EgbePath(__file__).with_name("connection_identity.py")
46
+ _connection_headers = _egbe_runpy.run_path(str(_egbe_adapter))["connection_headers"] if _egbe_adapter.is_file() else lambda _url: {}
47
+
48
+ import argparse
49
+ import fcntl
50
+ import hashlib
51
+ import html
52
+ import importlib.util
53
+ import json
54
+ import os
55
+ import re
56
+ import sys
57
+ import tempfile
58
+ import urllib.error
59
+ import urllib.parse
60
+ import urllib.request
61
+ import uuid
62
+ from collections import Counter
63
+ from dataclasses import dataclass, field
64
+ from datetime import date, datetime, timedelta, timezone
65
+ from email.utils import getaddresses, parseaddr, parsedate_to_datetime
66
+ from pathlib import Path
67
+ from typing import Any, Callable
68
+
69
+ SCHEMA_VERSION = 1
70
+ SECTION_ID = "lifecycle_capture"
71
+ SECTION_HEADING = "Lifecycle capture"
72
+ DISPLAY_NAME = "Lifecycle Capture"
73
+ DEFAULT_OS_URL = "https://os.hz.egbe.dev"
74
+ RUNTIME_REL = Path(_artifact_path("egbe-docs/research/market/work")) / SECTION_ID
75
+ EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/lifecycle-capture.json"))
76
+ REPORT_REL = Path(_artifact_path("egbe-docs/research/market/full-report.md"))
77
+ FUNNEL_TEARDOWN_EVIDENCE_REL = Path(_artifact_path("egbe-docs/research/market/evidence/funnel-teardown.json"))
78
+ SUBSCRIPTIONS_FILENAME = "subscriptions.json"
79
+ PLAN_FILENAME = "plan.json"
80
+ ENTRY_POINTS_FILENAME = "entry-points.json"
81
+ OPERATOR_ACTIONS_FILENAME = "operator-actions.json"
82
+ IDENTITY_REL = Path(_artifact_path("egbe-docs/research/market/work/identity-registry.json"))
83
+ FUNNEL_HANDOFF_REL = Path(_artifact_path("egbe-docs/research/market/work/funnel_teardown/lifecycle-handoff.json"))
84
+ FUNNEL_EVENTS_FILENAME = "funnel-events.json"
85
+ FUNNEL_KINDS = ("lead_form", "quiz", "webinar_registration")
86
+ ALIAS_PREFIX = "lc-"
87
+
88
+ MAX_COMPETITORS = 10
89
+ MAX_INBOX_CALLS_PER_ATTEMPT = 30 # one per subscription per harvest; each is one IMAP login at the mail server
90
+ MAX_MESSAGE_READS_PER_ATTEMPT = 60 # bodies are read once and cached for good
91
+ INBOX_PAGE = 50
92
+ HARVEST_AFTER_DAYS = 14 # the sequence worth reading has had time to arrive
93
+ EARLY_HARVEST_AFTER_DAYS = 3
94
+ EXCERPT_CHARS = 160
95
+ SUBSCRIPTION_KINDS = ("newsletter", "account", "trial", "waitlist", "lead_form")
96
+
97
+ _DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
98
+ _URL_RE = re.compile(r"^https?://[^\s<>\"']+$", re.IGNORECASE)
99
+ _CONFIRM_RE = re.compile(r"\b(?:confirm|verify|verification|activate|validate)\b", re.IGNORECASE)
100
+ _DEADLINE_RE = re.compile(r"\b(?:ends? (?:today|tonight|tomorrow|soon|in \d+ (?:hours?|days?))|expires?(?: in \d+ (?:hours?|days?))?|last (?:chance|day|call)|"
101
+ r"today only|\d+ hours? left|final hours|limited time|don't miss|hurry|offer ends|sale ends|closing soon)\b", re.IGNORECASE)
102
+ _DISCOUNT_RE = re.compile(r"(?<!\d)(\d{1,2})\s?%\s?(?:off|discount|savings?)|(?:save|discount of)\s*(?:up to\s*)?(\d{1,2})\s?%", re.IGNORECASE)
103
+ _AMOUNT = r"\d{1,3}(?:[.,  ]\d{3})+(?:[.,]\d{1,2})?|\d+(?:[.,]\d{1,2})?"
104
+ _PRICE_RE = re.compile(rf"(?:[$€£]\s?(?:{_AMOUNT})(?!\d)(?:\s*(?:/\s?(?:mo|month|yr|year)\b|per\s+(?:month|year)|a\s+(?:month|year)))?"
105
+ rf"|(?<!\d)(?:{_AMOUNT})\s?(?:USD|EUR|GBP|€|£)(?![a-z]))", re.IGNORECASE)
106
+ _FREE_RE = re.compile(r"\b(?:free trial|for free|100% free|try (?:it )?free|free (?:plan|month|week|course|lesson)|no credit card)\b", re.IGNORECASE)
107
+ _UNSUB_RE = re.compile(r"unsubscribe|manage (?:your )?(?:email )?preferences|opt[- ]out", re.IGNORECASE)
108
+ _PROGRESS_RE = re.compile(r"\b(?:your progress|pick up where|continue (?:your|learning|the course)|you(?:'| a)re almost|streak|keep going|don't stop|still waiting)\b", re.IGNORECASE)
109
+ _LINK_RE = re.compile(r"https?://[^\s<>\"')\]]+", re.IGNORECASE)
110
+
111
+
112
+ def _load_script(filename: str, module_name: str):
113
+ path = Path(__file__).with_name(filename)
114
+ spec = importlib.util.spec_from_file_location(module_name, path)
115
+ if spec is None or spec.loader is None:
116
+ raise RuntimeError(f"cannot load {filename}")
117
+ module = importlib.util.module_from_spec(spec)
118
+ sys.dont_write_bytecode = True
119
+ sys.modules[module_name] = module
120
+ spec.loader.exec_module(module)
121
+ return module
122
+
123
+
124
+ CF = _load_script("run-channel-footprint.py", "pmr_channel_footprint_for_lifecycle_capture")
125
+ LifecycleCaptureError = CF.ChannelFootprintError
126
+ BudgetExhausted = CF.BudgetExhausted
127
+ registrable_domain = CF.registrable_domain
128
+ brand_token = CF.brand_token
129
+ START = _load_script("lifecycle-start.py", "pmr_lifecycle_start")
130
+ FUNNEL = _load_script("funnel-model.py", "pmr_funnel_model_for_lifecycle_mail")
131
+
132
+ # Mail-specific access material, not a general personal-data filter. Ordinary
133
+ # bodies, addresses, offer codes and campaign URLs remain portable evidence.
134
+ _MAIL_LINK_RE = re.compile(r"(?:https?://|https?%3a%2f%2f)[^\s<>\"')\]]+", re.IGNORECASE)
135
+ _MAIL_SECRET_FIELDS = frozenset({"password", "access_token", "refresh_token", "id_token", "auth_token",
136
+ "token", "otp", "verification_code", "confirmation_code", "recovery_code"})
137
+ _MAIL_CREDENTIAL_LABEL = (r"(?:(?:one[- ]time|verification|confirmation|security|login|sign[- ]in|access|authentication)\s+(?:code|password)"
138
+ r"|(?:temporary\s+)?password|(?:access|refresh|auth)[-_ ]token|OTP)")
139
+ _MAIL_GAP = r"(?:\s|<[^>]*>)*"
140
+ _MAIL_CREDENTIAL_RE = re.compile(
141
+ rf"(\b{_MAIL_CREDENTIAL_LABEL}{_MAIL_GAP}(?:(?:is\b|[:=]){_MAIL_GAP})?[\"']?)"
142
+ r"([A-Za-z0-9][A-Za-z0-9._~+/=-]{2,})", re.IGNORECASE)
143
+ _MAIL_CREDENTIAL_BEFORE_LABEL_RE = re.compile(
144
+ rf"\b([A-Za-z0-9][A-Za-z0-9._~+/=-]{{2,}})(\s+is\s+(?:(?:your|the)\s+)?{_MAIL_CREDENTIAL_LABEL}\b)", re.IGNORECASE)
145
+ _MAIL_BEARER_RE = re.compile(r"\bBearer\s+[A-Za-z0-9._~+/=-]+", re.IGNORECASE)
146
+ _MAIL_SIGNIN_SUBJECT_RE = re.compile(r"\b(?:sign[- ]?in|log[- ]?in)\b", re.IGNORECASE)
147
+ _MAIL_ACCESS_RE = re.compile(r"\b(?:magic[- ]link|password reset|reset (?:your )?password|"
148
+ r"one[- ]time (?:code|password)|(?:verification|confirmation|security|access|authentication|login|sign[- ]in) code|OTP|"
149
+ r"code (?:to|for) (?:sign[- ]?in|log[- ]?in)|"
150
+ r"(?:confirm|verify|activate) (?:your )?(?:email|account))\b", re.IGNORECASE)
151
+ # Access messages often separate the label and value (including in their HTML
152
+ # alternative). Apply these only to message content, never IDs, dates or addresses.
153
+ _MAIL_NUMERIC_CODE_RE = re.compile(r"(?<![\w$€£./:-])(?:\d{3}[ \t\u00a0-]\d{3}|\d{4,10})(?![\w/-])")
154
+ _MAIL_CODE_LINE_RE = re.compile(r"(^|>)([ \t]*)([A-Z0-9]{4,16})([ \t]*)(?=$|<)", re.MULTILINE | re.IGNORECASE)
155
+ _MAIL_CONTENT_FIELDS = frozenset({"subject", "text", "html"})
156
+
157
+
158
+ def portable_mail_response(value: Any, *, _access_message: bool = False, _content: bool = False) -> Any:
159
+ """Remove usable access material before it reaches any versionable file."""
160
+ if isinstance(value, dict):
161
+ subject = str(value.get("subject") or "")
162
+ content = str(value.get("text") or "")[:600] + "\n" + re.sub(
163
+ r"<[^>]*>", " ", html.unescape(str(value.get("html") or "")))[:600]
164
+ _access_message = bool(_access_message or _CONFIRM_RE.search(subject)
165
+ or _MAIL_SIGNIN_SUBJECT_RE.search(subject)
166
+ or _MAIL_ACCESS_RE.search(subject + "\n" + content))
167
+ return {key: "[credential]" if key.lower().replace("-", "_") in _MAIL_SECRET_FIELDS and item
168
+ else portable_mail_response(item, _access_message=_access_message,
169
+ _content=key.lower() in _MAIL_CONTENT_FIELDS) for key, item in value.items()}
170
+ if isinstance(value, list):
171
+ return [portable_mail_response(item, _access_message=_access_message, _content=_content) for item in value]
172
+ if not isinstance(value, str):
173
+ return value
174
+
175
+ def link(match: re.Match[str]) -> str:
176
+ raw = match.group(0)
177
+ decoded = raw
178
+ for _ in range(3):
179
+ decoded = urllib.parse.unquote(html.unescape(decoded))
180
+ try:
181
+ parts = urllib.parse.urlsplit(decoded)
182
+ fragment_keys = {key.lower() for key, _ in urllib.parse.parse_qsl(parts.fragment)}
183
+ # Access mail can wrap its credential link in an opaque click URL.
184
+ # Preserve the body, but keep all its usable links in the local original.
185
+ sensitive = (_access_message or FUNNEL._private_mail_url(decoded, [], True) or parts.username or parts.password
186
+ or fragment_keys & (_MAIL_SECRET_FIELDS | {"code", "key", "auth", "signature"})
187
+ or re.search(r"/(?:login|sign-?in|reset-password|password-reset|recover|recovery|unsubscribe)(?:/|$)", parts.path, re.I))
188
+ except ValueError:
189
+ sensitive = True
190
+ return "[access-link]" if sensitive else raw
191
+
192
+ decoded = html.unescape(value) if _content else value
193
+ cleaned = _MAIL_LINK_RE.sub(link, decoded)
194
+ if _access_message and _content:
195
+ cleaned = _MAIL_NUMERIC_CODE_RE.sub("[credential]", cleaned)
196
+ cleaned = _MAIL_CODE_LINE_RE.sub(
197
+ lambda match: match.group(1) + match.group(2) + "[credential]" + match.group(4)
198
+ if any(c.isdigit() for c in match.group(3)) else match.group(0), cleaned)
199
+ cleaned = _MAIL_CREDENTIAL_BEFORE_LABEL_RE.sub(lambda match: "[credential]" + match.group(2), cleaned)
200
+ cleaned = _MAIL_BEARER_RE.sub("Bearer [credential]", _MAIL_CREDENTIAL_RE.sub(
201
+ lambda match: match.group(1) + "[credential]", cleaned))
202
+ return value if cleaned == decoded else cleaned
203
+
204
+
205
+ # ── client ──────────────────────────────────────────────────────────────────
206
+
207
+ @dataclass
208
+ class OsClient(CF.OsClient):
209
+ """The Channel Footprint client plus GET calls to the OS mail routes. The
210
+ mailbox address is cached for the runtime; inbox listings are cached per
211
+ harvest day; message bodies are cached once for good."""
212
+
213
+ max_inbox_calls: int = MAX_INBOX_CALLS_PER_ATTEMPT
214
+ max_message_reads: int = MAX_MESSAGE_READS_PER_ATTEMPT
215
+ inbox_calls: int = 0
216
+ message_reads: int = 0
217
+ shared_dir: Path | None = None
218
+ secret_dir: Path | None = None
219
+ get: Callable[..., dict[str, Any]] | None = None
220
+ inbox_checks: dict[str, dict[str, Any]] = field(default_factory=dict)
221
+ last_get_at: str | None = None
222
+
223
+ def _isolate_mail_credentials(self, entry: dict[str, Any]) -> dict[str, Any]:
224
+ original = entry["response"]
225
+ portable = portable_mail_response(original)
226
+ if portable == original:
227
+ return entry
228
+ if isinstance(original, dict) and isinstance(original.get("text"), str):
229
+ portable["mail_content_metrics"] = {"links": len(_LINK_RE.findall(original["text"])),
230
+ "word_count": len(original["text"].split()),
231
+ "unsubscribe_link": bool(_UNSUB_RE.search(original["text"])),
232
+ "confirmation_in_prefix": bool(_CONFIRM_RE.search(original["text"][:600]))}
233
+ # Store the original only inside the existing ignored .local boundary,
234
+ # before even a temporary versionable cache file can contain credentials.
235
+ secret_dir = self.secret_dir or self._shared() / ".local" / "mail-cache"
236
+ if ".local" not in secret_dir.parts:
237
+ raise LifecycleCaptureError("credential-bearing mail requires a .local cache directory")
238
+ for parent in (secret_dir, *secret_dir.parents):
239
+ if parent.is_symlink():
240
+ raise LifecycleCaptureError("credential-bearing mail cache must not use symlinks")
241
+ if parent.name == ".local":
242
+ break
243
+ secret_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
244
+ encoded = json.dumps(entry, ensure_ascii=False, sort_keys=True)
245
+ digest = hashlib.sha256(encoded.encode("utf-8")).hexdigest()
246
+ with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=secret_dir, suffix=".tmp", delete=False) as handle:
247
+ temporary = Path(handle.name)
248
+ try:
249
+ handle.write(encoded)
250
+ handle.flush()
251
+ os.fsync(handle.fileno())
252
+ os.replace(temporary, secret_dir / f"{digest}.json")
253
+ finally:
254
+ temporary.unlink(missing_ok=True)
255
+ return {**entry, "response": portable, "credentials_isolated": True}
256
+
257
+ def _shared(self) -> Path:
258
+ path = self.shared_dir or self.cache_dir
259
+ path.mkdir(parents=True, exist_ok=True)
260
+ return path
261
+
262
+ def _http_get(self, path: str, params: dict[str, Any] | None = None, timeout: float = 60.0) -> dict[str, Any]:
263
+ query = urllib.parse.urlencode({k: v for k, v in (params or {}).items() if v not in (None, "")})
264
+ url = f"{self.os_url.rstrip('/')}{path}" + (f"?{query}" if query else "")
265
+ req = urllib.request.Request(url, headers={"Authorization": f"Bearer {self.api_key}", "Accept": "application/json"}, method="GET")
266
+ try:
267
+ for _name, _value in _connection_headers(req.full_url).items():
268
+ req.add_unredirected_header(_name, _value)
269
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
270
+ envelope = json.loads(resp.read().decode("utf-8"))
271
+ except urllib.error.HTTPError as exc:
272
+ # Upstream bodies may echo mail access material; status is enough
273
+ # for retry/coverage decisions and is safe for caller output.
274
+ raise LifecycleCaptureError(f"OS {path} failed: HTTP {exc.code}") from exc
275
+ except urllib.error.URLError as exc:
276
+ raise CF.OsUnreachable(f"OS {path} unreachable: {exc.reason}") from exc
277
+ except (OSError, ValueError) as exc:
278
+ raise LifecycleCaptureError(f"OS {path} failed in transit: {type(exc).__name__}: {str(exc)[:200]}") from exc
279
+ data = envelope.get("data")
280
+ if envelope.get("code") not in (None, 200) or not isinstance(data, dict):
281
+ raise LifecycleCaptureError(f"OS {path} returned no valid mail data")
282
+ return data
283
+
284
+ def _cached_get(self, kind: str, path: str, params: dict[str, Any], *, cache_dir: Path, budget_attr: str, budget_max: int, label: str,
285
+ timeout: float) -> tuple[dict[str, Any], str]:
286
+ digest = hashlib.sha256(json.dumps({"kind": kind, "path": path, "params": params}, sort_keys=True).encode("utf-8")).hexdigest()
287
+ cache_file = cache_dir / f"{digest}.json"
288
+ self.touched.add(digest)
289
+ entry = self._cache_read(cache_file)
290
+ if entry is not None:
291
+ self.cache_hits += 1
292
+ self.last_get_at = entry.get("retrieved_at")
293
+ response = entry.get("response")
294
+ if portable_mail_response(response) != response:
295
+ raise LifecycleCaptureError("saved mail cache contains access material; secure the local original and sanitize the cache before resuming")
296
+ return (response if isinstance(response, dict) else {}), entry.get("retrieved_on") or date.today().isoformat()
297
+ if getattr(self, budget_attr) >= budget_max:
298
+ raise BudgetExhausted(f"{label} budget exhausted ({budget_max} per attempt); {self.cache_writes} new answer(s) cached in this attempt; rerun to continue from the cache")
299
+ setattr(self, budget_attr, getattr(self, budget_attr) + 1)
300
+ transport = self.get or self._http_get
301
+ attempts = 0
302
+ while True:
303
+ try:
304
+ response = transport(path, params, timeout)
305
+ break
306
+ except LifecycleCaptureError as exc:
307
+ unreachable = isinstance(exc, CF.OsUnreachable)
308
+ if attempts >= 1 or not (unreachable or re.search(r"HTTP 5\d\d", str(exc))):
309
+ raise
310
+ attempts += 1
311
+ (self.sleep or __import__("time").sleep)(CF.OS_UNREACHABLE_RETRY_DELAY_SECONDS if unreachable else CF.FREE_RETRY_DELAY_SECONDS)
312
+ retrieved_on = date.today().isoformat()
313
+ self.last_get_at = datetime.now().astimezone().isoformat()
314
+ entry = self._isolate_mail_credentials({"kind": kind, "path": path, "params": params, "response": response,
315
+ "retrieved_on": retrieved_on, "retrieved_at": self.last_get_at})
316
+ cache_dir.mkdir(parents=True, exist_ok=True)
317
+ tmp = cache_file.with_name(cache_file.name + ".tmp")
318
+ tmp.write_text(json.dumps(entry, ensure_ascii=False), encoding="utf-8")
319
+ os.replace(tmp, cache_file)
320
+ self.cache_writes += 1
321
+ return entry["response"], retrieved_on
322
+
323
+ def mail_account(self) -> str:
324
+ response, _ = self._cached_get("mail_account", "/mail/account", {}, cache_dir=self._shared(), budget_attr="inbox_calls",
325
+ budget_max=self.max_inbox_calls, label="mail account", timeout=30.0)
326
+ email = str(response.get("email") or "").strip().lower()
327
+ if "@" not in email:
328
+ raise LifecycleCaptureError("the claw has no mailbox: GET /mail/account returned no address")
329
+ return email
330
+
331
+ def reserve_aliases(self, capture_id: str, domains: list[str], mailbox: str) -> dict[str, str]:
332
+ """The OS registry atomically reserves neutral aliases across workspaces.
333
+ A retry uses the same durable capture id, including after a lost response."""
334
+ payload = {"capture_id": capture_id, "domains": domains}
335
+ data = self._http_post("/mail/lifecycle-aliases/reserve", payload)
336
+ aliases = data.get("aliases")
337
+ if str(data.get("mailbox") or "").lower() != mailbox or not isinstance(aliases, dict):
338
+ raise LifecycleCaptureError("alias reservation returned a different mailbox or no mapping")
339
+ local, _, host = mailbox.partition("@")
340
+ for domain in domains:
341
+ alias = str(aliases.get(domain) or "").lower()
342
+ if not re.fullmatch(re.escape(local) + r"\+[1-9][0-9]@" + re.escape(host), alias):
343
+ raise LifecycleCaptureError("alias reservation returned an invalid neutral alias")
344
+ if len({aliases[d] for d in domains}) != len(domains):
345
+ raise LifecycleCaptureError("alias reservation reused a number for different competitors")
346
+ return {d: str(aliases[d]).lower() for d in domains}
347
+
348
+ def _http_post(self, path: str, payload: dict[str, Any], timeout: float = 30.0) -> dict[str, Any]:
349
+ req = urllib.request.Request(f"{self.os_url.rstrip('/')}{path}", data=json.dumps(payload).encode(),
350
+ headers={"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"}, method="POST")
351
+ try:
352
+ for _name, _value in _connection_headers(req.full_url).items():
353
+ req.add_unredirected_header(_name, _value)
354
+ with urllib.request.urlopen(req, timeout=timeout) as response:
355
+ envelope = json.loads(response.read().decode())
356
+ except urllib.error.HTTPError as exc:
357
+ raise LifecycleCaptureError(f"alias reservation failed: HTTP {exc.code}; the neutral number pool may be exhausted or unavailable") from exc
358
+ except (urllib.error.URLError, OSError, ValueError) as exc:
359
+ raise LifecycleCaptureError(f"alias reservation unavailable: {type(exc).__name__}; rerun with the same capture") from exc
360
+ if envelope.get("code") != 200 or not isinstance(envelope.get("data"), dict):
361
+ raise LifecycleCaptureError("alias reservation returned no mapping; rerun with the same capture")
362
+ return envelope["data"]
363
+
364
+ def inbox(self, alias: str, since: str) -> tuple[list[dict[str, Any]], str]:
365
+ response, retrieved_on = self._cached_get("inbox", "/mail/inbox", {"to": alias, "since": since, "limit": INBOX_PAGE}, cache_dir=self.cache_dir,
366
+ budget_attr="inbox_calls", budget_max=self.max_inbox_calls, label="inbox listing", timeout=90.0)
367
+ messages = response.get("messages")
368
+ if not isinstance(messages, list):
369
+ raise LifecycleCaptureError("OS inbox response has no messages list")
370
+ self.inbox_checks[alias] = {"checked_at": self.last_get_at, "inbox_since": since, "query_succeeded": True}
371
+ return [m for m in messages if isinstance(m, dict) and m.get("uid")], retrieved_on
372
+
373
+ def read_message(self, uid: str) -> dict[str, Any]:
374
+ response, _ = self._cached_get("message", f"/mail/message/{urllib.parse.quote(str(uid))}", {}, cache_dir=self._shared() / "messages",
375
+ budget_attr="message_reads", budget_max=self.max_message_reads, label="message read", timeout=90.0)
376
+ return response
377
+
378
+ def ledger_totals(self) -> dict[str, Any]:
379
+ counts = {"mail_account": 0, "inbox": 0, "message": 0}
380
+ dates: list[str] = []
381
+ for base in (self.cache_dir, self._shared(), self._shared() / "messages"):
382
+ if not base.is_dir():
383
+ continue
384
+ for digest in sorted(self.touched):
385
+ path = base / f"{digest}.json"
386
+ if not path.is_file():
387
+ continue
388
+ try:
389
+ entry = json.loads(path.read_text(encoding="utf-8"))
390
+ except (OSError, json.JSONDecodeError):
391
+ continue
392
+ kind = entry.get("kind")
393
+ if kind in counts:
394
+ counts[kind] += 1
395
+ if entry.get("retrieved_on"):
396
+ dates.append(str(entry["retrieved_on"]))
397
+ total = sum(counts.values())
398
+ return {**counts, "provider_calls": total, "observations": total,
399
+ "first_retrieved_on": min(dates) if dates else date.today().isoformat(),
400
+ "last_retrieved_on": max(dates) if dates else date.today().isoformat()}
401
+
402
+
403
+ # ── plan and subscriptions ──────────────────────────────────────────────────
404
+
405
+ def alias_for(mailbox: str, domain: str) -> str:
406
+ """One alias per registrable domain: `lc-coursera-org`, `lc-mural-co`.
407
+ The suffix keeps two competitors with the same brand stem apart."""
408
+ local, _, host = mailbox.partition("@")
409
+ tag = ALIAS_PREFIX + re.sub(r"[^a-z0-9-]", "", registrable_domain(domain).replace(".", "-"))[:48]
410
+ return f"{local}+{tag}@{host}"
411
+
412
+
413
+ def funnel_lineage(item: dict[str, Any]) -> dict[str, Any]:
414
+ """Public attribution; mailbox addresses and submission receipts stay private."""
415
+ return {key: item[key] for key in ("funnel_id", "campaign_refs", "path_refs") if key in item}
416
+
417
+
418
+ def identity_registry(workspace: Path, mailbox: str, aliases: dict[str, str]) -> dict[str, Any]:
419
+ """Reserve ordinary 02 aliases without replacing the stable identities from 05."""
420
+ path = workspace / IDENTITY_REL
421
+ path.parent.mkdir(parents=True, exist_ok=True)
422
+ with path.with_name(path.name + ".lock").open("a") as lock:
423
+ fcntl.flock(lock, fcntl.LOCK_EX)
424
+ if path.is_file():
425
+ try:
426
+ registry = json.loads(path.read_text(encoding="utf-8"))
427
+ except (OSError, ValueError) as exc:
428
+ raise LifecycleCaptureError("identity registry is not valid JSON") from exc
429
+ if not isinstance(registry, dict) or registry.get("schema_version") != 1 or registry.get("mailbox") != mailbox or not isinstance(registry.get("identities"), list):
430
+ raise LifecycleCaptureError("identity registry needs schema_version 1 and the current research mailbox")
431
+ else:
432
+ registry = {"schema_version": 1, "mailbox": mailbox, "identities": []}
433
+ identities = registry["identities"]
434
+ if any(not isinstance(item, dict) for item in identities):
435
+ raise LifecycleCaptureError("identity registry identities must be objects")
436
+ if len({item.get("alias") for item in identities}) != len(identities):
437
+ raise LifecycleCaptureError("identity registry aliases must be unique")
438
+ for domain, alias in aliases.items():
439
+ matches = [item for item in identities if item.get("alias") == alias]
440
+ if any(item.get("domain") != domain or item.get("funnel_id") is not None
441
+ or item.get("kind") not in {"lifecycle", "lifecycle_reserved"} for item in matches):
442
+ raise LifecycleCaptureError("ordinary lifecycle alias collides with another research identity")
443
+ # A restored, actually recorded legacy subscription can replace an
444
+ # unused neutral plan alias. Preserve that address's reservation,
445
+ # but keep one current mapping for the domain-scoped receipt hash.
446
+ # Changing a used mapping still invalidates its previous receipt.
447
+ for item in identities:
448
+ if item.get("kind") == "lifecycle" and item.get("domain") == domain and item.get("alias") != alias:
449
+ item.update(kind="lifecycle_reserved", identity_id="lifecycle-reservation:" + domain + ":" +
450
+ hashlib.sha256(item["alias"].encode()).hexdigest()[:20])
451
+ if matches:
452
+ matches[0].update(kind="lifecycle", identity_id=f"lifecycle:{domain}")
453
+ else:
454
+ identities.append({"identity_id": f"lifecycle:{domain}", "domain": domain, "funnel_id": None, "alias": alias, "kind": "lifecycle"})
455
+ _write_atomic(path, json.dumps(registry, indent=2, ensure_ascii=False) + "\n")
456
+ return registry
457
+
458
+
459
+ def load_funnel_handoff(workspace: Path, mailbox: str, registry: dict[str, Any]) -> dict[str, Any]:
460
+ """Import only journalled, approved registrations with reserved per-funnel aliases.
461
+
462
+ This accepts several funnels for one competitor. It neither creates accounts
463
+ nor relaxes the form/phone rules of ordinary Lifecycle Capture subscriptions.
464
+ """
465
+ path = workspace / FUNNEL_HANDOFF_REL
466
+ if not path.is_file():
467
+ return {"subscriptions": [], "events": []}
468
+ try:
469
+ raw = json.loads(path.read_text(encoding="utf-8"))
470
+ except (OSError, ValueError) as exc:
471
+ raise LifecycleCaptureError("funnel lifecycle handoff is not valid JSON") from exc
472
+ if not isinstance(raw, dict) or raw.get("schema_version") != 1 or not isinstance(raw.get("subscriptions"), list) or not isinstance(raw.get("events", []), list):
473
+ raise LifecycleCaptureError("funnel lifecycle handoff needs schema_version 1, subscriptions and events lists")
474
+ subscriptions: list[dict[str, Any]] = []
475
+ seen: set[str] = set()
476
+ for item in raw["subscriptions"]:
477
+ if not isinstance(item, dict):
478
+ raise LifecycleCaptureError("funnel subscription must be an object")
479
+ funnel_id = str(item.get("funnel_id") or "")
480
+ domain = registrable_domain(str(item.get("domain") or ""))
481
+ alias = str(item.get("alias") or "").strip().lower()
482
+ if not funnel_id or not domain or funnel_id in seen:
483
+ raise LifecycleCaptureError("funnel handoff needs one subscription per distinct funnel_id")
484
+ seen.add(funnel_id)
485
+ local, _, host = mailbox.partition("@")
486
+ if not re.fullmatch(re.escape(local) + r"\+\d{2,}@" + re.escape(host), alias):
487
+ raise LifecycleCaptureError("funnel handoff alias must be a numeric plus alias of the research mailbox")
488
+ if not any(identity.get("funnel_id") == funnel_id and identity.get("domain") == domain and identity.get("alias") == alias for identity in registry["identities"]):
489
+ raise LifecycleCaptureError("funnel handoff alias is not reserved for this funnel in the identity registry")
490
+ if not item.get("approval_ref") or not item.get("submission_ref"):
491
+ raise LifecycleCaptureError("funnel handoff requires approval_ref and submission_ref")
492
+ if item.get("kind") not in FUNNEL_KINDS:
493
+ raise LifecycleCaptureError("funnel handoff accepts lead_form, quiz or webinar_registration only")
494
+ try:
495
+ subscribed = date.fromisoformat(str(item.get("subscribed_on") or ""))
496
+ except ValueError as exc:
497
+ raise LifecycleCaptureError("funnel handoff needs a valid subscribed_on date") from exc
498
+ if subscribed > date.today():
499
+ raise LifecycleCaptureError("funnel handoff subscription is in the future")
500
+ entry_url = str(item.get("entry_url") or "")
501
+ if not _URL_RE.fullmatch(entry_url):
502
+ raise LifecycleCaptureError("funnel handoff entry_url needs its approved HTTP(S) form URL")
503
+ for key in ("campaign_refs", "path_refs"):
504
+ if not isinstance(item.get(key), list) or not item[key] or any(not isinstance(ref, str) or not ref for ref in item[key]):
505
+ raise LifecycleCaptureError(f"funnel handoff needs nonempty {key}")
506
+ if item.get("confirmed") is not None and not isinstance(item["confirmed"], bool):
507
+ raise LifecycleCaptureError("funnel handoff confirmed must be boolean or null")
508
+ expectations = item.get("email_expectations", {})
509
+ if not isinstance(expectations, dict) or any(expectations.get(key) is not None and type(expectations[key]) is not bool for key in ("confirmation", "result")):
510
+ raise LifecycleCaptureError("funnel handoff email expectations must be boolean or null")
511
+ if any(expectations.get(key) is not None for key in ("confirmation", "result")) and not str(expectations.get("source_ref") or "").startswith("browser:"):
512
+ raise LifecycleCaptureError("funnel handoff email expectations need a browser observation reference")
513
+ email = _load_script("funnel-email-review.py", "pmr_lifecycle_handoff_email")
514
+ if expectations.get("wait_until") is not None and (email.timestamp(expectations["wait_until"]) is None or
515
+ not any(expectations.get(key) is True for key in ("confirmation", "result")) or
516
+ not str(expectations.get("source_ref") or "").startswith("browser:")):
517
+ raise LifecycleCaptureError("expected email deadline needs an observed timestamp and browser reference")
518
+ if "submission_checks" in item:
519
+ checks = item["submission_checks"]
520
+ if not isinstance(checks, list) or not checks:
521
+ raise LifecycleCaptureError("funnel handoff submission_checks must describe each submitted path")
522
+ for check in checks:
523
+ if not isinstance(check, dict) or not check.get("submission_ref") or email.timestamp(check.get("submitted_at")) is None or check.get("path_id") not in item["path_refs"] or not isinstance(check.get("campaign_refs"), list) or not check["campaign_refs"] or not set(check["campaign_refs"]).issubset(item["campaign_refs"]):
524
+ raise LifecycleCaptureError("funnel handoff submission check is not attributable to its subscription")
525
+ per_submission = check.get("email_expectations", expectations)
526
+ if not isinstance(per_submission, dict) or any(type(per_submission.get(key)) not in (bool, type(None)) for key in ("confirmation", "result")) or (check.get("confirmed") is not None and type(check["confirmed"]) is not bool):
527
+ raise LifecycleCaptureError("funnel handoff submission expectations/confirmation must be observed booleans or unknown")
528
+ if any(per_submission.get(key) is not None for key in ("confirmation", "result")) and not str(per_submission.get("source_ref") or "").startswith("browser:"):
529
+ raise LifecycleCaptureError("funnel handoff submission expectations need their browser source")
530
+ if per_submission.get("wait_until") is not None and (email.timestamp(per_submission["wait_until"]) is None or email.timestamp(per_submission["wait_until"]) < email.timestamp(check["submitted_at"]) or not any(per_submission.get(key) is True for key in ("confirmation", "result"))):
531
+ raise LifecycleCaptureError("funnel handoff submission email deadline is invalid")
532
+ if item.get("unsubscribed_on") is not None:
533
+ try:
534
+ unsubscribed = date.fromisoformat(item["unsubscribed_on"])
535
+ except (ValueError, TypeError) as exc:
536
+ raise LifecycleCaptureError("funnel handoff has an invalid unsubscribed_on") from exc
537
+ if not subscribed <= unsubscribed <= date.today():
538
+ raise LifecycleCaptureError("funnel handoff has an invalid unsubscribed_on")
539
+ subscriptions.append({**item, "domain": domain, "alias": alias, "confirmed": item.get("confirmed"), "unsubscribed_on": item.get("unsubscribed_on"), "notes": item.get("notes")})
540
+ events: list[dict[str, Any]] = []
541
+ for event in raw.get("events", []):
542
+ if not isinstance(event, dict) or not event.get("event_id") or not event.get("occurred_at") or not event.get("source_ref"):
543
+ raise LifecycleCaptureError("funnel handoff events need event_id, occurred_at and source_ref")
544
+ subscription = next((item for item in subscriptions if item["funnel_id"] == event.get("funnel_id")), None)
545
+ if subscription is None or event.get("alias") != subscription["alias"] or event.get("type") not in ("email_confirmation", "email_result"):
546
+ raise LifecycleCaptureError("funnel handoff event is not attributed to a reserved subscription")
547
+ if not isinstance(event.get("campaign_refs"), list) or not set(event["campaign_refs"]).issubset(subscription["campaign_refs"]):
548
+ raise LifecycleCaptureError("funnel handoff event campaign refs do not match its subscription")
549
+ if event.get("path_id") is not None and event["path_id"] not in subscription["path_refs"]:
550
+ raise LifecycleCaptureError("funnel handoff event path does not match its subscription")
551
+ if event["type"] == "email_confirmation" and event.get("verification_status") == "confirmed":
552
+ subscription["confirmed"] = True
553
+ events.append(event)
554
+ return {"subscriptions": subscriptions, "events": events}
555
+
556
+
557
+ _RESULT_RE = re.compile(r"\b(?:your (?:quiz |assessment |audit |test )?results?|your (?:personalized|personalised) (?:plan|report)|assessment complete|quiz complete)\b", re.IGNORECASE)
558
+
559
+
560
+ def capture_funnel_events(client: OsClient, workspace: Path, handoff: dict[str, Any], *, refresh: bool = False) -> dict[str, Any]:
561
+ """Read same-day confirmation/results without completing section 02.
562
+
563
+ --refresh uses a new listing snapshot, even on the same day; message bodies
564
+ remain cached once. Receiving a confirmation is never treated as clicking it.
565
+ """
566
+ runtime = workspace / RUNTIME_REL
567
+ path = runtime / FUNNEL_EVENTS_FILENAME
568
+ previous = json.loads(path.read_text()) if path.is_file() else {"events": []}
569
+ events = {str(event["event_id"]): event for event in previous.get("events", [])}
570
+ events.update({str(event["event_id"]): event for event in handoff["events"]})
571
+ day = date.today().isoformat()
572
+ snapshot_path = runtime / "funnel-mail-snapshot.json"
573
+ snapshot_state = json.loads(snapshot_path.read_text()) if snapshot_path.is_file() else {}
574
+ if snapshot_state.get("day") == day and (not snapshot_state.get("completed") or not refresh):
575
+ snapshot = snapshot_state["snapshot"]
576
+ else:
577
+ snapshot = datetime.now(timezone.utc).strftime("%H%M%S%f") if refresh else "initial"
578
+ _write_atomic(snapshot_path, json.dumps({"day": day, "snapshot": snapshot, "completed": False}))
579
+ client.cache_dir = runtime / "cache" / "funnel-mail" / day / snapshot
580
+ coverage: list[dict[str, Any]] = []
581
+ messages, _ = harvest(client, handoff["subscriptions"], date.today(), coverage)
582
+ for message in messages:
583
+ subscription = next(item for item in handoff["subscriptions"] if item["alias"] == message["alias"])
584
+ body = client.read_message(message["uid"])
585
+ kind = "email_confirmation" if message["is_confirmation"] else "email_result" if _RESULT_RE.search(message["subject"] + "\n" + str(body.get("text") or "")[:1000]) else None
586
+ if kind is None:
587
+ continue
588
+ if not message["received_at"]:
589
+ CF._coverage_gap(coverage, "message", [subscription["domain"]], "confirmation/result arrival time was not observed")
590
+ continue
591
+ digest = hashlib.sha256(f"{subscription['funnel_id']}:{message['uid']}:{kind}".encode()).hexdigest()[:20]
592
+ event_id = f"mail-{digest}"
593
+ if event_id in events and events[event_id].get("verification_status") == "confirmed":
594
+ continue
595
+ events[event_id] = {"event_id": event_id, "type": kind, "funnel_id": subscription["funnel_id"],
596
+ "path_id": subscription["path_refs"][0] if len(subscription["path_refs"]) == 1 else None,
597
+ "campaign_refs": subscription["campaign_refs"], "alias": subscription["alias"],
598
+ "occurred_at": message["received_at"], "source_ref": f"mail:{message['uid']}", "message_ref": message["uid"],
599
+ "verification_status": "received", "result": {"subject": message["subject"], "excerpt": _excerpt(str(body.get("text") or ""))} if kind == "email_result" else None}
600
+ pending, checks = [], list(previous.get("checks", []))
601
+ email = _load_script("funnel-email-review.py", "pmr_lifecycle_email_review")
602
+ for subscription in handoff["subscriptions"]:
603
+ query = client.inbox_checks.get(subscription["alias"], {})
604
+ check = {"funnel_id": subscription["funnel_id"], "path_refs": subscription["path_refs"],
605
+ "campaign_refs": subscription["campaign_refs"], "alias": subscription["alias"],
606
+ "checked_at": query.get("checked_at"), "inbox_since": query.get("inbox_since") or
607
+ (date.fromisoformat(subscription["subscribed_on"]) - timedelta(days=1)).isoformat(),
608
+ "query_succeeded": query.get("query_succeeded") is True,
609
+ "source_ref": f"inbox_snapshot:{day}:{snapshot}"}
610
+ if check not in checks:
611
+ checks.append(check)
612
+ review = email.review(subscription, {"checks": checks, "events": list(events.values())}, harvest_after_days=HARVEST_AFTER_DAYS)
613
+ if review["pending"]:
614
+ reason = ("mailbox_check_failed" if query.get("query_succeeded") is not True else
615
+ "mailbox_check_time_unknown" if not query.get("checked_at") else review["detail"])
616
+ pending.append({"funnel_id": subscription["funnel_id"], "path_refs": subscription["path_refs"], "alias": subscription["alias"], "reason": reason})
617
+ if review["gaps"]:
618
+ coverage.append({"type": "coverage", **funnel_lineage(subscription), "status": "email_coverage_gap",
619
+ "detail": review["detail"], "gaps": review["gaps"], "unknown": review["unknown"], "terminal": not review["pending"], "tier": "C"})
620
+ result = {"schema_version": 1, "events": sorted(events.values(), key=lambda event: str(event.get("occurred_at") or "")),
621
+ "checks": checks, "pending": pending, "coverage": coverage, "checked_at": datetime.now(timezone.utc).isoformat()}
622
+ _write_atomic(path, json.dumps(result, indent=2, ensure_ascii=False) + "\n")
623
+ _write_atomic(snapshot_path, json.dumps({"day": day, "snapshot": snapshot, "completed": True}))
624
+ return result
625
+
626
+
627
+ def entry_points(workspace: Path, domains: list[str]) -> dict[str, list[str]]:
628
+ """Sign-up, lead-form, pricing and home addresses per competitor from a
629
+ completed Funnel Teardown; the home page otherwise."""
630
+ points: dict[str, list[str]] = {d: [f"https://{d}/"] for d in domains}
631
+ evidence = _load_script("research-evidence.py", "pmr_research_evidence_for_lifecycle_entry_points")
632
+ try:
633
+ outcome = evidence.inspect_section(workspace, "funnel_teardown", require_materialized=False)
634
+ except ValueError:
635
+ return points
636
+ if outcome.get("status") != "completed":
637
+ return points
638
+ try:
639
+ document = json.loads((workspace / FUNNEL_TEARDOWN_EVIDENCE_REL).read_text(encoding="utf-8"))
640
+ rows = document.get("rows") or []
641
+ except (OSError, json.JSONDecodeError, AttributeError):
642
+ return points
643
+ preferred = ("signup_page", "lead_form", "dedicated_landing", "pricing_page", "homepage")
644
+ browser_contract = document.get("schema_version", 1) >= 2
645
+ for domain in domains:
646
+ entries = {r.get("address") for r in rows if isinstance(r, dict) and r.get("type") == "funnel_walk"
647
+ and r.get("domain") == domain and r.get("role") != "own"}
648
+ pages = []
649
+ for row in rows:
650
+ if not isinstance(row, dict) or row.get("type") != "landing_page" or row.get("domain") != domain or row.get("role") == "own":
651
+ continue
652
+ address = str(row.get("address") or "")
653
+ if not _URL_RE.fullmatch(address) or registrable_domain(urllib.parse.urlsplit(address).hostname or "") != domain:
654
+ continue
655
+ if browser_contract:
656
+ # V2 publishes browser screens without the legacy fetch_status.
657
+ # A personalized result/confirmation is not a sign-up entry.
658
+ if row.get("source") != "browser" or not (row.get("page_type") in preferred or row.get("lead_fields") or address in entries):
659
+ continue
660
+ elif row.get("fetch_status") != "ok":
661
+ continue
662
+ pages.append(row)
663
+ pages.sort(key=lambda r: (preferred.index(r.get("page_type")) if r.get("page_type") in preferred else len(preferred),
664
+ 0 if browser_contract else -(r.get("weight") or 0)))
665
+ chosen = [r["address"] for r in pages[:4]]
666
+ if chosen:
667
+ points[domain] = list(dict.fromkeys(chosen + points[domain]))[:5]
668
+ return points
669
+
670
+
671
+ def load_subscriptions(path: Path, domains: list[str], aliases: dict[str, str], planned: dict[str, dict[str, Any]] | None = None) -> dict[str, Any] | None:
672
+ """The agent's subscriptions file, strictly validated. Missing means no
673
+ subscriptions yet. Every alias must be the planned alias of its
674
+ competitor: the capture never uses another address."""
675
+ if not path.is_file():
676
+ return None
677
+ try:
678
+ raw = json.loads(path.read_text(encoding="utf-8"))
679
+ except (OSError, json.JSONDecodeError) as exc:
680
+ raise LifecycleCaptureError(f"{path.name} is not valid JSON: {exc}") from exc
681
+ if not isinstance(raw, dict) or raw.get("schema_version") != 1 or not isinstance(raw.get("subscriptions"), list):
682
+ raise LifecycleCaptureError(f"{path.name} must be an object with schema_version 1 and a subscriptions list")
683
+ subscriptions: list[dict[str, Any]] = []
684
+ seen: set[str] = set()
685
+ for index, item in enumerate(raw["subscriptions"], 1):
686
+ label = f"subscription #{index}"
687
+ if not isinstance(item, dict):
688
+ raise LifecycleCaptureError(f"{label} must be an object")
689
+ domain = registrable_domain(str(item.get("domain") or ""))
690
+ if domain not in domains:
691
+ continue # recorded for a competitor outside this run; kept in the file, not published here
692
+ if domain in seen:
693
+ raise LifecycleCaptureError(f"{label} subscribes `{domain}` twice")
694
+ seen.add(domain)
695
+ alias = str(item.get("alias") or "").strip().lower()
696
+ if alias != aliases[domain]:
697
+ raise LifecycleCaptureError(f"{label} must use the planned alias {aliases[domain]} (got {item.get('alias')!r})")
698
+ subscribed_on = str(item.get("subscribed_on") or "")
699
+ if not _DATE_RE.match(subscribed_on):
700
+ raise LifecycleCaptureError(f"{label} needs subscribed_on as YYYY-MM-DD")
701
+ try:
702
+ subscribed = date.fromisoformat(subscribed_on)
703
+ except ValueError as exc:
704
+ raise LifecycleCaptureError(f"{label} has an invalid subscribed_on date") from exc
705
+ if subscribed > date.today():
706
+ raise LifecycleCaptureError(f"{label} is subscribed in the future")
707
+ entry_url = str(item.get("entry_url") or "").strip()
708
+ has_reviewed_entry = planned is not None and any(planned[domain].get(key) for key in ("selected", "fallback"))
709
+ if not _URL_RE.match(entry_url) or (not has_reviewed_entry and registrable_domain(urllib.parse.urlsplit(entry_url).hostname or "") != domain):
710
+ raise LifecycleCaptureError(f"{label} needs entry_url as an https address on {domain}")
711
+ kind = str(item.get("kind") or "").strip().lower()
712
+ if kind not in SUBSCRIPTION_KINDS:
713
+ raise LifecycleCaptureError(f"{label} needs kind in {', '.join(SUBSCRIPTION_KINDS)}")
714
+ confirmed = item.get("confirmed", None)
715
+ if confirmed is not None and not isinstance(confirmed, bool):
716
+ raise LifecycleCaptureError(f"{label} has a non-boolean confirmed")
717
+ unsubscribed_on = item.get("unsubscribed_on")
718
+ if unsubscribed_on is not None:
719
+ if not isinstance(unsubscribed_on, str) or not _DATE_RE.match(unsubscribed_on) or not (subscribed <= date.fromisoformat(unsubscribed_on) <= date.today()):
720
+ raise LifecycleCaptureError(f"{label} has an invalid unsubscribed_on date")
721
+ bound_entry = START.subscription_entry({**item, "domain": domain}, planned[domain]) if planned is not None else {}
722
+ subscriptions.append({"domain": domain, "alias": alias, "subscribed_on": subscribed_on, "entry_url": entry_url, "kind": kind, **bound_entry,
723
+ "confirmed": confirmed, "unsubscribed_on": unsubscribed_on,
724
+ "notes": " ".join(str(item.get("notes") or "").split())[:240] or None})
725
+ if len({s["alias"] for s in subscriptions}) != len(subscriptions):
726
+ raise LifecycleCaptureError("two subscriptions share one alias; every competitor gets its own")
727
+ skipped: list[dict[str, str]] = []
728
+ for index, item in enumerate(raw.get("skipped") or [], 1):
729
+ if not isinstance(item, dict):
730
+ raise LifecycleCaptureError(f"skipped entry #{index} must be an object")
731
+ domain = registrable_domain(str(item.get("domain") or ""))
732
+ reason = " ".join(str(item.get("reason") or "").split())
733
+ if not domain or not reason:
734
+ raise LifecycleCaptureError(f"skipped entry #{index} needs a domain and a reason")
735
+ if domain not in domains:
736
+ continue
737
+ if domain in seen:
738
+ raise LifecycleCaptureError(f"skipped entry #{index} names `{domain}`, which is also subscribed")
739
+ skipped.append({"domain": domain, "reason": reason[:240]})
740
+ return {"subscriptions": subscriptions, "skipped": skipped}
741
+
742
+
743
+ # ── harvest ─────────────────────────────────────────────────────────────────
744
+
745
+ def _parse_received(message: dict[str, Any]) -> datetime | None:
746
+ for key in ("received_at", "date"):
747
+ value = message.get(key)
748
+ if not isinstance(value, str) or not value.strip():
749
+ continue
750
+ try:
751
+ if key == "received_at":
752
+ return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)
753
+ return parsedate_to_datetime(value).astimezone(timezone.utc)
754
+ except (ValueError, TypeError):
755
+ continue
756
+ return None
757
+
758
+
759
+ def _excerpt(text: str) -> str:
760
+ """The opening words without any link: confirmation and tracking links
761
+ carry per-recipient tokens and never leave the runtime cache."""
762
+ plain = " ".join(_LINK_RE.sub("[link]", text or "").split())
763
+ return plain[:EXCERPT_CHARS] + ("…" if len(plain) > EXCERPT_CHARS else "")
764
+
765
+
766
+ _ADDRESS_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
767
+
768
+
769
+ PRIVATE_ROW_FIELDS = ("alias", "uid", "message_ref", "source_ref", "approval_ref", "submission_ref")
770
+ _MESSAGE_REF_RE = re.compile(r"/mail/message/[^\s\"')\]]+|\buid\s+[A-Za-z0-9_-]+", re.IGNORECASE)
771
+ _WELCOME_RE = re.compile(r"\bwelcome\b", re.IGNORECASE)
772
+
773
+
774
+ def private_strings(aliases: Any, mailbox: str) -> tuple[str, ...]:
775
+ """Every spelling of an address that must not be published: the planned
776
+ aliases and the mailbox, plain and percent-encoded (an alias echoed in a
777
+ confirmation link or a server error comes back encoded)."""
778
+ out: list[str] = []
779
+ for value in [*aliases, mailbox]:
780
+ value = str(value or "").strip()
781
+ if not value:
782
+ continue
783
+ out += [value, urllib.parse.quote(value, safe=""), value.replace("+", "%2B").replace("@", "%40"),
784
+ value.replace("@", "%40"), value.replace("+", "%2B")] # half-encoded spellings a server may echo
785
+ return tuple(dict.fromkeys(out))
786
+
787
+
788
+ def public_text(value: Any, private: tuple[str, ...]) -> Any:
789
+ """Published text never carries an address or a message reference."""
790
+ if isinstance(value, str):
791
+ return _MESSAGE_REF_RE.sub("a message", _scrub(value, private))
792
+ if isinstance(value, list):
793
+ return [public_text(v, private) for v in value]
794
+ if isinstance(value, dict):
795
+ return {k: public_text(v, private) for k, v in value.items()}
796
+ return value
797
+
798
+
799
+ def public_row(row: dict[str, Any], private: tuple[str, ...] = ()) -> dict[str, Any]:
800
+ """The evidence copy of a row: the mailbox alias and the message id stay
801
+ in the runtime folder (plan.json, subscriptions.json, the cache), and
802
+ every published string is scrubbed of addresses and message references."""
803
+ return {k: public_text(v, private) for k, v in row.items() if k not in PRIVATE_ROW_FIELDS}
804
+
805
+
806
+ def public_entry_url(url: str) -> str:
807
+ """The entry point without its query and fragment (a confirmation link
808
+ can carry the address)."""
809
+ parts = urllib.parse.urlsplit(url)
810
+ return urllib.parse.urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
811
+
812
+
813
+ def _scrub(value: str, aliases: tuple[str, ...]) -> str:
814
+ """No mailbox address in published text: the alias, the mailbox or any
815
+ other address a sender echoes back becomes `[address]`."""
816
+ out = value
817
+ for alias in aliases:
818
+ if alias:
819
+ out = re.sub(re.escape(alias), "[address]", out, flags=re.IGNORECASE)
820
+ return _ADDRESS_RE.sub("[address]", out)
821
+
822
+
823
+ def recipient_addresses(item: dict[str, Any]) -> set[str]:
824
+ """Every address the listing names as a recipient (To and Delivered-To),
825
+ parsed and normalised, so attribution compares whole addresses."""
826
+ found: set[str] = set()
827
+ for field in ("to", "delivered_to"):
828
+ raw = item.get(field)
829
+ values = raw if isinstance(raw, list) else [raw]
830
+ for value in values:
831
+ if not isinstance(value, str) or not value.strip():
832
+ continue
833
+ for _, address in getaddresses([value]):
834
+ address = address.strip().lower()
835
+ if "@" in address:
836
+ found.add(address)
837
+ return found
838
+
839
+
840
+ def message_row(listing: dict[str, Any], body: dict[str, Any], subscription: dict[str, Any], index: int) -> dict[str, Any]:
841
+ text = str(body.get("text") or "")
842
+ aliases = (subscription["alias"], subscription["alias"].partition("+")[0] + "@" + subscription["alias"].partition("@")[2])
843
+ subject = _scrub(" ".join(str(listing.get("subject") or body.get("subject") or "").split()), aliases)
844
+ sender_name, sender_address = parseaddr(str(listing.get("from") or body.get("from") or ""))
845
+ received = _parse_received(listing) or _parse_received(body)
846
+ subscribed = date.fromisoformat(subscription["subscribed_on"])
847
+ # The inbox is asked from the day before the subscription (server days are
848
+ # UTC, the recorded date is local); a message from that margin is day 0.
849
+ day_offset = max(0, (received.date() - subscribed).days) if received else None
850
+ haystack = f"{subject}\n{text}"
851
+ discounts = [int(a or b) for a, b in _DISCOUNT_RE.findall(haystack) if (a or b)]
852
+ prices = list(dict.fromkeys(m.group(0).strip() for m in _PRICE_RE.finditer(haystack)))[:4]
853
+ deadline = _DEADLINE_RE.search(haystack)
854
+ is_confirmation = bool(_CONFIRM_RE.search(subject)) or (index == 1 and
855
+ body.get("mail_content_metrics", {}).get("confirmation_in_prefix", bool(_CONFIRM_RE.search(text[:600]))))
856
+ is_offer = bool(discounts or prices or _FREE_RE.search(haystack))
857
+ kind = "confirmation" if is_confirmation else "offer" if is_offer else "nudge" if _PROGRESS_RE.search(haystack) else "welcome" if index <= 2 else "content"
858
+ is_welcome = kind == "welcome" or (not is_confirmation and bool(_WELCOME_RE.search(subject)))
859
+ return {
860
+ **funnel_lineage(subscription),
861
+ "type": "lifecycle_message", "domain": subscription["domain"], "alias": subscription["alias"], "uid": str(listing.get("uid")),
862
+ "received_at": received.isoformat().replace("+00:00", "Z") if received else None, "day_offset": day_offset,
863
+ "hour_utc": received.hour if received else None, "sender_domain": registrable_domain(sender_address.partition("@")[2]) if "@" in sender_address else None,
864
+ "subject": subject[:200], "kind": kind, "is_confirmation": is_confirmation, "is_offer": is_offer, "is_welcome": is_welcome,
865
+ "discount_percent": max(discounts) if discounts else None, "prices": prices, "free_mention": bool(_FREE_RE.search(haystack)),
866
+ "deadline": deadline.group(0).lower() if deadline else None,
867
+ "unsubscribe_link": body.get("mail_content_metrics", {}).get("unsubscribe_link", bool(_UNSUB_RE.search(text))),
868
+ "links": body.get("mail_content_metrics", {}).get("links", len(_LINK_RE.findall(text))),
869
+ "word_count": body.get("mail_content_metrics", {}).get("word_count", len(text.split())), "tier": "A",
870
+ }
871
+
872
+
873
+ def sequence_end(subscription: dict[str, Any], harvested_on: date) -> date:
874
+ """The last day a sequence is observed: the harvest cutoff, or the
875
+ unsubscribe date when it came first."""
876
+ if subscription.get("unsubscribed_on"):
877
+ return min(harvested_on, date.fromisoformat(subscription["unsubscribed_on"]))
878
+ return harvested_on
879
+
880
+
881
+ def sequence_row(subscription: dict[str, Any], messages: list[dict[str, Any]], harvested_on: date, truncated: bool = False) -> dict[str, Any]:
882
+ subscribed = date.fromisoformat(subscription["subscribed_on"])
883
+ end = sequence_end(subscription, harvested_on)
884
+ days_covered = max(0, (end - subscribed).days)
885
+ offsets = sorted(m["day_offset"] for m in messages if isinstance(m.get("day_offset"), int))
886
+ offers = [m for m in messages if m["is_offer"]]
887
+ gaps = [b - a for a, b in zip(offsets, offsets[1:])]
888
+ kinds = Counter(m["kind"] for m in messages)
889
+ return {
890
+ **funnel_lineage(subscription),
891
+ "type": "lifecycle_sequence", "domain": subscription["domain"], "alias": subscription["alias"], "kind": subscription["kind"],
892
+ "subscribed_on": subscription["subscribed_on"], "harvested_on": harvested_on.isoformat(), "days_covered": days_covered,
893
+ "observation_status": "full" if days_covered >= HARVEST_AFTER_DAYS else "partial", "target_days": HARVEST_AFTER_DAYS,
894
+ "observed_from": subscribed.isoformat(), "observed_through": end.isoformat(),
895
+ "early_harvest_due_on": (subscribed + timedelta(days=EARLY_HARVEST_AFTER_DAYS)).isoformat(),
896
+ "full_harvest_due_on": (subscribed + timedelta(days=HARVEST_AFTER_DAYS)).isoformat(),
897
+ "coverage_complete": not truncated and subscription.get("confirmed") is not False,
898
+ "listing_succeeded": True,
899
+ "messages": len(messages), "messages_per_week": round(len(messages) * 7 / days_covered, 2) if days_covered >= 7 else None,
900
+ "first_message_day": offsets[0] if offsets else None, "last_message_day": offsets[-1] if offsets else None,
901
+ "first_welcome_day": min((m["day_offset"] for m in messages if m.get("is_welcome") and isinstance(m.get("day_offset"), int)), default=None),
902
+ "median_gap_days": sorted(gaps)[len(gaps) // 2] if gaps else None, "confirmations": kinds.get("confirmation", 0),
903
+ "offers": len(offers), "first_offer_day": min((m["day_offset"] for m in offers if isinstance(m.get("day_offset"), int)), default=None),
904
+ "max_discount_percent": max((m["discount_percent"] for m in messages if m["discount_percent"]), default=None),
905
+ "deadlines": sum(1 for m in messages if m["deadline"]), "nudges": kinds.get("nudge", 0),
906
+ "prices": list(dict.fromkeys(p for m in messages for p in m["prices"]))[:6],
907
+ "sender_domains": list(dict.fromkeys(m["sender_domain"] for m in messages if m["sender_domain"]))[:4],
908
+ "unsubscribe_present": any(m["unsubscribe_link"] for m in messages), "subjects": [m["subject"] for m in messages[:8]],
909
+ "listing_truncated": truncated, "tier": "A" if messages else "C",
910
+ }
911
+
912
+
913
+ _PATTERN_LABELS = {
914
+ "double_opt_in": "A confirmation step before the sequence starts",
915
+ "welcome_same_day": "A welcome message within a day",
916
+ "offer_in_first_week": "A paid offer within the first seven days",
917
+ "discount_offered": "A percentage discount is offered",
918
+ "deadline_used": "A deadline or scarcity line is used",
919
+ "nudges_sent": "Progress or come-back nudges are sent",
920
+ "daily_or_more": "Five or more messages a week",
921
+ "weekly_or_less": "At most one message a week",
922
+ "prices_named": "Prices are named inside the emails",
923
+ }
924
+
925
+
926
+ def lifecycle_patterns(sequences: list[dict[str, Any]]) -> list[dict[str, Any]]:
927
+ active = [s for s in sequences if s["messages"]]
928
+ if not active:
929
+ return []
930
+ hits: dict[str, list[str]] = {k: [] for k in _PATTERN_LABELS}
931
+ for s in active:
932
+ d = s["domain"]
933
+ if s["confirmations"]:
934
+ hits["double_opt_in"].append(d)
935
+ if s.get("first_welcome_day") is not None and s["first_welcome_day"] <= 1:
936
+ hits["welcome_same_day"].append(d)
937
+ if s["first_offer_day"] is not None and s["first_offer_day"] <= 7:
938
+ hits["offer_in_first_week"].append(d)
939
+ if s["max_discount_percent"]:
940
+ hits["discount_offered"].append(d)
941
+ if s["deadlines"]:
942
+ hits["deadline_used"].append(d)
943
+ if s["nudges"]:
944
+ hits["nudges_sent"].append(d)
945
+ if s["days_covered"] >= 7 and (s["messages_per_week"] or 0) >= 5:
946
+ hits["daily_or_more"].append(d)
947
+ elif s["days_covered"] >= 7 and (s["messages_per_week"] or 0) <= 1:
948
+ hits["weekly_or_less"].append(d)
949
+ if s["prices"]:
950
+ hits["prices_named"].append(d)
951
+ rows = []
952
+ for pattern, domains in hits.items():
953
+ if not domains:
954
+ continue
955
+ supporting = [s for s in active if s["domain"] in domains]
956
+ windows = [s for s in supporting if s.get("observed_from") and s.get("observed_through")]
957
+ rows.append({"type": "lifecycle_pattern", "pattern": pattern, "label": _PATTERN_LABELS[pattern], "competitors": sorted(set(domains)),
958
+ "competitors_with": len(set(domains)), "competitors_total": len({s["domain"] for s in active}), "share": round(len(set(domains)) / len({s["domain"] for s in active}), 2),
959
+ "recurring": len(set(domains)) >= 2, "eligible": True,
960
+ "observation_status": "full" if all(s["days_covered"] >= HARVEST_AFTER_DAYS for s in supporting) else "partial",
961
+ "target_days": HARVEST_AFTER_DAYS, "days_covered": min(s["days_covered"] for s in supporting),
962
+ "observed_from": min((s["observed_from"] for s in windows), default=None),
963
+ "observed_through": max((s["observed_through"] for s in windows), default=None),
964
+ "coverage_complete": all(s.get("coverage_complete", True) for s in supporting),
965
+ "tier": "B" if len(set(domains)) >= 2 else "C"})
966
+ rows.sort(key=lambda r: (-r["competitors_with"], r["pattern"]))
967
+ return rows
968
+
969
+
970
+ def harvest(client: OsClient, subscriptions: list[dict[str, Any]], harvested_on: date, coverage: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
971
+ """Every recorded subscription is harvested, unsubscribed ones included:
972
+ their mail is still in the mailbox and the sequence they produced is the
973
+ evidence. Messages after the harvest day or after the unsubscribe date are
974
+ left out."""
975
+ messages_out: list[dict[str, Any]] = []
976
+ sequences: list[dict[str, Any]] = []
977
+ for subscription in subscriptions:
978
+ if date.fromisoformat(subscription["subscribed_on"]) > harvested_on:
979
+ coverage.append({"type": "coverage", "domain": subscription["domain"], "source": "inbox", "status": "pending",
980
+ "detail": "subscription had not started at this historical cutoff", "tier": "C"})
981
+ continue
982
+ since = (date.fromisoformat(subscription["subscribed_on"]) - timedelta(days=1)).isoformat()
983
+ try:
984
+ listing, _ = client.inbox(subscription["alias"], since)
985
+ except BudgetExhausted:
986
+ raise
987
+ except LifecycleCaptureError as exc:
988
+ if isinstance(exc, CF.OsUnreachable) or re.search(r"HTTP (?:401|402|403|429)\b", str(exc)):
989
+ raise # the key, the mailbox or the quota is refused: not a fact about this competitor
990
+ CF._coverage_gap(coverage, "inbox", [subscription["domain"]], CF._provider_detail(exc))
991
+ failed_sequence = sequence_row(subscription, [], harvested_on)
992
+ failed_sequence["coverage_complete"] = False
993
+ failed_sequence["listing_succeeded"] = False
994
+ sequences.append(failed_sequence)
995
+ continue
996
+ alias = subscription["alias"].lower()
997
+ end = sequence_end(subscription, harvested_on)
998
+ kept = []
999
+ for item in listing:
1000
+ if alias not in recipient_addresses(item):
1001
+ continue # the server filter is a substring match; keep exact alias hits only
1002
+ received = _parse_received(item)
1003
+ if received is not None and received.date() > end:
1004
+ continue
1005
+ kept.append(item)
1006
+ truncated = len(listing) >= INBOX_PAGE
1007
+ if truncated:
1008
+ CF._coverage_gap(coverage, "inbox", [subscription["domain"]],
1009
+ f"the inbox listing returned the newest {INBOX_PAGE} messages since {since}; earlier messages of this sequence may be missing")
1010
+ rows: list[dict[str, Any]] = []
1011
+ ordered = sorted(kept, key=lambda m: (str(m.get("received_at") or ""), str(m.get("uid"))))
1012
+ for index, item in enumerate(ordered, 1):
1013
+ try:
1014
+ body = client.read_message(str(item["uid"]))
1015
+ except BudgetExhausted:
1016
+ raise
1017
+ except LifecycleCaptureError as exc:
1018
+ if not re.search(r"HTTP (?:404|410)\b", str(exc)):
1019
+ raise # only a message that vanished between the listing and the read is a fact about this sequence
1020
+ status_match = re.search(r"HTTP (\d{3})", str(exc))
1021
+ CF._coverage_gap(coverage, "message", [subscription["domain"]], f"a message of this sequence could not be read: HTTP {status_match.group(1) if status_match else 'error'}")
1022
+ continue
1023
+ rows.append(message_row(item, body, subscription, index))
1024
+ messages_out.extend(rows)
1025
+ sequence = sequence_row(subscription, rows, harvested_on, truncated)
1026
+ if any(g.get("domain") == subscription["domain"] and g.get("status") != "ok" for g in coverage):
1027
+ sequence["coverage_complete"] = False
1028
+ sequences.append(sequence)
1029
+ return messages_out, sequences
1030
+
1031
+
1032
+ # ── rendering ───────────────────────────────────────────────────────────────
1033
+
1034
+ def _cell(value: Any, cap: int = 90) -> str:
1035
+ return CF._cell(value, cap)
1036
+
1037
+
1038
+ def render_section(*, domains: list[str], mailbox_domain: str, subscriptions: list[dict[str, Any]], skipped: list[dict[str, str]],
1039
+ sequences: list[dict[str, Any]], messages: list[dict[str, Any]], patterns: list[dict[str, Any]], coverage: list[dict[str, Any]],
1040
+ phase: str, harvested_on: str | None, totals: dict[str, Any]) -> str:
1041
+ lines: list[str] = []
1042
+ add = lines.append
1043
+ if phase == "started":
1044
+ add(f"Research subscriptions started at {len({s['domain'] for s in subscriptions})} of {len(domains)} competitor(s) with separate plus-addressed aliases on the claw's `@{mailbox_domain}` mailbox; "
1045
+ f"the early harvest reads the inbox after {EARLY_HARVEST_AFTER_DAYS} days and the full window after {HARVEST_AFTER_DAYS} days from each actual sign-up.")
1046
+ else:
1047
+ add(f"Competitor email sequences harvested on {harvested_on}: {len(messages)} message(s) from {sum(1 for s in sequences if s['messages'])} of {len(subscriptions)} subscription(s), "
1048
+ f"attributed by the exact subscription alias and, where present, its funnel and campaign references.")
1049
+ fully_observed = sequences and len(sequences) == len(subscriptions) and all(s["observation_status"] == "full" for s in sequences)
1050
+ pending_signups = any(g.get("source") == "subscription" and g.get("status") == "pending" for g in coverage)
1051
+ add("Observation: " + ("full" if fully_observed and not pending_signups else "partial") +
1052
+ "; conclusions apply only to the stated observation windows. Full duration does not imply complete inbox coverage.")
1053
+ add("")
1054
+ add("### Subscriptions")
1055
+ add("")
1056
+ add("| Competitor | Kind | Entry point | Subscribed on | Confirmed | Status | Notes |")
1057
+ add("| --- | --- | --- | --- | --- | --- | --- |")
1058
+ for s in subscriptions:
1059
+ status = f"unsubscribed {s['unsubscribed_on']}" if s.get("unsubscribed_on") else "active"
1060
+ confirmed = "yes" if s.get("confirmed") else "no" if s.get("confirmed") is False else "not observed"
1061
+ add(f"| {s['domain']} | {s['kind'].replace('_', ' ')} | {_cell(s['entry_url'].replace('https://', ''), 60)} | {s['subscribed_on']} | {confirmed} | {status} | {_cell(s.get('notes') or '—', 80)} |")
1062
+ for s in skipped:
1063
+ add(f"| {s['domain']} | — | — | — | — | skipped | {_cell(s['reason'], 80)} |")
1064
+ add("")
1065
+ if phase == "harvested":
1066
+ add("### Sequences by competitor")
1067
+ add("")
1068
+ add("| Competitor | Observation | Window | Coverage | Days covered | Messages | Per week | First message (day) | First offer (day) | Max discount | Deadlines | Nudges | Prices named |")
1069
+ add("| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |")
1070
+ for s in sequences:
1071
+ add(f"| {s['domain']} | {s['observation_status']} | {s['observed_from']}..{s['observed_through']} | {'complete' if s['coverage_complete'] else 'gaps'} | {s['days_covered']} | {s['messages']} | {s['messages_per_week'] if s['messages_per_week'] is not None else '—'} | "
1072
+ f"{s['first_message_day'] if s['first_message_day'] is not None else '—'} | {s['first_offer_day'] if s['first_offer_day'] is not None else '—'} | "
1073
+ f"{str(s['max_discount_percent']) + '%' if s['max_discount_percent'] else '—'} | {s['deadlines']} | {s['nudges']} | {_cell(', '.join(s['prices']) or '—', 60)} |")
1074
+ add("")
1075
+ for s in sequences:
1076
+ rows = [m for m in messages if m["domain"] == s["domain"] and m.get("funnel_id") == s.get("funnel_id")]
1077
+ if not rows:
1078
+ continue
1079
+ add(f"#### {s['domain']}" + (f" — {s['funnel_id']}" if s.get("funnel_id") else ""))
1080
+ add("")
1081
+ add("| Day | Kind | Subject | Offer or deadline |")
1082
+ add("| ---: | --- | --- | --- |")
1083
+ for m in rows:
1084
+ offer = ", ".join(x for x in ([f"{m['discount_percent']}% off"] if m["discount_percent"] else []) + m["prices"][:2] + ([m["deadline"]] if m["deadline"] else []))
1085
+ add(f"| {m['day_offset'] if m['day_offset'] is not None else '—'} | {m['kind']} | {_cell(m['subject'] or '(no subject)', 90)} | {_cell(offer or '—', 60)} |")
1086
+ add("")
1087
+ add("### Recurring lifecycle patterns")
1088
+ add("")
1089
+ if patterns:
1090
+ add("| Pattern | Competitors | Observation | Share |")
1091
+ add("| --- | --- | --- | ---: |")
1092
+ for r in patterns:
1093
+ add(f"| {r['label']} | {', '.join(r['competitors'])} ({r['competitors_with']} of {r['competitors_total']}) | {r['observation_status']}, {r['days_covered']} days | {int(r['share'] * 100)}% |")
1094
+ else:
1095
+ add("No message has arrived yet; patterns need at least one harvested sequence.")
1096
+ add("")
1097
+ add("### Method and policy")
1098
+ add("")
1099
+ add("- One neutral plus-addressed alias per competitor per capture, reserved uniquely for the mailbox; every message is attributed by an exact To or Delivered-To address match; "
1100
+ "stable per-funnel aliases imported from section 05 retain their attribution; the inbox is read through OS by alias and start date, each message body once. Bodies stay in the runtime cache; the section carries subjects, "
1101
+ f"and extracted facts. Reads in this cutoff: {totals.get('inbox', 0)} inbox listing(s), {totals.get('message', 0)} message(s).")
1102
+ add("- Research sign-ups only: newsletters, free accounts, trials and waitlists that need no payment; no card details, no CAPTCHA bypass, no real "
1103
+ "third-party identities; no required phone. A human with verified access to the planned aliases and mailbox signs up, confirms and unsubscribes; never engage sales representatives. Replicate observed offer structure; cadence needs at least a week of observation. Never copy competitor wording or claims.")
1104
+ gaps = [g for g in coverage if g.get("status") != "ok"]
1105
+ if gaps:
1106
+ add("- Not read in this run: " + "; ".join(f"{g['domain']} ({g['detail']})" for g in gaps[:10]) + ".")
1107
+ add("")
1108
+ add("### Sources")
1109
+ add("")
1110
+ add("- The claw's mailbox through OS `GET /mail/inbox` (filters `to` and `since`) and `GET /mail/message/{uid}`; subscriptions recorded by the executing agent.")
1111
+ return "\n".join(lines).strip() + "\n"
1112
+
1113
+
1114
+ # ── orchestration ───────────────────────────────────────────────────────────
1115
+
1116
+ def _arg(args: Any, name: str, default: Any) -> Any:
1117
+ value = getattr(args, name, None)
1118
+ return default if value is None else value
1119
+
1120
+
1121
+ def _write_atomic(path: Path, content: str) -> None:
1122
+ path.parent.mkdir(parents=True, exist_ok=True)
1123
+ tmp = path.with_name(path.name + ".tmp")
1124
+ tmp.write_text(content, encoding="utf-8")
1125
+ os.replace(tmp, path)
1126
+
1127
+
1128
+ def resolve_harvest_day(runtime: Path, override: str, refresh: bool, not_before: date | None = None) -> date:
1129
+ """A harvest is a day's snapshot of the inbox: reruns on the same cutoff
1130
+ replay the cache; `--refresh` reads the inbox again once the current
1131
+ harvest completed."""
1132
+ marker = runtime / "harvest-as-of.json"
1133
+ try:
1134
+ state = json.loads(marker.read_text(encoding="utf-8"))
1135
+ current = date.fromisoformat(state["as_of"])
1136
+ completed = bool(state.get("completed"))
1137
+ except (OSError, ValueError, KeyError, json.JSONDecodeError):
1138
+ current, completed = None, False
1139
+ if override:
1140
+ if not _DATE_RE.match(override):
1141
+ raise LifecycleCaptureError("--as-of must be YYYY-MM-DD")
1142
+ chosen = date.fromisoformat(override)
1143
+ if chosen > date.today():
1144
+ raise LifecycleCaptureError("--as-of cannot be in the future")
1145
+ if not_before is not None and chosen < not_before:
1146
+ raise LifecycleCaptureError(f"--as-of cannot be before the first subscription day ({not_before.isoformat()})")
1147
+ elif current is None or (completed and (refresh or current < date.today())):
1148
+ # A harvest is a day's snapshot: a completed one is replayed only on
1149
+ # its own day; the next day's run reads the inbox again.
1150
+ chosen = date.today()
1151
+ else:
1152
+ chosen = current
1153
+ if chosen != current:
1154
+ completed = False
1155
+ _write_atomic(marker, json.dumps({"as_of": chosen.isoformat(), "completed": completed}))
1156
+ return chosen
1157
+
1158
+
1159
+ def mark_harvest_completed(runtime: Path, as_of: date) -> None:
1160
+ _write_atomic(runtime / "harvest-as-of.json", json.dumps({"as_of": as_of.isoformat(), "completed": True}))
1161
+
1162
+
1163
+ def _plan(mailbox: str, domains: list[str], aliases: dict[str, str], points: dict[str, list[str]], subscriptions_path: Path,
1164
+ *, capture_id: str, reviewed: dict[str, dict[str, Any]], previous: dict[str, Any], product_domain: str) -> dict[str, Any]:
1165
+ old = {c["domain"]: c for c in previous.get("competitors", []) if isinstance(c, dict) and "domain" in c}
1166
+ competitors = []
1167
+ for d in domains:
1168
+ entry = reviewed.get(d) or ({k: old[d].get(k) for k in ("selected", "fallback", "skip_reason", "discovery_status", "legacy")} if d in old else {})
1169
+ urls = [v["entry_url"] for v in (entry.get("selected"), entry.get("fallback")) if v]
1170
+ competitors.append({"domain": d, "alias": aliases[d], "entry_points": urls or points.get(d) or [f"https://{d}/"],
1171
+ "discovery_status": "pending", **entry})
1172
+ return {
1173
+ "schema_version": SCHEMA_VERSION, "section_id": SECTION_ID, "mailbox": mailbox, "subscriptions_path": subscriptions_path.as_posix(),
1174
+ "capture_id": capture_id, "product_domain": product_domain, "kinds": list(SUBSCRIPTION_KINDS),
1175
+ "early_harvest_after_days": EARLY_HARVEST_AFTER_DAYS, "harvest_after_days": HARVEST_AFTER_DAYS,
1176
+ "competitors": competitors,
1177
+ "rules": [
1178
+ "Assign signups to any authorized human with verified access to the planned aliases and mailbox, regardless of role. Use the planned alias only at the reviewed selected entry or fallback; no payment is allowed. If nobody has that access, leave actions pending.",
1179
+ "Never enter a required phone, card or payment details, never bypass a CAPTCHA or bot check, never use a real third-party identity; try a verified fallback or record a skip reason.",
1180
+ "The assigned human confirms double opt-in from the inbox and reports the result; the agent records confirmed: true. Never reply to sales representatives.",
1181
+ f"Record one entry per competitor in {subscriptions_path.name}: domain, alias, subscribed_on, entry_url, kind, confirmed, notes; competitors you could not subscribe go to `skipped` with a reason.",
1182
+ "After the full harvest the assigned human unsubscribes through the link in the emails and reports the actual date; record unsubscribed_on.",
1183
+ ],
1184
+ }
1185
+
1186
+
1187
+ def subscription_scope(item: dict[str, Any]) -> str:
1188
+ return item["domain"] + (":funnel:" + item["funnel_id"] if item.get("funnel_id") else "")
1189
+
1190
+
1191
+ def write_operator_actions(runtime: Path, plan: dict[str, Any], subscriptions: list[dict[str, Any]], skipped: list[dict[str, Any]], harvested_on: str | None,
1192
+ harvested_domains: list[str] | None = None) -> None:
1193
+ path = runtime / OPERATOR_ACTIONS_FILENAME
1194
+ previous = START.read_object(path)
1195
+ actions, harvests = [], []
1196
+ for competitor in plan["competitors"]:
1197
+ domain = competitor["domain"]
1198
+ observed = [s for s in subscriptions if s["domain"] == domain]
1199
+ for subscription in observed or [None]:
1200
+ scope = subscription_scope(subscription) if subscription else domain
1201
+ old = {**previous}
1202
+ for key in ("actions", "harvests"):
1203
+ old[key] = [{**item, "id": domain + item["id"][len(scope):]}
1204
+ for item in previous.get(key, []) if item.get("id", "").startswith(scope + ":")]
1205
+ generated = START.operator_actions({**plan, "competitors": [competitor]}, [subscription] if subscription else [], skipped,
1206
+ harvested_on, old, [domain] if scope in (harvested_domains or []) else [])
1207
+ for key, output in (("actions", actions), ("harvests", harvests)):
1208
+ for item in generated[key]:
1209
+ item["id"] = scope + item["id"][len(domain):]
1210
+ if item.get("blocked_by"):
1211
+ item["blocked_by"] = scope + item["blocked_by"][len(domain):]
1212
+ if subscription:
1213
+ item.update(funnel_lineage(subscription))
1214
+ output.append(item)
1215
+ _write_atomic(path, json.dumps({"schema_version": 1, "section_id": SECTION_ID, "capture_id": plan["capture_id"],
1216
+ "product_domain": plan.get("product_domain", ""), "actions": actions, "harvests": harvests}, indent=1))
1217
+
1218
+
1219
+ PHASE_FILENAME = "phase.json"
1220
+
1221
+
1222
+ def _previous_phase(runtime: Path) -> str | None:
1223
+ """The phase of the last completed publication (a durable marker: the
1224
+ receipt itself is reset to pending at the start of every attempt)."""
1225
+ try:
1226
+ state = json.loads((runtime / PHASE_FILENAME).read_text(encoding="utf-8"))
1227
+ except (OSError, json.JSONDecodeError):
1228
+ state = {}
1229
+ if state.get("phase") in ("started", "harvested"):
1230
+ return str(state["phase"])
1231
+ if (runtime / "harvest-as-of.json").is_file():
1232
+ return "harvested"
1233
+ return None
1234
+
1235
+
1236
+ def _pending(runtime: Path, reason: str, *, extra: dict[str, Any], client: OsClient) -> dict[str, Any]:
1237
+ state = {"schema_version": SCHEMA_VERSION, "section_id": SECTION_ID, "status": "pending", "error": None, "reason": reason, **extra,
1238
+ "research_context": {"path": (RUNTIME_REL / "input-context.json").as_posix(),
1239
+ "sha256": hashlib.sha256((runtime / "input-context.json").read_bytes()).hexdigest()},
1240
+ "attempt_inbox_calls": client.inbox_calls, "attempt_message_reads": client.message_reads, "attempt_cache_hits": client.cache_hits}
1241
+ _write_atomic(runtime / "request.json", json.dumps(state, indent=1))
1242
+ return state
1243
+
1244
+
1245
+ def _run(workspace: Path, args: Any) -> dict[str, Any]:
1246
+ api_key = (os.environ.get("OS_API_KEY") or "").strip()
1247
+ if not api_key:
1248
+ raise LifecycleCaptureError("OS_API_KEY is required")
1249
+ os_url = (os.environ.get("OS_URL") or DEFAULT_OS_URL).strip() or DEFAULT_OS_URL
1250
+ runtime = workspace / RUNTIME_REL
1251
+ runtime.mkdir(parents=True, exist_ok=True)
1252
+ if bool(_arg(args, "funnel_mail", False)):
1253
+ client = OsClient(os_url=os_url, api_key=api_key, cache_dir=runtime / "cache" / "start", shared_dir=runtime / "cache" / "shared",
1254
+ secret_dir=workspace / REPORT_REL.parent / ".local" / SECTION_ID / "mail-cache",
1255
+ max_inbox_calls=int(_arg(args, "max_inbox_calls", MAX_INBOX_CALLS_PER_ATTEMPT)),
1256
+ max_message_reads=int(_arg(args, "max_message_reads", MAX_MESSAGE_READS_PER_ATTEMPT)), key_scope=f"{SECTION_ID}:")
1257
+ mailbox = client.mail_account()
1258
+ registry = identity_registry(workspace, mailbox, {})
1259
+ handoff = load_funnel_handoff(workspace, mailbox, registry)
1260
+ result = capture_funnel_events(client, workspace, handoff, refresh=bool(_arg(args, "refresh", False)))
1261
+ return {"status": "pending" if result["pending"] or any(not gap.get("terminal") for gap in result["coverage"]) else "completed", "phase": "funnel-mail",
1262
+ "events_path": (RUNTIME_REL / FUNNEL_EVENTS_FILENAME).as_posix(), "events": len(result["events"]), "pending": result["pending"],
1263
+ "coverage": result["coverage"], "subscriptions": len(handoff["subscriptions"])}
1264
+ product_domain = registrable_domain(str(_arg(args, "product_domain", "") or "")) if _arg(args, "product_domain", "") else ""
1265
+ max_competitors = max(1, int(_arg(args, "max_competitors", MAX_COMPETITORS)))
1266
+ explicit = [d for d in CF.parse_domains(str(_arg(args, "domains", "") or ""), CF.MAX_EXPLICIT_DOMAINS) if d != product_domain]
1267
+ evidence = _load_script("research-evidence.py", "pmr_research_evidence_for_lifecycle_capture")
1268
+ tiered = evidence.competitor_set_inputs(workspace, consumer=SECTION_ID, cap=MAX_COMPETITORS)
1269
+ rejected = evidence.competitor_set_excluded_domains(workspace) - set(explicit)
1270
+ discovered = evidence.keyword_inputs(workspace, consumer=SECTION_ID, row_type="competitor_domain", field="domain", cap=MAX_COMPETITORS, exclude=rejected)
1271
+ domains: list[str] = []
1272
+ domain_sources: dict[str, str] = {}
1273
+ for source, candidates in (("explicit", explicit), ("competitor_set", tiered), ("keyword_economics", discovered)):
1274
+ for domain in candidates:
1275
+ domain = registrable_domain(domain)
1276
+ if len(domains) < max_competitors and domain and domain != product_domain and domain not in domains:
1277
+ domains.append(domain)
1278
+ domain_sources[domain] = source
1279
+ if not domains:
1280
+ raise LifecycleCaptureError("no competitor domains available: pass --domains, or complete the Competitor Set (or Keyword Economics) section first")
1281
+
1282
+ subscriptions_path = Path(_arg(args, "subscriptions", "") or (runtime / SUBSCRIPTIONS_FILENAME))
1283
+ if not subscriptions_path.is_absolute():
1284
+ subscriptions_path = workspace / subscriptions_path
1285
+ want_harvest = bool(_arg(args, "harvest", False))
1286
+ want_start = bool(_arg(args, "start", False))
1287
+ client = OsClient(os_url=os_url, api_key=api_key, cache_dir=runtime / "cache" / "start", shared_dir=runtime / "cache" / "shared",
1288
+ secret_dir=workspace / REPORT_REL.parent / ".local" / SECTION_ID / "mail-cache",
1289
+ max_inbox_calls=int(_arg(args, "max_inbox_calls", MAX_INBOX_CALLS_PER_ATTEMPT)),
1290
+ max_message_reads=int(_arg(args, "max_message_reads", MAX_MESSAGE_READS_PER_ATTEMPT)), key_scope=f"{SECTION_ID}:")
1291
+ mailbox = client.mail_account()
1292
+ previous_plan = START.read_object(runtime / PLAN_FILENAME)
1293
+ if previous_plan and previous_plan.get("mailbox") != mailbox:
1294
+ raise LifecycleCaptureError("this capture belongs to another mailbox; use a fresh research workspace")
1295
+ if previous_plan.get("product_domain") and previous_plan["product_domain"] != product_domain:
1296
+ raise LifecycleCaptureError("this capture belongs to another product; use a fresh research workspace")
1297
+ marker = START.read_object(runtime / "capture.json")
1298
+ capture_id = str(previous_plan.get("capture_id") or marker.get("capture_id") or uuid.uuid4())
1299
+ try:
1300
+ uuid.UUID(capture_id)
1301
+ except ValueError as exc:
1302
+ raise LifecycleCaptureError("capture_id must be a UUID") from exc
1303
+ # Persist before the network call so a lost reservation response is safely retryable.
1304
+ _write_atomic(runtime / "capture.json", json.dumps({"capture_id": capture_id}))
1305
+ existing_subscriptions = START.read_object(subscriptions_path)
1306
+ active_legacy = {registrable_domain(str(s.get("domain") or "")): str(s.get("alias") or "").strip().lower()
1307
+ for s in existing_subscriptions.get("subscriptions", []) if isinstance(s, dict)}
1308
+ legacy_aliases = {c["domain"]: c["alias"] for c in previous_plan.get("competitors", [])
1309
+ if c.get("domain") in domains and active_legacy.get(c["domain"]) == c.get("alias") == alias_for(mailbox, c["domain"])}
1310
+ for c in previous_plan.get("competitors", []):
1311
+ c["legacy"] = c.get("domain") in legacy_aliases
1312
+ reserve_domains = [d for d in domains if d not in legacy_aliases]
1313
+ aliases = {**legacy_aliases, **(client.reserve_aliases(capture_id, reserve_domains, mailbox) if reserve_domains else {})}
1314
+ if len(set(aliases.values())) != len(aliases):
1315
+ raise LifecycleCaptureError("two competitors map to one alias; name them with distinct registrable domains")
1316
+ registry = identity_registry(workspace, mailbox, aliases)
1317
+ handoff = load_funnel_handoff(workspace, mailbox, registry)
1318
+ points = entry_points(workspace, domains)
1319
+ discovery_path = Path(_arg(args, "entry_points", "") or (runtime / ENTRY_POINTS_FILENAME))
1320
+ if not discovery_path.is_absolute():
1321
+ discovery_path = workspace / discovery_path
1322
+ reviewed = START.discovery(discovery_path, domains, registrable_domain)
1323
+ plan = _plan(mailbox, domains, aliases, points, subscriptions_path, capture_id=capture_id, reviewed=reviewed,
1324
+ previous=previous_plan, product_domain=product_domain)
1325
+ plan["imported_funnel_subscriptions"] = handoff["subscriptions"]
1326
+ imported_domains = {item["domain"] for item in handoff["subscriptions"] if item["domain"] in domains}
1327
+ _write_atomic(runtime / PLAN_FILENAME, json.dumps(plan, indent=1, ensure_ascii=False))
1328
+ signup_plan = {**plan, "competitors": [c for c in plan["competitors"] if c["domain"] not in imported_domains]}
1329
+ signup_list = START.render_signups(signup_plan)
1330
+ for item in handoff["subscriptions"]:
1331
+ if item["domain"] in imported_domains:
1332
+ signup_list += f"\n## {item['domain']} — {item['funnel_id']}\n\nAlready registered through approved section 05; do not register again.\nAddress: `{item['alias']}`\nEntry: {item['entry_url']}\n"
1333
+ _write_atomic(runtime / "lifecycle-signups.md", signup_list)
1334
+ planned = {c["domain"]: c for c in plan["competitors"]}
1335
+ recorded = load_subscriptions(subscriptions_path, domains, aliases, planned)
1336
+ if imported_domains:
1337
+ recorded = recorded or {"subscriptions": [], "skipped": []}
1338
+ recorded["subscriptions"].extend(item for item in handoff["subscriptions"] if item["domain"] in domains)
1339
+ recorded["skipped"] = [item for item in recorded["skipped"] if item["domain"] not in imported_domains]
1340
+ phase_marker = START.read_object(runtime / PHASE_FILENAME)
1341
+ previous_windows = phase_marker.get("subscription_windows") or {}
1342
+ old_scopes = phase_marker.get("harvested_scopes", phase_marker.get("harvested_domains", []))
1343
+ still_observed = [subscription_scope(s) for s in (recorded or {}).get("subscriptions", [])
1344
+ if subscription_scope(s) in old_scopes
1345
+ and previous_windows.get(subscription_scope(s)) == {"alias": s["alias"], "subscribed_on": s["subscribed_on"]}]
1346
+ write_operator_actions(runtime, plan, (recorded or {}).get("subscriptions", []), (recorded or {}).get("skipped", []),
1347
+ phase_marker.get("harvested_on"), still_observed)
1348
+ accounted = {s["domain"] for s in (recorded or {}).get("subscriptions", [])} | {s["domain"] for s in (recorded or {}).get("skipped", [])}
1349
+ missing = [d for d in domains if d not in accounted]
1350
+ skipped_domains = {s["domain"] for s in (recorded or {}).get("skipped", [])}
1351
+ discovery_missing = [d for d in domains if d not in skipped_domains and d not in imported_domains and planned[d].get("discovery_status") != "reviewed" and not planned[d].get("legacy")]
1352
+ if discovery_missing and not (want_harvest and (recorded or {}).get("subscriptions")):
1353
+ return _pending(runtime, "awaiting reviewed entry-point discovery", extra={"capture_id": capture_id,
1354
+ "plan_path": (RUNTIME_REL / PLAN_FILENAME).as_posix(), "subscriptions_path": subscriptions_path.as_posix(),
1355
+ "entry_points_path": discovery_path.as_posix(), "missing_entry_points": discovery_missing,
1356
+ "unsubscribed": missing, "mailbox_domain": mailbox.partition("@")[2]}, client=client)
1357
+ if recorded is None or missing:
1358
+ if want_harvest and not (recorded or {}).get("subscriptions"):
1359
+ raise LifecycleCaptureError("harvest needs at least one recorded subscription first")
1360
+ if not want_harvest:
1361
+ return _pending(runtime, "awaiting research subscriptions for the planned aliases",
1362
+ extra={"plan_path": (RUNTIME_REL / PLAN_FILENAME).as_posix(), "subscriptions_path": subscriptions_path.as_posix(),
1363
+ "capture_id": capture_id, "unsubscribed": missing or domains, "mailbox_domain": mailbox.partition("@")[2]}, client=client)
1364
+ subscriptions, skipped = recorded["subscriptions"], recorded["skipped"]
1365
+ coverage: list[dict[str, Any]] = []
1366
+ for d in missing:
1367
+ coverage.append({"type": "coverage", "domain": d, "source": "subscription", "status": "pending",
1368
+ "detail": "entry-point discovery is pending; no observation window has started" if d in discovery_missing
1369
+ else "human sign-up is pending; no observation window has started", "tier": "C"})
1370
+ # The first run with every competitor recorded publishes the subscriptions
1371
+ # (phase started); later runs harvest. Flags override that order.
1372
+ previously = _previous_phase(runtime)
1373
+ if want_harvest:
1374
+ phase = "harvested"
1375
+ elif previously == "harvested":
1376
+ phase = "harvested" # A start refresh must never erase an already observed sequence.
1377
+ elif want_start or previously is None:
1378
+ phase = "started"
1379
+ else:
1380
+ phase = "harvested"
1381
+ if phase == "harvested" and not subscriptions:
1382
+ phase = "started"
1383
+ harvested_on: date | None = None
1384
+ messages: list[dict[str, Any]] = []
1385
+ sequences: list[dict[str, Any]] = []
1386
+ patterns: list[dict[str, Any]] = []
1387
+ if phase == "harvested":
1388
+ requested_as_of = str(_arg(args, "as_of", "") or "")
1389
+ if requested_as_of and phase_marker.get("harvested_on") and requested_as_of < phase_marker["harvested_on"]:
1390
+ raise LifecycleCaptureError("--as-of cannot replace a newer published harvest; read its private snapshot instead")
1391
+ harvested_on = resolve_harvest_day(runtime, str(_arg(args, "as_of", "") or ""), bool(_arg(args, "refresh", False)),
1392
+ not_before=min(date.fromisoformat(s["subscribed_on"]) for s in subscriptions))
1393
+ client.cache_dir = runtime / "cache" / "harvest" / harvested_on.isoformat()
1394
+ messages, sequences = harvest(client, subscriptions, harvested_on, coverage)
1395
+ patterns = lifecycle_patterns(sequences)
1396
+ else:
1397
+ coverage.append({"type": "coverage", "domain": "all", "source": "inbox", "status": "pending",
1398
+ "detail": f"subscriptions started; the inbox is harvested after {EARLY_HARVEST_AFTER_DAYS} days for early evidence and {HARVEST_AFTER_DAYS} days for the full window", "tier": "C"})
1399
+ for s in skipped:
1400
+ CF._coverage_gap(coverage, "subscription", [s["domain"]], f"skipped: {s['reason']}")
1401
+ totals = client.ledger_totals()
1402
+ if totals["observations"] == 0:
1403
+ raise LifecycleCaptureError("no OS mail call succeeded; rerun later")
1404
+ if phase == "harvested" and totals["inbox"] == 0:
1405
+ raise LifecycleCaptureError("no inbox listing succeeded in this harvest; rerun later")
1406
+
1407
+ private = private_strings([*aliases.values(), *(item["alias"] for item in subscriptions)], mailbox)
1408
+ subscription_rows = [public_row({**funnel_lineage(s), "type": "subscription", "domain": s["domain"], "kind": s["kind"], "entry_url": public_entry_url(s["entry_url"]),
1409
+ "entry_host": registrable_domain(urllib.parse.urlsplit(s["entry_url"]).hostname or ""), "subscribed_on": s["subscribed_on"],
1410
+ "entry_type": s.get("entry_type"), "source_url": public_entry_url(s.get("source_url") or s["entry_url"]),
1411
+ "early_harvest_due_on": (date.fromisoformat(s["subscribed_on"]) + timedelta(days=EARLY_HARVEST_AFTER_DAYS)).isoformat(),
1412
+ "full_harvest_due_on": (date.fromisoformat(s["subscribed_on"]) + timedelta(days=HARVEST_AFTER_DAYS)).isoformat(),
1413
+ "confirmed": s["confirmed"], "unsubscribed_on": s.get("unsubscribed_on"), "status": "unsubscribed" if s.get("unsubscribed_on") else "active",
1414
+ "notes": s.get("notes"), "tier": "A"}, private) for s in subscriptions]
1415
+ sequences = [public_row(r, private) for r in sequences]
1416
+ messages = [public_row(r, private) for r in messages]
1417
+ patterns = [public_row(r, private) for r in patterns]
1418
+ coverage = [public_row(r, private) for r in coverage]
1419
+ skipped = [public_text(s, private) for s in skipped]
1420
+ rows = subscription_rows + sequences + messages + patterns + coverage
1421
+ observation = {
1422
+ "observation_status": "pending" if phase == "started" else "full" if sequences and not missing and len(sequences) == len(subscriptions) and all(s["observation_status"] == "full" for s in sequences) else "partial",
1423
+ "target_days": HARVEST_AFTER_DAYS,
1424
+ "observed_from": min((s["observed_from"] for s in sequences), default=None),
1425
+ "observed_through": harvested_on.isoformat() if harvested_on else None,
1426
+ "days_covered": min((s["days_covered"] for s in sequences), default=0),
1427
+ "coverage_complete": bool(sequences) and not missing and not skipped and len(sequences) == len(subscriptions) and all(s["coverage_complete"] for s in sequences),
1428
+ "early_harvest_due_on": min((s["early_harvest_due_on"] for s in subscription_rows), default=None),
1429
+ "full_harvest_due_on": max((s["full_harvest_due_on"] for s in subscription_rows), default=None),
1430
+ }
1431
+ retrieved_on = totals["first_retrieved_on"] if totals["first_retrieved_on"] == totals["last_retrieved_on"] else f"{totals['first_retrieved_on']}..{totals['last_retrieved_on']}"
1432
+ evidence_doc = {
1433
+ "schema_version": SCHEMA_VERSION, "section_id": SECTION_ID, "generated_by": "run-lifecycle-capture.py", "retrieved_on": retrieved_on,
1434
+ "inputs": {"domains": domains, "domain_sources": domain_sources, "product_domain": product_domain, "mailbox_domain": mailbox.partition("@")[2],
1435
+ "capture_id": capture_id, "phase": phase, "harvested_on": harvested_on.isoformat() if harvested_on else None,
1436
+ "harvest_after_days": HARVEST_AFTER_DAYS, "early_harvest_after_days": EARLY_HARVEST_AFTER_DAYS, **observation},
1437
+ "rows": rows,
1438
+ "totals": {**totals, "subscriptions": len(subscriptions), "skipped": len(skipped), "messages": len(messages), "patterns": len(patterns)},
1439
+ }
1440
+ evidence_json = json.dumps(evidence_doc, indent=1, ensure_ascii=False)
1441
+ _write_atomic(workspace / EVIDENCE_REL, evidence_json)
1442
+ section_markdown = render_section(domains=domains, mailbox_domain=mailbox.partition("@")[2], subscriptions=subscription_rows, skipped=skipped, sequences=sequences,
1443
+ messages=messages, patterns=patterns, coverage=coverage, phase=phase,
1444
+ harvested_on=harvested_on.isoformat() if harvested_on else None, totals=totals)
1445
+ _write_atomic(runtime / "result.md", section_markdown)
1446
+ upsert = _load_script("upsert-markdown-section.py", "pmr_upsert_markdown_section_for_lifecycle_capture")
1447
+ report_path = workspace / REPORT_REL
1448
+ report_path.parent.mkdir(parents=True, exist_ok=True)
1449
+ upsert.upsert_markdown_section(report_path, SECTION_HEADING, section_markdown)
1450
+ persist = _load_script("persist-research-report.py", "pmr_persist_research_report_for_lifecycle_capture")
1451
+ persistence = persist.persist_research_report(workspace, commit_message=f"research: update {SECTION_HEADING.lower()} section",
1452
+ extra_paths=(Path(_artifact_path("egbe-docs/research/market/evidence/lifecycle-capture.json")),))
1453
+ io = _load_script("research-artifact-io.py", "pmr_artifact_io_for_lifecycle_capture")
1454
+ state = {
1455
+ "schema_version": SCHEMA_VERSION, "section_id": SECTION_ID, "status": "completed", "error": None, "phase": phase,
1456
+ "capture_id": capture_id, **observation,
1457
+ "domains": domains, "domain_sources": domain_sources, "product_domain": product_domain, "mailbox_domain": mailbox.partition("@")[2],
1458
+ "harvested_on": harvested_on.isoformat() if harvested_on else None, "subscriptions": len(subscriptions), "skipped": len(skipped),
1459
+ "messages": len(messages), "patterns": len(patterns), "provider_calls": totals["provider_calls"], "observations": totals["observations"],
1460
+ "inbox_calls": totals["inbox"], "message_reads": totals["message"],
1461
+ "attempt_inbox_calls": client.inbox_calls, "attempt_message_reads": client.message_reads, "attempt_cache_hits": client.cache_hits,
1462
+ "subscriptions_sha256": hashlib.sha256(subscriptions_path.read_bytes()).hexdigest() if subscriptions_path.is_file() else None,
1463
+ "funnel_handoff_sha256": hashlib.sha256((workspace / FUNNEL_HANDOFF_REL).read_bytes()).hexdigest() if (workspace / FUNNEL_HANDOFF_REL).is_file() else None,
1464
+ "identity_registry_binding": _load_script("research_integrity.py", "pmr_lifecycle_identity_integrity").identity_binding(
1465
+ registry, lifecycle_domains=domains, funnel_ids=[s["funnel_id"] for s in subscriptions if s.get("funnel_id")]),
1466
+ "research_context": io.context_binding(workspace, SECTION_ID),
1467
+ "publication_mode": persistence.publication_mode, "report_commit_sha": persistence.commit_sha,
1468
+ "evidence_path": EVIDENCE_REL.as_posix(), "evidence_sha256": hashlib.sha256(evidence_json.encode("utf-8")).hexdigest(),
1469
+ "result_path": (RUNTIME_REL / "result.md").as_posix(), "result_sha256": hashlib.sha256(section_markdown.encode("utf-8")).hexdigest(),
1470
+ }
1471
+ _write_atomic(runtime / "request.json", json.dumps(state, indent=1))
1472
+ successful_scopes = [subscription_scope(s) for s in sequences if s["listing_succeeded"]]
1473
+ successful_domains = sorted({s["domain"] for s in sequences if s["listing_succeeded"]})
1474
+ _write_atomic(runtime / PHASE_FILENAME, json.dumps({"phase": phase, "published_on": date.today().isoformat(),
1475
+ "harvested_on": harvested_on.isoformat() if harvested_on else phase_marker.get("harvested_on"),
1476
+ "harvested_domains": successful_domains if harvested_on else sorted({s["domain"] for s in subscriptions if subscription_scope(s) in still_observed}),
1477
+ "harvested_scopes": successful_scopes if harvested_on else still_observed,
1478
+ "subscription_windows": {subscription_scope(s): {"alias": s["alias"], "subscribed_on": s["subscribed_on"]} for s in subscriptions}
1479
+ if harvested_on else previous_windows}))
1480
+ if harvested_on is not None:
1481
+ mark_harvest_completed(runtime, harvested_on)
1482
+ write_operator_actions(runtime, plan, subscriptions, recorded["skipped"], harvested_on.isoformat(), successful_scopes)
1483
+ # Immutable private provenance for handed-over packs, even for multiple revisions on one day.
1484
+ snapshot = runtime / "snapshots" / (harvested_on or date.today()).isoformat() / state["evidence_sha256"]
1485
+ for name, contents in (("evidence.json", evidence_json), ("result.md", section_markdown), ("request.json", json.dumps(state, indent=1))):
1486
+ path = snapshot / name
1487
+ if not path.exists():
1488
+ _write_atomic(path, contents)
1489
+ return state
1490
+
1491
+
1492
+ def run(workspace: Path, args: Any) -> dict[str, Any]:
1493
+ workspace = Path(workspace).resolve()
1494
+ io = _load_script("research-artifact-io.py", "pmr_artifact_io")
1495
+ if bool(_arg(args, "funnel_mail", False)):
1496
+ # A same-day mailbox check is a 05 continuation, not a publication or
1497
+ # a failed/completed receipt for the independent section 02 capture.
1498
+ with io.file_lock(workspace / RUNTIME_REL / ".run.lock", timeout=1):
1499
+ io.bind_context(workspace, SECTION_ID, context_file=getattr(args, "context_file", None))
1500
+ return _run(workspace, args)
1501
+ with io.section_attempt(workspace, SECTION_ID, context_file=getattr(args, "context_file", None)):
1502
+ try:
1503
+ return _run(workspace, args)
1504
+ except ValueError as exc:
1505
+ raise LifecycleCaptureError(str(exc)) from exc
1506
+
1507
+
1508
+ def main() -> int:
1509
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
1510
+ parser.add_argument("--workspace", type=Path, required=True)
1511
+ parser.add_argument("--context-file", type=Path, help="Optional selected research context; existing snapshots are reused on resume.")
1512
+ parser.add_argument("--domains", default="", help="Comma-separated competitor domains; merged ahead of the Competitor Set and Keyword Economics domains.")
1513
+ parser.add_argument("--product-domain", default="", help="The product's own domain; never subscribed to as a competitor.")
1514
+ parser.add_argument("--max-competitors", type=int, default=MAX_COMPETITORS)
1515
+ parser.add_argument("--subscriptions", default="", help=_artifact_path(f"Path to the subscriptions file (default egbe-docs/research/market/work/{SECTION_ID}/{SUBSCRIPTIONS_FILENAME})."))
1516
+ parser.add_argument("--entry-points", default="", help="Reviewed entry discovery JSON (default: the private runtime entry-points.json).")
1517
+ mode = parser.add_mutually_exclusive_group()
1518
+ mode.add_argument("--funnel-mail", action="store_true", help="Read confirmations/results for imported 05 aliases today; writes runtime events only, without completing 02.")
1519
+ mode.add_argument("--start", action="store_true", help="Publish recorded subscriptions before the first harvest; later refreshes preserve the harvested phase.")
1520
+ mode.add_argument("--harvest", action="store_true", help="Read the inbox by alias and publish the sequences (the default once subscriptions are recorded).")
1521
+ parser.add_argument("--as-of", default="", help="Harvest day YYYY-MM-DD (default: today for a new harvest, kept across reruns).")
1522
+ parser.add_argument("--refresh", action="store_true", help="For --funnel-mail, refresh today’s inbox listing; for ordinary harvest, advance a completed daily snapshot.")
1523
+ parser.add_argument("--max-inbox-calls", type=int, default=MAX_INBOX_CALLS_PER_ATTEMPT)
1524
+ parser.add_argument("--max-message-reads", type=int, default=MAX_MESSAGE_READS_PER_ATTEMPT)
1525
+ args = parser.parse_args()
1526
+ try:
1527
+ state = run(args.workspace, args)
1528
+ except (LifecycleCaptureError, ValueError) as exc:
1529
+ print(f"{DISPLAY_NAME}: {exc}", file=sys.stderr)
1530
+ return 1
1531
+ if state.get("phase") == "funnel-mail":
1532
+ print(json.dumps(state, indent=1))
1533
+ return 2 if state["status"] == "pending" else 0
1534
+ if state["status"] == "pending":
1535
+ print(f"{DISPLAY_NAME}: {state['reason']} — read {state['plan_path']} and the lifecycle workflow; "
1536
+ f"the assigned human's results go in {state['subscriptions_path']}. Continue other research while waiting.", file=sys.stderr)
1537
+ return 2
1538
+ print(json.dumps({k: state[k] for k in ("status", "phase", "domains", "subscriptions", "skipped", "messages", "patterns")}, indent=1))
1539
+ return 0
1540
+
1541
+
1542
+ if __name__ == "__main__":
1543
+ raise SystemExit(main())