@iann29/rastro 0.10.9 → 0.11.0

package/src/component/schema.ts

@@ -28,6 +28,13 @@ import {
   reportRollupQueueFields,
   rollupFoldStateFields,
 } from "./rollupStore.js";
+import {
+  firstContactValidator,
+  formChoiceAnswerValidator,
+  formLeadDecisionValidator,
+  formRuleValidator,
+  metaFormQuestionValidator,
+} from "./leadAdsValidators.js";
 
 export default defineSchema({
   sites: defineTable(siteFieldsValidator)
@@ -424,7 +431,8 @@ export default defineSchema({
   leadDailyRollups: defineTable({
     siteId: v.id("sites"),
     dayStart: v.number(),
-    dimension: v.union(v.literal("origin"), v.literal("ad")),
+    // `form` (since 0.11.0): a Meta instant form's leads, keyed by form id.
+    dimension: v.union(v.literal("origin"), v.literal("ad"), v.literal("form")),
     key: v.string(),
     leads: v.number(),
     qualified: v.number(),
@@ -765,4 +773,182 @@ export default defineSchema({
     Lead: v.number(),
     Schedule: v.number(),
   }).index("by_siteId_and_dayStart_and_shard", ["siteId", "dayStart", "shard"]),
+
+  // Lead Ads (metaLeads.ts, since 0.11.0). No table here ever holds a lead's
+  // name, phone, e-mail or free answer: those go from Meta to the host's
+  // handler in memory.
+
+  // A Facebook Page an owner's connection lists, with the Page's own token
+  // sealed with RASTRO_SECRETS_KEY (one row per connection and Page: two
+  // owners may list the same Page). No query returns the cipher.
+  metaPages: defineTable({
+    ownerKey: v.string(),
+    connectionId: v.id("metaConnections"),
+    pageId: v.string(),
+    name: v.string(),
+    tokenCipher: v.string(),
+    tokenIv: v.string(),
+    /** The Page tasks Meta granted the login: `ADVERTISE`, `MANAGE`… */
+    tasks: v.array(v.string()),
+    /** The login was granted `leads_retrieval` for this Page. */
+    leadsAllowed: v.boolean(),
+    /** False once a new listing left the Page out; kept while linked. */
+    listed: v.boolean(),
+    /** When Meta refused the Page's token; a new listing clears it. */
+    invalidSince: v.optional(v.number()),
+    createdAt: v.number(),
+    updatedAt: v.number(),
+  })
+    .index("by_connectionId_and_pageId", ["connectionId", "pageId"])
+    .index("by_pageId", ["pageId"]),
+
+  // A Page feeding a site's leads: subscribed to Meta's `leadgen` webhook and
+  // reconciled by the chain in metaLeads.ts. One Page feeds one site.
+  sitePages: defineTable({
+    siteId: v.id("sites"),
+    pageId: v.string(),
+    name: v.string(),
+    /** The owner whose connection (and Page token) the link reads with. */
+    ownerKey: v.string(),
+    connectionId: v.id("metaConnections"),
+    /** When Meta confirmed the `leadgen` subscription. */
+    subscribedAt: v.optional(v.number()),
+    /**
+     * When the reconciliation last made sure the app is still subscribed to
+     * the Page: the subscription is the Page's and the app's, shared by every
+     * host, so another host unlinking the Page may have taken it off.
+     */
+    subscriptionCheckedAt: v.optional(v.number()),
+    /** Leads created from here on are the link's (ms). */
+    reconcileFrom: v.number(),
+    nextReconcileAt: v.number(),
+    /** The run holding the link, as in the spend sync. */
+    runId: v.optional(v.string()),
+    lastReconcileAt: v.optional(v.number()),
+    lastLeadAt: v.optional(v.number()),
+    failures: v.optional(v.number()),
+    lastError: v.optional(v.string()),
+    createdAt: v.number(),
+    updatedAt: v.number(),
+  })
+    .index("by_siteId", ["siteId"])
+    .index("by_pageId", ["pageId"])
+    .index("by_connectionId", ["connectionId"])
+    .index("by_nextReconcileAt", ["nextReconcileAt"]),
+
+  // A Page's instant forms as Meta lists them.
+  metaForms: defineTable({
+    pageId: v.string(),
+    formId: v.string(),
+    name: v.string(),
+    status: v.string(),
+    questions: v.array(metaFormQuestionValidator),
+    /** The end screen's WhatsApp button; absent when Meta does not say. */
+    whatsappButton: v.optional(v.boolean()),
+    createdTime: v.optional(v.number()),
+    syncedAt: v.number(),
+    /**
+     * How far the reconciliation read this form's leads (Meta's seconds),
+     * for the link it read them for: a new link starts from its own time.
+     */
+    cursor: v.optional(
+      v.object({ linkId: v.id("sitePages"), sec: v.number() }),
+    ),
+    /** The last lead that came in from the form, for the site it fed. */
+    lastLead: v.optional(v.object({ siteId: v.id("sites"), at: v.number() })),
+  })
+    .index("by_pageId", ["pageId"])
+    .index("by_pageId_and_status", ["pageId", "status"])
+    .index("by_formId", ["formId"]),
+
+  // Where a site's form sends its leads. Editing affects the next leads only.
+  formRules: defineTable({
+    siteId: v.id("sites"),
+    formId: v.string(),
+    entryColumn: v.optional(v.string()),
+    rules: v.array(formRuleValidator),
+    /** Each rule's identity, for its monthly count (see leadFormRules.ts). */
+    signatures: v.array(v.string()),
+    firstContact: firstContactValidator,
+    whatsappButtonConfirmed: v.optional(v.boolean()),
+    /** The owner that saved them, when the host named one. */
+    setBy: v.optional(v.string()),
+    updatedAt: v.number(),
+  }).index("by_siteId_and_formId", ["siteId", "formId"]),
+
+  // One per ingested form lead: its origin, the choice answers and what the
+  // rules decided. `leadgenId` makes ingestion idempotent.
+  formLeads: defineTable({
+    siteId: v.id("sites"),
+    pageId: v.string(),
+    formId: v.string(),
+    leadgenId: v.string(),
+    leadId: v.id("leads"),
+    receivedAt: v.number(),
+    adId: v.optional(v.string()),
+    adsetId: v.optional(v.string()),
+    campaignId: v.optional(v.string()),
+    platform: v.optional(v.string()),
+    isOrganic: v.boolean(),
+    answers: v.array(formChoiceAnswerValidator),
+    decision: formLeadDecisionValidator,
+    rulePath: v.array(v.number()),
+    ruleIndex: v.optional(v.number()),
+    ruleSignature: v.optional(v.string()),
+    /** A discarded lead sent back to the funnel, and the reason it had. */
+    returned: v.optional(v.object({ at: v.number(), reason: v.string() })),
+    createdAt: v.number(),
+  })
+    .index("by_leadgenId", ["leadgenId"])
+    .index("by_leadId", ["leadId"])
+    .index("by_siteId_and_receivedAt", ["siteId", "receivedAt"])
+    .index("by_siteId_and_formId_and_receivedAt", [
+      "siteId",
+      "formId",
+      "receivedAt",
+    ])
+    .index("by_siteId_and_formId_and_ruleSignature_and_receivedAt", [
+      "siteId",
+      "formId",
+      "ruleSignature",
+      "receivedAt",
+    ]),
+
+  // A form lead the host removed (`leads.remove`): its leadgen id only, so
+  // no later read of the same lead (the webhook retried, the reconciliation
+  // reading the cursor's second, a new link's backfill) brings it back.
+  formLeadTombstones: defineTable({
+    leadgenId: v.string(),
+    removedAt: v.number(),
+  }).index("by_leadgenId", ["leadgenId"]),
+
+  // A form lead whose commit failed (the host's handler threw): its ids, how
+  // many times and the error's class or code, never its data. After
+  // `MAX_LEAD_ATTEMPTS` while later leads of the form come in, the
+  // reconciliation skips it (`skippedAt`) so they are not held behind it.
+  // Deleted when the lead comes in.
+  formLeadFailures: defineTable({
+    leadgenId: v.string(),
+    pageId: v.string(),
+    formId: v.string(),
+    attempts: v.number(),
+    lastCode: v.string(),
+    lastAt: v.number(),
+    skippedAt: v.optional(v.number()),
+  }).index("by_leadgenId", ["leadgenId"]),
+
+  // The host's form lead handler: a function handle the component calls with
+  // each lead (see `setFormLeadHandler`). One per deployment.
+  metaFormLeadHandler: defineTable({
+    key: v.literal("handler"),
+    handle: v.string(),
+    updatedAt: v.number(),
+  }).index("by_key", ["key"]),
+
+  // The single scheduler chain that reconciles every linked Page.
+  metaLeadsLoop: defineTable({
+    key: v.literal("loop"),
+    scheduledFor: v.number(),
+    jobId: v.id("_scheduled_functions"),
+  }).index("by_key", ["key"]),
 });

package/src/component/validators.ts

@@ -1,4 +1,5 @@
 import { v, type Infer } from "convex/values";
+import { formLeadDetailValidator } from "./leadAdsValidators.js";
 
 export const personAttributeValueValidator = v.union(
   v.string(),
@@ -384,7 +385,14 @@ export const trackedLinkFieldsValidator = v.object({
  * behind it. Today the channel is WhatsApp; the stage follows the host's
  * pipeline as `lead` → `qualified` → `won`, or `lost`.
  */
-export const leadChannelValidator = v.literal("whatsapp");
+/**
+ * Where the lead talks to the host: a WhatsApp conversation, or (since
+ * 0.11.0) a Meta instant form the lead filled in (`lead_form` origin).
+ */
+export const leadChannelValidator = v.union(
+  v.literal("whatsapp"),
+  v.literal("form"),
+);
 export const leadStageValidator = v.union(
   v.literal("lead"),
   v.literal("qualified"),
@@ -440,6 +448,8 @@ export const leadFieldsValidator = v.object({
   /** The host's board column the lead is in now (see `setPipeline`). */
   column: v.optional(v.string()),
   columnAt: v.optional(v.number()),
+  /** Since 0.11.0: the Meta instant form the lead filled in (metaLeads.ts). */
+  form: v.optional(v.object({ formId: v.string(), pageId: v.string() })),
   updatedAt: v.number(),
 });
 
@@ -847,6 +857,12 @@ export const leadJourneyValidator = v.union(
       }),
     ),
     visitorId: v.union(v.string(), v.null()),
+    /**
+     * Since 0.11.0, on a Meta instant form's lead: its choice answers, where
+     * it came from and the path the form's rules took. Null when the lead
+     * has no ingestion record left.
+     */
+    form: v.optional(v.union(formLeadDetailValidator, v.null())),
     events: v.array(
       v.object({
         type: leadEventTypeValidator,

package/src/tracker/generated.ts

@@ -1,5 +1,5 @@
 // Generated by scripts/build-tracker.mjs. Do not edit.
-export const RASTRO_VERSION = "0.10.9";
+export const RASTRO_VERSION = "0.11.0";
 export const TRACKER_SOURCE = "(()=>{const t=document.currentScript,e=t?.dataset.site,i=t?.dataset.endpoint||t?.src&&new URL(\"events\",t.src).href;if(!e||!i)return;const n=()=>crypto.randomUUID?.()||Math.random()+\"\",r=new URLSearchParams(location.search),s=r.get(\"ref\")?.slice(0,64);let o;for(const t of[\"source\",\"medium\",\"campaign\",\"term\",\"content\"]){const e=r.get(\"utm_\"+t)?.slice(0,64);e&&((o??={})[\"utm_\"+t]=e)}const a=\"gclid gbraid wbraid msclkid ttclid twclid li_fat_id ScCid rdt_cid dclid epik srsltid mc_cid mc_eid _hsenc igsh igshid fbclid\".split(\" \").filter(t=>r.has(t)).slice(0,8);let c,d,l,h,f=t.dataset.visitor||\"\",p=Boolean(f);try{f||=localStorage._rv||=n()}catch{}try{d=sessionStorage,l=d._r,c=d._ri==f&&d._rt,h=s||d._a,h&&(d._a=h),o||=d._u&&JSON.parse(d._u),o&&(d._u=JSON.stringify(o))}catch{h=s}let v,g=0;const u=[];let m=document.referrer.split(\"/\",3).join(\"/\")||void 0;const _=t=>{const n=u.splice(0,50);if(!n.length)return;const r=()=>new Blob([JSON.stringify({siteId:e,events:n,sentAt:Date.now()})]);let s=r();for(;s.size>64e3&&n.length>1;)u.unshift(n.pop()),s=r();t&&navigator.sendBeacon(i,s)||fetch(i,{method:\"POST\",body:s,keepalive:1}).catch(()=>{}),u.length&&_(t)},y=(t,e,i)=>{const r=Date.now(),s=r-c;c=r,s<18e5||(l=n(),\"pageview\"!=t&&(S=\"\",w()));try{d._r=l,d._ri=f,d._rt=r}catch{}u.push({eventId:`${l}.${r.toString(36)}.${g}`,sessionId:l,visitorId:f||l,type:t,name:e,path:location.pathname.slice(0,256),referrer:m,timestamp:r,sequence:g++,affiliateSlug:h,...i}),clearTimeout(v),v=setTimeout(_,1200)};let S;const w=()=>{const t=location.pathname;t!=S&&(S=t,y(\"pageview\",void 0,{properties:o,clid:a.length?a:void 0}))};window.rastro=(e,i,r)=>{if(\"context\"==e)return{sessionId:l,visitorId:f||l,identified:p};if(\"identify\"==e||\"reset\"==e){if(\"identify\"==e&&(!i||i.length>128||!/^[A-Za-z0-9][A-Za-z0-9_.:-]*$/.test(i)))return;if(\"identify\"==e&&i==f)return;_();const r=\"reset\"==e||p;if(p=\"identify\"==e,f=\"identify\"==e?i:n(),t.dataset.visitor=\"identify\"==e?f:\"\",r){h=void 0,o=void 0,m=void 0,a.length=0;try{\"reset\"==e&&(localStorage._rv=f),d._a=\"\",d._u=\"\"}catch{}}c=0,g=0,S=\"\",w()}\"event\"==e&&y(\"custom\",`${i||\"event\"}`.slice(0,80),r&&JSON.stringify(r).length>2e4?void 0:{properties:r})},document.addEventListener(\"click\",t=>{const e=t.target?.closest(\"a,button,[data-rastro-event]\");if(!e)return;const i=e.dataset.rastroEvent,n=e.href,r=n&&new URL(n),s=/^https?:$/.test(r?.protocol),o=s&&r.origin!=location.origin;y(i?\"custom\":o?\"outbound\":\"click\",i,{target:(e.dataset.rastroLabel||e.innerText||e.tagName).slice(0,64),href:s?((o?r.origin:\"\")+r.pathname).slice(0,256):void 0})},1);for(const t of[\"pushState\",\"replaceState\"]){const e=history[t];history[t]=(...t)=>{e.apply(history,t),w()}}addEventListener(\"popstate\",w);let b=0;const I=()=>{y(\"heartbeat\"),_()},L=()=>{b&&(clearInterval(b),b=0,y(\"leave\"),_(1))},E=()=>{document.hidden?L():b||(I(),b=setInterval(I,2e4))};addEventListener(\"pagehide\",L),addEventListener(\"pageshow\",E),document.addEventListener(\"visibilitychange\",E),b=setInterval(I,2e4),w(),E()})();";
 export const TRACKER_GZIP_BASE64 = "H4sIAAAAAAACA31W23LbOBL9FRmVYgHrNiI7l8pSi6gmO3nwludSq+RlXR4FIpsiNhTAAZrSaET++xRIUYrt1LyQALqBbjTOOQDnQr0/ZM4GmpDKXdZs0JLMGu/R0iLzpiZARXOZa9IBSQZDCObbEbR57YyltqW5DD5LEou7yef/3nGGW7QUGFAcF7L0WMxMwS+wbS+M8EiNt7MhulUxlczva3LSa5u7zefPtz/OJRdt+5Om8jjIxSVj4NUxxgK1z8pftdebwCuXaTLOytCPCgjKyzUSZx4LJuYyVCZDPoW3r8WsQpq4WeE8P25/4op7FlzjM2TANpibZsOAZXpTa7O2DBih70ecJbTEHsSxcjiGaWizZJf0JBImCeduPleHTtyPPg8KRTdM14qts8rkk/XKa5NPdsNvE7Lqq8knRL2Rdv2vMstC09Lkk0X2b5NPfE7LzOSTvLdibb5Ogg8VxQWy3rLJlmjyybIMaLOJWYey/5h8UqziJCZDXRnibMKELExF6Dmp916WOnAS4rSXd0PRMsihghIKRScMbE0w5HzbMga1+uBchdryQszI7w9F26p4NNWCnNdrlEu/bVtluegyTVl56KJXrgKGYJw9ekGlcrn0kPU/o1SRJLFFUKrQtrlcaiiThMeGKgW4to2eTZL8Z/HLz7LWPmA0NgLc4Nao3hLIG7s2xZ47MWZQqtDFvW1hraZHRDbq/qHf8OZMDI8Feo9+LNlLBq+E/L8zNrZF226dySfjCktFJ3ZZ1fST+kq+mYqeB1ZWaNdUPqaC76kQAf6hcit+/yTrQ6TgbZ4iDPRKLQS09AOlP2pCad2Oi048DIcVlOeiR/ksyGD+xPdvX+OrJBkjv7+eiUY2NpSmIG5l7WoueuJwMaMksXpr1pqclwFt/gF15iw3EETbFkhZyQ0cNkily1P26y+LTwxWLt+nAb4i1royW0yvOyH7KvNebDoBzTF4kiw5iQ72ihMgmLMWeXXeS0zmKptlykP41/U7fNO2vIroAVbrNW4N7tiFoiThC8UY7LgQA+4iWlQFA3yK/k/Kj5hrZN2Ekh/6It7m6ZcXh6qTLw5eklv0xeav3oo4su6+wBGbt3lawRHst3latG0FtK8xJbB6gylCralMT0IUe9Fw4tDNm7cCRhSlGyCzwUB6U6ceAv7eoM0wXV9egi4KUxlNuKiadVqClNJ0ArIKtf9kNuga4lsBWxWQxv4Srm+mU9H1J784ommnHmn8s9RmdKEWffEI9vxcUhigDIfauxo9GQypgygYqT6e31yng1MnRDfbGZu7nfQ6kHeKIxjwMbIp+CCZfxBTCo9YP/xNRU2OlkxhME/rLhKFHUf2cYG2ZR4DDosdnlqThF+YtjUnfN+8a9uLl7/d/3D1P3315/Tqnw/n5lKmVw//ePFSEgbiRoiRiM9XjQI0WpdcnKh6zqVt6zivVo9mQvG4PzdphO4z6XziVaTxlhOHUh2PwY2NzdgYT0FNe7SfE0kS/lRuVSGgF0rGoFdCxkYedJmaRtWDkT3dcG0PK+05y5pAbsPgy4uDadujsftyvhamAvxRdc8q5cV4Ajf4ej6knH6LJd+JDk7CqvP8Y1z4zgRCi56zrDLZVwZnBUVFkrRfI81lVrkQj4xpWDVEzsJ9rOfVAL6rPscHNogsPlZXo/BU+8G9DwxWYf9AiW+L8xPGRvl5+VtJVId5OiLFz2XtHbnMVQKcCknipfNmbezFmWDDwGzPzXysYermzDW0co3NWTpu0cBh2FfKn6Z2p1dYtS1KYy36T/gHxQ7p9c96g+LbVwbE3NMwjy+NMZmUMXHpT0wXj1ToRN0OrsWzd1DUxgVpik8hj3WlMxy63zx6ShPI+f09PczOTcWllBR5j1LXdbXnRxuQ6NHVPT/q2tVhiLUbbq3V6RK+HdRrz1mJ2tMKNTEBSy46uBtMqyThvSreWkK/1RVfCVipaRSzCvUWe/9rITr4OMw4ga40eY52fsdFumpbfsvjxIB0WukWbvB11Lbv5KzXWJocGdwJ+L45lG7H4KP4O5RH9q9MZWifldqusff/XhKxdvAx3uxczP4CGKXLCLcLAAA=";
 export const TRACKER_RAW_BYTES = 2999;