@iann29/rastro 0.10.9 → 0.11.0

package/src/component/meta.ts

@@ -32,6 +32,7 @@ import {
   MAX_LINKS_PER_CONNECTION,
   PAUSED_AT,
 } from "./metaSync.js";
+import { removeConnectionPages, syncOwnerPages } from "./metaLeads.js";
 import { sanitizeOpaqueId } from "./sanitize.js";
 import { decryptSecret, encryptSecret, hasSecretsKey } from "./secrets.js";
 import {
@@ -584,6 +585,13 @@ async function connectToken(
       lastError: login?.lastError ?? null,
     },
   );
+  // The Pages and their own tokens, on a host that opted into Lead Ads. Never
+  // fails the login: a login without the Page permissions simply lists none.
+  await syncOwnerPages(ctx, client, {
+    connectionId,
+    token,
+    tokenIv: sealed.iv,
+  });
   return { connectionId, businessName, adAccounts, pixels };
 }
 
@@ -683,8 +691,8 @@ export const listAdAccounts = action({
     let adAccounts: MetaAdAccount[];
     let pixels: MetaPixel[];
     let pixelsComplete: boolean;
+    const client = graph();
     try {
-      const client = graph();
       adAccounts = (await client.adAccounts(token, optionalLoginApp())).map(
         toAccount,
       );
@@ -710,6 +718,13 @@ export const listAdAccounts = action({
       pixels,
       pixelsComplete,
     });
+    // The Pages too (Lead Ads hosts): a Page granted since the login shows
+    // up, one taken back stops.
+    await syncOwnerPages(ctx, client, {
+      connectionId: connection.connectionId,
+      token,
+      tokenIv: connection.tokenIv,
+    });
     return adAccounts;
   },
 });
@@ -787,6 +802,10 @@ async function removeOwnerBatch(
         .take(UNLINK_BATCH)
     : [];
   for (const link of links) await unlinkRow(ctx, link);
+  // Its Pages and their links go with it (the leads already in stay).
+  const morePages = connectionId
+    ? await removeConnectionPages(ctx, connectionId, UNLINK_BATCH)
+    : false;
   // A pixel picked through the connection sent with its token: it goes too.
   const connectionPixels = connectionId
     ? await ctx.db
@@ -819,6 +838,7 @@ async function removeOwnerBatch(
   const cursor = ownerPixels.at(-1)?._creationTime;
   if (
     links.length === UNLINK_BATCH ||
+    morePages ||
     connectionPixels.length === UNLINK_BATCH ||
     ownerPixels.length === UNLINK_BATCH
   ) {

package/src/component/metaFake.ts

@@ -16,7 +16,10 @@ import {
  * which Meta refuses like an expired token. A token
  * `rastro-fake-sample-token:<days>` moves every date by that many whole days,
  * so a demo seeded today reads the sample as its last 30 days, sync after
- * sync. Conversions API sends are accepted. No network call is made.
+ * sync. Conversions API sends are accepted. Lead Ads: a Facebook Page per
+ * client (and "Sorriso Kids", whose leads Meta did not grant), their instant
+ * forms, and a steady stream of made-up leads (`FAKE_PAGES` below). No
+ * network call is made.
  */
 
 const BUSINESS = { id: "sample-business", name: "Connection Company" };
@@ -177,6 +180,375 @@ function account(adAccountId: string): FakeAccount {
   return found;
 }
 
+// ---------------------------------------------------------------------------
+// Lead Ads: a Facebook Page per sample client, their instant forms, and a
+// steady stream of leads. Lead `k` of a form is created at `k × every +
+// offset` (UTC ms), so the leads before now always exist and new ones show up
+// as time passes, for the webhook-free reconciliation to find. Answers and
+// contacts derive from `k` (made-up people, never real ones).
+
+const HOUR_MS = 3_600_000;
+const FAKE_PAGE_TOKEN = "rastro-fake-page-token:";
+const LEAD_HISTORY_MS = 90 * DAY_MS;
+/** How many leads one fake listing holds at most (Meta's cap); tests lower it. */
+export const fakeLimits = { maxLeads: 2_000 };
+/**
+ * The Pages the app is subscribed to, as Meta keeps it: one subscription per
+ * Page and app, whichever host asked for it. Tests read it, and take a Page
+ * off it the way another host's unsubscription would.
+ */
+export const fakeSubscriptions = new Set<string>();
+
+type FakeQuestion = {
+  key: string;
+  label: string;
+  type: string;
+  options?: Array<{ key: string; value: string }>;
+};
+
+interface FakeForm {
+  formId: string;
+  /** The two digits every leadgen id of this form starts with (after "7"). */
+  serial: string;
+  name: string;
+  status: string;
+  questions: FakeQuestion[];
+  whatsappButton: boolean | null;
+  /** How long after its creation (days before now) the form exists. */
+  createdDaysAgo: number;
+  /** A lead every this many ms; 0 for a form that brings none. */
+  everyMs: number;
+  offsetMs: number;
+}
+
+interface FakePage {
+  pageId: string;
+  name: string;
+  /** The sample client whose ads bring the leads, or null. */
+  sampleSiteId: string | null;
+  leadsAllowed: boolean;
+  forms: FakeForm[];
+}
+
+const CONTACT_QUESTIONS: FakeQuestion[] = [
+  { key: "full_name", label: "Nome completo", type: "FULL_NAME" },
+  { key: "phone_number", label: "Telefone", type: "PHONE" },
+];
+const EMAIL_QUESTION: FakeQuestion = {
+  key: "email",
+  label: "E-mail",
+  type: "EMAIL",
+};
+const choice = (
+  key: string,
+  label: string,
+  options: Array<[string, string]>,
+): FakeQuestion => ({
+  key,
+  label,
+  type: "CUSTOM",
+  options: options.map(([optionKey, value]) => ({ key: optionKey, value })),
+});
+
+/** The fake Pages: one per sample client, plus one without lead access. */
+export const FAKE_PAGES: readonly FakePage[] = [
+  {
+    pageId: "4100000000001",
+    name: "Loja X",
+    sampleSiteId: "sample-loja-x",
+    leadsAllowed: true,
+    forms: [
+      {
+        formId: "4200000000101",
+        serial: "11",
+        name: "Provador VIP",
+        status: "ACTIVE",
+        questions: [
+          ...CONTACT_QUESTIONS,
+          choice("tamanho", "Qual o seu tamanho?", [
+            ["p", "P"],
+            ["m", "M"],
+            ["g", "G"],
+          ]),
+        ],
+        whatsappButton: true,
+        createdDaysAgo: 120,
+        everyMs: 8 * HOUR_MS,
+        offsetMs: 25 * 60_000,
+      },
+    ],
+  },
+  {
+    pageId: "4100000000002",
+    name: "Clínica Sorriso",
+    sampleSiteId: "sample-clinica-sorriso",
+    leadsAllowed: true,
+    forms: [
+      {
+        formId: "4200000000201",
+        serial: "21",
+        name: "Avaliação gratuita",
+        status: "ACTIVE",
+        questions: [
+          ...CONTACT_QUESTIONS,
+          EMAIL_QUESTION,
+          choice("orcamento", "Orçamento para o tratamento", [
+            ["ate_1000", "Até R$ 1.000"],
+            ["de_1000_a_5000", "R$ 1.000 a R$ 5.000"],
+            ["acima_5000", "Acima de R$ 5.000"],
+          ]),
+          choice("agendar", "Quer agendar a avaliação?", [
+            ["sim", "Sim"],
+            ["nao", "Não"],
+          ]),
+          choice("tratamento", "Tratamento de interesse", [
+            ["implante", "Implante"],
+            ["lente", "Lente de contato"],
+            ["clareamento", "Clareamento"],
+          ]),
+        ],
+        whatsappButton: false,
+        createdDaysAgo: 200,
+        everyMs: 675 * 60_000,
+        offsetMs: 4 * 60_000,
+      },
+      {
+        formId: "4200000000202",
+        serial: "22",
+        name: "Clareamento — orçamento",
+        status: "ACTIVE",
+        questions: [
+          ...CONTACT_QUESTIONS,
+          choice("quando", "Quando quer começar?", [
+            ["esta_semana", "Esta semana"],
+            ["este_mes", "Este mês"],
+            ["sem_pressa", "Sem pressa"],
+          ]),
+        ],
+        whatsappButton: true,
+        createdDaysAgo: 90,
+        everyMs: 40 * HOUR_MS,
+        offsetMs: 19 * HOUR_MS + 10 * 60_000,
+      },
+      {
+        formId: "4200000000203",
+        serial: "23",
+        name: "Lente de contato dental",
+        status: "ACTIVE",
+        questions: [
+          ...CONTACT_QUESTIONS,
+          EMAIL_QUESTION,
+          choice("lente_dentes", "Quantos dentes?", [
+            ["ate_4", "Até 4"],
+            ["de_6_a_10", "De 6 a 10"],
+            ["arcada", "A arcada toda"],
+          ]),
+        ],
+        whatsappButton: null,
+        createdDaysAgo: 2,
+        everyMs: 12 * HOUR_MS,
+        offsetMs: 9 * HOUR_MS + 2 * 60_000,
+      },
+      {
+        formId: "4200000000204",
+        serial: "24",
+        name: "Promo setembro",
+        status: "ARCHIVED",
+        questions: [
+          ...CONTACT_QUESTIONS,
+          EMAIL_QUESTION,
+          choice("promo", "Qual tratamento da promoção?", [
+            ["limpeza", "Limpeza"],
+            ["clareamento", "Clareamento"],
+          ]),
+        ],
+        whatsappButton: null,
+        createdDaysAgo: 60,
+        everyMs: 0,
+        offsetMs: 0,
+      },
+    ],
+  },
+  {
+    pageId: "4100000000003",
+    name: "Pet Mania",
+    sampleSiteId: "sample-pet-mania",
+    leadsAllowed: true,
+    forms: [
+      {
+        formId: "4200000000301",
+        serial: "31",
+        name: "Agende o banho",
+        status: "ACTIVE",
+        questions: [
+          ...CONTACT_QUESTIONS,
+          choice("porte", "Porte do pet", [
+            ["pequeno", "Pequeno"],
+            ["medio", "Médio"],
+            ["grande", "Grande"],
+          ]),
+        ],
+        whatsappButton: null,
+        createdDaysAgo: 150,
+        everyMs: 10 * HOUR_MS,
+        offsetMs: 2 * HOUR_MS,
+      },
+    ],
+  },
+  {
+    pageId: "4100000000004",
+    name: "Casa Aurora",
+    sampleSiteId: "sample-casa-aurora",
+    leadsAllowed: true,
+    forms: [
+      {
+        formId: "4200000000401",
+        serial: "41",
+        name: "Orçamento de móveis",
+        status: "ACTIVE",
+        questions: [
+          ...CONTACT_QUESTIONS,
+          EMAIL_QUESTION,
+          choice("ambiente", "Qual ambiente?", [
+            ["sala", "Sala"],
+            ["quarto", "Quarto"],
+            ["cozinha", "Cozinha"],
+          ]),
+          {
+            key: "detalhes",
+            label: "Conte o que você precisa",
+            type: "CUSTOM",
+          },
+        ],
+        whatsappButton: false,
+        createdDaysAgo: 100,
+        everyMs: 20 * HOUR_MS,
+        offsetMs: 13 * HOUR_MS,
+      },
+    ],
+  },
+  {
+    pageId: "4100000000099",
+    name: "Sorriso Kids",
+    sampleSiteId: null,
+    leadsAllowed: false,
+    forms: [],
+  },
+];
+
+const FAKE_FIRST = [
+  "Marina",
+  "Rafael",
+  "Juliana",
+  "Bruno",
+  "Patrícia",
+  "Thiago",
+  "Larissa",
+  "Felipe",
+  "Camila",
+  "Gustavo",
+];
+const FAKE_LAST = ["Costa", "Souza", "Lima", "Alves", "Rocha", "Pires"];
+
+const fakePage = (pageId: string) =>
+  FAKE_PAGES.find((page) => page.pageId === pageId) ?? null;
+
+function fakeForm(formId: string) {
+  for (const page of FAKE_PAGES) {
+    const form = page.forms.find((row) => row.formId === formId);
+    if (form) return { page, form };
+  }
+  return null;
+}
+
+const formCreatedAt = (form: FakeForm, now: number) =>
+  Math.floor((now - form.createdDaysAgo * DAY_MS) / DAY_MS) * DAY_MS;
+
+/** The `k`-th lead of a form, as Meta returns it. */
+function fakeLead(page: FakePage, form: FakeForm, k: number) {
+  const createdTime = k * form.everyMs + form.offsetMs;
+  const first = FAKE_FIRST[k % FAKE_FIRST.length]!;
+  const last = FAKE_LAST[(k * 7) % FAKE_LAST.length]!;
+  const digits = String((k * 7919 + Number(form.serial) * 104_729) % 1e8)
+    .padStart(8, "0")
+    .slice(0, 8);
+  const fields = form.questions.map((question, index) => {
+    switch (question.type) {
+      case "FULL_NAME":
+        return { name: question.key, values: [`${first} ${last}`] };
+      case "PHONE":
+        return { name: question.key, values: [`+55549${digits}`] };
+      case "EMAIL":
+        return {
+          name: question.key,
+          values: [
+            `${first.normalize("NFD").replace(/[̀-ͯ]/g, "").toLowerCase()}.${k}@example.com`,
+          ],
+        };
+      default: {
+        const options = question.options ?? [];
+        if (options.length === 0) {
+          return { name: question.key, values: ["Preciso de um orçamento"] };
+        }
+        // Meta returns the option's shown value; the ingestion maps it back.
+        const option = options[(k + index * 3) % options.length]!;
+        return { name: question.key, values: [option.value] };
+      }
+    }
+  });
+  const ads =
+    page.sampleSiteId === null
+      ? []
+      : (SAMPLE_SEED.find(
+          (site) => site.siteId === page.sampleSiteId,
+        )?.campaigns.flatMap((campaign) =>
+          campaign.adsets.flatMap((adset) =>
+            adset.ads.map((ad) => ({
+              adId: ad.adId,
+              adsetId: adset.adsetId,
+              campaignId: campaign.campaignId,
+            })),
+          ),
+        ) ?? []);
+  const ad = ads.length > 0 ? ads[k % ads.length]! : null;
+  return {
+    leadgenId: `7${form.serial}${String(k).padStart(9, "0")}`,
+    createdTime,
+    formId: form.formId,
+    adId: ad?.adId ?? null,
+    adsetId: ad?.adsetId ?? null,
+    campaignId: ad?.campaignId ?? null,
+    platform: k % 3 === 0 ? "ig" : "fb",
+    isOrganic: ad === null,
+    fields,
+  };
+}
+
+function checkPageToken(token: string, pageId?: string) {
+  checkToken(token);
+  if (pageId !== undefined && token !== `${FAKE_PAGE_TOKEN}${pageId}`) {
+    throw new MetaGraphError(
+      "permission",
+      "the Page token does not act for this Page (fake)",
+      200,
+    );
+  }
+}
+
+const pageIdOfToken = (token: string) =>
+  token.startsWith(FAKE_PAGE_TOKEN) ? token.slice(FAKE_PAGE_TOKEN.length) : "";
+
+function leadPermission(page: FakePage | null): FakePage {
+  if (!page?.leadsAllowed) {
+    throw new MetaGraphError(
+      "permission",
+      "leads_retrieval was not granted for this Page (fake)",
+      200,
+    );
+  }
+  return page;
+}
+
 export function fakeMetaGraph(): MetaGraph {
   return {
     async exchangeCode(_app, code) {
@@ -277,5 +649,117 @@ export function fakeMetaGraph(): MetaGraph {
       checkToken(token);
       return { eventsReceived: payload.data.length, fbtraceId: "fake-trace" };
     },
+    async pages(token) {
+      checkToken(token);
+      return {
+        pages: FAKE_PAGES.map((page) => ({
+          pageId: page.pageId,
+          name: page.name,
+          accessToken: `${FAKE_PAGE_TOKEN}${page.pageId}`,
+          tasks: ["ADVERTISE", "ANALYZE", "CREATE_CONTENT", "MANAGE"],
+        })),
+        complete: true,
+      };
+    },
+    async leadAccess(token) {
+      checkToken(token);
+      return {
+        all: false,
+        pageIds: FAKE_PAGES.filter((page) => page.leadsAllowed).map(
+          (page) => page.pageId,
+        ),
+      };
+    },
+    async subscribePage(pageToken, pageId) {
+      checkPageToken(pageToken, pageId);
+      leadPermission(fakePage(pageId));
+      fakeSubscriptions.add(pageId);
+    },
+    async leadForms(pageToken, pageId) {
+      checkPageToken(pageToken, pageId);
+      const page = leadPermission(fakePage(pageId));
+      const now = Date.now();
+      return {
+        forms: page.forms.map((form) => ({
+          formId: form.formId,
+          name: form.name,
+          status: form.status,
+          questions: form.questions.map((question) => ({ ...question })),
+          whatsappButton: form.whatsappButton,
+          createdTime: formCreatedAt(form, now),
+        })),
+        usage: USAGE,
+        complete: true,
+      };
+    },
+    async lead(pageToken, leadgenId) {
+      checkToken(pageToken);
+      const match = /^7(\d{2})(\d{9})$/.exec(leadgenId);
+      const found = match
+        ? FAKE_PAGES.flatMap((page) =>
+            page.forms
+              .filter((form) => form.serial === match[1])
+              .map((form) => ({ page, form })),
+          )[0]
+        : undefined;
+      const now = Date.now();
+      const lead =
+        found && found.form.everyMs > 0
+          ? fakeLead(found.page, found.form, Number(match![2]))
+          : null;
+      if (
+        !found ||
+        !lead ||
+        pageIdOfToken(pageToken) !== found.page.pageId ||
+        lead.createdTime > now ||
+        lead.createdTime < formCreatedAt(found.form, now)
+      ) {
+        throw new MetaGraphError(
+          "permission",
+          "Unsupported get request: the lead does not exist (fake)",
+          100,
+        );
+      }
+      return lead;
+    },
+    async formLeads(pageToken, formId, sinceSec, untilSec) {
+      checkToken(pageToken);
+      const found = fakeForm(formId);
+      if (!found || pageIdOfToken(pageToken) !== found.page.pageId) {
+        throw new MetaGraphError(
+          "permission",
+          "the form is not this Page's (fake)",
+          100,
+        );
+      }
+      leadPermission(found.page);
+      const { page, form } = found;
+      const now = Date.now();
+      if (form.everyMs <= 0) return { leads: [], usage: USAGE, complete: true };
+      const from = Math.max(
+        sinceSec * 1000 + 1,
+        now - LEAD_HISTORY_MS,
+        formCreatedAt(form, now),
+      );
+      const to =
+        untilSec === undefined ? now : Math.min(now, untilSec * 1000 - 1);
+      const first = Math.max(
+        Math.ceil((from - form.offsetMs) / form.everyMs),
+        0,
+      );
+      const last = Math.floor((to - form.offsetMs) / form.everyMs);
+      // Meta lists the newest first and stops at its cap: a truncated
+      // listing holds the newest leads of the window, never the oldest.
+      const leads = [];
+      for (let k = last; k >= first; k -= 1) {
+        if (leads.length >= fakeLimits.maxLeads) break;
+        leads.push(fakeLead(page, form, k));
+      }
+      return {
+        leads,
+        usage: USAGE,
+        complete: last - first + 1 <= fakeLimits.maxLeads,
+      };
+    },
   };
 }