@iann29/rastro 0.9.0 → 0.10.1

package/src/component/pipeline.ts

@@ -0,0 +1,410 @@
+import { v, type Infer } from "convex/values";
+import type { Doc, Id } from "./_generated/dataModel.js";
+import {
+  mutation,
+  query,
+  type MutationCtx,
+  type QueryCtx,
+} from "./_generated/server.js";
+import { fail } from "./errors.js";
+import { ownerKeyOf } from "./meta.js";
+import { enqueueColumnEvent, type MetaUserData } from "./metaCapi.js";
+import { cleanString } from "./sanitize.js";
+import {
+  metaColumnEventValidator,
+  metaEventMapValidator,
+  pipelineColumnValidator,
+} from "./validators.js";
+
+/**
+ * The host's board for a site and the Meta event each column sends. The host
+ * declares its columns (`setPipeline`), moves leads between them (`column` on
+ * `leads.record` / `leads.setStage`), and an owner maps columns to events
+ * (`setEventMap`). Rastro counts the entries per column per day for the map's
+ * monthly figures. `Purchase` never comes from a column: it is sent when the
+ * host records the sale (`won` with a value). See docs/meta-ads.md.
+ */
+
+export type MetaColumnEvent = Infer<typeof metaColumnEventValidator>;
+
+export const MAX_PIPELINE_COLUMNS = 40;
+const COLUMN_KEY = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$/;
+const COLOR = /^[#A-Za-z0-9(),.% -]{1,32}$/;
+const MAX_COLUMN_NAME = 80;
+const MAX_GROUP_NAME = 60;
+const DAY_MS = 86_400_000;
+const MONTH_DAYS = 30;
+/** The column moves `leads.remove` clears per lead in one transaction. */
+const MAX_COLUMN_MOVES_PER_LEAD = 1_000;
+
+const dayOf = (at: number) => Math.floor(at / DAY_MS) * DAY_MS;
+
+function columnKeyOf(value: string): string {
+  const key = value.trim();
+  if (!COLUMN_KEY.test(key)) {
+    fail(
+      "INVALID_ARGUMENT",
+      "a column key is 1-64 letters, digits, '_', '.', ':' or '-'",
+    );
+  }
+  return key;
+}
+
+async function pipelineOf(ctx: MutationCtx, siteId: Id<"sites">) {
+  return await ctx.db
+    .query("sitePipelines")
+    .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+    .unique();
+}
+
+async function requireSite(ctx: QueryCtx, siteId: Id<"sites">): Promise<void> {
+  if (!(await ctx.db.get("sites", siteId))) fail("NOT_FOUND", "site not found");
+}
+
+/**
+ * Declares the host's board columns, in order. Declaring again replaces names,
+ * colors, groups and order; a column that is gone leaves the event map, and
+ * one that stays keeps its event.
+ */
+export const setPipeline = mutation({
+  args: { siteId: v.id("sites"), columns: v.array(pipelineColumnValidator) },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    await requireSite(ctx, args.siteId);
+    if (args.columns.length > MAX_PIPELINE_COLUMNS) {
+      fail(
+        "LIMIT_EXCEEDED",
+        `a pipeline has at most ${MAX_PIPELINE_COLUMNS} columns`,
+      );
+    }
+    const existing = await pipelineOf(ctx, args.siteId);
+    const events = new Map(
+      (existing?.columns ?? []).map((column) => [column.key, column.event]),
+    );
+    const seen = new Set<string>();
+    const columns = args.columns.map((column) => {
+      const key = columnKeyOf(column.key);
+      if (seen.has(key)) fail("INVALID_ARGUMENT", `column ${key} is repeated`);
+      seen.add(key);
+      const name = cleanString(column.name, MAX_COLUMN_NAME);
+      if (!name) fail("INVALID_ARGUMENT", `column ${key} needs a name`);
+      const color = column.color?.trim();
+      if (color !== undefined && color !== "" && !COLOR.test(color)) {
+        fail("INVALID_ARGUMENT", `column ${key} has an invalid color`);
+      }
+      const group = column.group
+        ? cleanString(column.group, MAX_GROUP_NAME)
+        : "";
+      const event = events.get(key);
+      return {
+        key,
+        name,
+        ...(color ? { color } : {}),
+        ...(group ? { group } : {}),
+        ...(event ? { event } : {}),
+      };
+    });
+    const now = Date.now();
+    if (existing) {
+      await ctx.db.patch("sitePipelines", existing._id, {
+        columns,
+        updatedAt: now,
+      });
+    } else {
+      await ctx.db.insert("sitePipelines", {
+        siteId: args.siteId,
+        columns,
+        sendLeadEvent: await pixelSendsLead(ctx, args.siteId),
+        updatedAt: now,
+      });
+    }
+    return null;
+  },
+});
+
+async function pixelSendsLead(ctx: MutationCtx, siteId: Id<"sites">) {
+  const pixel = await ctx.db
+    .query("sitePixels")
+    .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+    .unique();
+  return pixel?.sendLeadEvent ?? false;
+}
+
+/**
+ * Which event each declared column sends, replacing the whole map: a column
+ * left out, or mapped to null, sends nothing. `sendLeadEvent` turns `Lead` on
+ * for every new lead (kept on the pixel too, where the sender reads it).
+ */
+export const setEventMap = mutation({
+  args: {
+    siteId: v.id("sites"),
+    columns: v.record(v.string(), v.union(metaColumnEventValidator, v.null())),
+    sendLeadEvent: v.boolean(),
+    /**
+     * The owner saving the map. The map feeds the site's pixel, so whoever
+     * set the pixel decides it: with `ownerKey`, a pixel another owner (or
+     * the host) set is a `CONFLICT`. A site without a pixel takes anyone's.
+     */
+    ownerKey: v.optional(v.string()),
+  },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    await requireSite(ctx, args.siteId);
+    const pixel = await ctx.db
+      .query("sitePixels")
+      .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
+      .unique();
+    if (
+      args.ownerKey !== undefined &&
+      pixel &&
+      pixel.setBy !== ownerKeyOf(args.ownerKey)
+    ) {
+      fail(
+        "CONFLICT",
+        "the site's pixel was set by another owner on this host; change its events there",
+      );
+    }
+    const existing = await pipelineOf(ctx, args.siteId);
+    const declared = new Set((existing?.columns ?? []).map((row) => row.key));
+    for (const key of Object.keys(args.columns)) {
+      if (!declared.has(key)) {
+        fail(
+          "INVALID_ARGUMENT",
+          `column ${key.slice(0, 64)} is not on the site's pipeline`,
+        );
+      }
+    }
+    const columns = (existing?.columns ?? []).map(
+      ({ event: _event, ...column }) => {
+        const event = args.columns[column.key];
+        return event ? { ...column, event } : column;
+      },
+    );
+    const now = Date.now();
+    if (existing) {
+      await ctx.db.patch("sitePipelines", existing._id, {
+        columns,
+        sendLeadEvent: args.sendLeadEvent,
+        updatedAt: now,
+      });
+    } else {
+      await ctx.db.insert("sitePipelines", {
+        siteId: args.siteId,
+        columns,
+        sendLeadEvent: args.sendLeadEvent,
+        updatedAt: now,
+      });
+    }
+    // `updatedAt` stays: it says when the pixel itself was set, which is what
+    // removing its owner compares against.
+    if (pixel && pixel.sendLeadEvent !== args.sendLeadEvent) {
+      await ctx.db.patch("sitePixels", pixel._id, {
+        sendLeadEvent: args.sendLeadEvent,
+      });
+    }
+    return null;
+  },
+});
+
+/**
+ * The site's columns with their events and how many leads entered each in the
+ * 30 UTC days up to `now` (the caller's clock, so the query stays cacheable;
+ * without it, or before the host ever sent a column, the counts are null).
+ */
+export const eventMap = query({
+  args: { siteId: v.id("sites"), now: v.optional(v.number()) },
+  returns: metaEventMapValidator,
+  handler: async (ctx, args) => {
+    await requireSite(ctx, args.siteId);
+    if (
+      args.now !== undefined &&
+      (!Number.isSafeInteger(args.now) || args.now < 0)
+    ) {
+      fail("INVALID_ARGUMENT", "now must be a Unix time in milliseconds");
+    }
+    const pipeline = await ctx.db
+      .query("sitePipelines")
+      .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
+      .unique();
+    const pixel = await ctx.db
+      .query("sitePixels")
+      .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
+      .unique();
+    const counted =
+      args.now !== undefined && pipeline?.columnsSince !== undefined;
+    const to = dayOf(args.now ?? 0);
+    const from = to - (MONTH_DAYS - 1) * DAY_MS;
+    const columns = [];
+    for (const column of pipeline?.columns ?? []) {
+      let monthCount: number | null = null;
+      if (counted) {
+        // At most 30 rows: one per day.
+        const rows = await ctx.db
+          .query("leadColumnDaily")
+          .withIndex("by_siteId_and_column_and_dayStart", (range) =>
+            range
+              .eq("siteId", args.siteId)
+              .eq("column", column.key)
+              .gte("dayStart", from)
+              .lte("dayStart", to),
+          )
+          .take(MONTH_DAYS);
+        monthCount = rows.reduce((sum, row) => sum + row.entries, 0);
+      }
+      columns.push({
+        key: column.key,
+        name: column.name,
+        color: column.color ?? null,
+        group: column.group ?? null,
+        event: column.event ?? null,
+        monthCount,
+      });
+    }
+    return {
+      columns,
+      sendLeadEvent: pipeline?.sendLeadEvent ?? pixel?.sendLeadEvent ?? false,
+      purchase: { enabled: pixel?.enabled ?? false },
+    };
+  },
+});
+
+async function bumpDay(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  column: string,
+  dayStart: number,
+  delta: 1 | -1,
+) {
+  const row = await ctx.db
+    .query("leadColumnDaily")
+    .withIndex("by_siteId_and_column_and_dayStart", (range) =>
+      range.eq("siteId", siteId).eq("column", column).eq("dayStart", dayStart),
+    )
+    .unique();
+  if (!row) {
+    if (delta > 0) {
+      await ctx.db.insert("leadColumnDaily", {
+        siteId,
+        column,
+        dayStart,
+        entries: 1,
+      });
+    }
+    return;
+  }
+  const entries = row.entries + delta;
+  if (entries <= 0) await ctx.db.delete("leadColumnDaily", row._id);
+  else await ctx.db.patch("leadColumnDaily", row._id, { entries });
+}
+
+/**
+ * Moves a lead into the host's column `key`, from `leads.record` or
+ * `leads.setStage`. A key the site's pipeline does not declare is ignored,
+ * never an error, and so is a move into the column the lead is already in or
+ * one older than the lead's last move (`columnAt`): a late retry never moves
+ * a lead back. Only an accepted move counts and reaches the event map's seam.
+ * Returns the fields to patch on the lead, or null when nothing moved.
+ *
+ * The monthly counts are distinct leads: each lead keeps one mark per column
+ * on the day it last entered it, and the day's count follows the mark, so a
+ * lead that leaves and comes back is counted once, on its latest entry.
+ */
+export async function enterColumn(
+  ctx: MutationCtx,
+  lead: Pick<Doc<"leads">, "_id" | "siteId" | "column" | "columnAt">,
+  key: string | undefined,
+  at: number,
+  meta?: MetaUserData,
+): Promise<{ column: string; columnAt: number } | null> {
+  const column = key?.trim();
+  if (!column || column === lead.column) return null;
+  if (lead.columnAt !== undefined && at < lead.columnAt) return null;
+  const pipeline = await pipelineOf(ctx, lead.siteId);
+  const declared = pipeline?.columns.find((row) => row.key === column);
+  if (!pipeline || !declared) return null;
+  const dayStart = dayOf(at);
+  const mark = await ctx.db
+    .query("leadColumnMarks")
+    .withIndex("by_leadId_and_column", (range) =>
+      range.eq("leadId", lead._id).eq("column", column),
+    )
+    .unique();
+  if (!mark) {
+    await ctx.db.insert("leadColumnMarks", {
+      siteId: lead.siteId,
+      leadId: lead._id,
+      column,
+      dayStart,
+    });
+    await bumpDay(ctx, lead.siteId, column, dayStart, 1);
+  } else if (mark.dayStart < dayStart) {
+    await bumpDay(ctx, lead.siteId, column, mark.dayStart, -1);
+    await bumpDay(ctx, lead.siteId, column, dayStart, 1);
+    await ctx.db.patch("leadColumnMarks", mark._id, { dayStart });
+  }
+  if (pipeline.columnsSince === undefined) {
+    await ctx.db.patch("sitePipelines", pipeline._id, { columnsSince: at });
+  }
+  if (declared.event) {
+    await enqueueColumnEvent(ctx, {
+      siteId: lead.siteId,
+      leadId: lead._id,
+      eventName: declared.event,
+      at,
+      meta,
+    });
+  }
+  return { column, columnAt: at };
+}
+
+/**
+ * A column move the host sent alone (`setLeadStage` with a column and no
+ * stage), idempotent by its `eventId` like a stage change. Returns whether
+ * the eventId was new, and the lead it was first recorded for.
+ */
+export async function recordColumnMove(
+  ctx: MutationCtx,
+  move: {
+    siteId: Id<"sites">;
+    leadId: Id<"leads">;
+    eventId: string;
+    column: string;
+    at: number;
+  },
+): Promise<{ fresh: boolean; leadId: Id<"leads"> }> {
+  const seen = await ctx.db
+    .query("leadColumnMoves")
+    .withIndex("by_siteId_and_eventId", (range) =>
+      range.eq("siteId", move.siteId).eq("eventId", move.eventId),
+    )
+    .unique();
+  if (seen) return { fresh: false, leadId: seen.leadId };
+  await ctx.db.insert("leadColumnMoves", {
+    ...move,
+    column: move.column.slice(0, 64),
+  });
+  return { fresh: true, leadId: move.leadId };
+}
+
+/**
+ * Takes a removed lead out of the monthly counts and forgets its column
+ * marks and moves (both bounded per lead).
+ */
+export async function forgetLeadColumns(
+  ctx: MutationCtx,
+  lead: Pick<Doc<"leads">, "_id" | "siteId">,
+): Promise<void> {
+  const marks = await ctx.db
+    .query("leadColumnMarks")
+    .withIndex("by_leadId_and_column", (range) => range.eq("leadId", lead._id))
+    .take(MAX_PIPELINE_COLUMNS * 2);
+  for (const mark of marks) {
+    await bumpDay(ctx, lead.siteId, mark.column, mark.dayStart, -1);
+    await ctx.db.delete("leadColumnMarks", mark._id);
+  }
+  const moves = await ctx.db
+    .query("leadColumnMoves")
+    .withIndex("by_leadId", (range) => range.eq("leadId", lead._id))
+    .take(MAX_COLUMN_MOVES_PER_LEAD);
+  for (const move of moves) await ctx.db.delete("leadColumnMoves", move._id);
+}

package/src/component/schema.ts

@@ -6,6 +6,13 @@ import {
   aggregateFieldsValidator,
   leadEventTypeValidator,
   leadFieldsValidator,
+  metaColumnEventValidator,
+  metaConnectionFieldsValidator,
+  metaConversionEventNameValidator,
+  metaUserDataValidator,
+  pipelineColumnValidator,
+  siteAdAccountFieldsValidator,
+  sitePixelFieldsValidator,
   batchedEventValidator,
   funnelFieldsValidator,
   goalFieldsValidator,
@@ -430,12 +437,132 @@ export default defineSchema({
     "key",
   ]),
 
-  // Names the host fetches from the Meta Marketing API for the ads it saw.
+  // Names the Meta sync (or the host) fetches from the Marketing API.
   adCatalog: defineTable(adCatalogFieldsValidator).index("by_siteId_and_adId", [
     "siteId",
     "adId",
   ]),
 
+  // A Meta token the host pasted for one of its owners (see meta.ts),
+  // encrypted with RASTRO_SECRETS_KEY. No query returns the cipher.
+  metaConnections: defineTable(metaConnectionFieldsValidator).index(
+    "by_ownerKey",
+    ["ownerKey"],
+  ),
+
+  // A site's spend source: one Meta ad account, synced by the chain in
+  // metaSync.ts. `nextSyncAt` is when the chain takes it next.
+  siteAdAccounts: defineTable(siteAdAccountFieldsValidator)
+    .index("by_siteId", ["siteId"])
+    .index("by_connectionId", ["connectionId"])
+    .index("by_nextSyncAt", ["nextSyncAt"])
+    .index("by_siteId_and_adAccountId", ["siteId", "adAccountId"])
+    // An ad account feeds one site: linking checks it here.
+    .index("by_adAccountId", ["adAccountId"]),
+
+  // What one ad spent on one account-local day (`dayStart` is that date's
+  // UTC midnight). Meta's figures replace the row on every resync.
+  adSpendDaily: defineTable({
+    siteId: v.id("sites"),
+    adAccountId: v.string(),
+    adId: v.string(),
+    dayStart: v.number(),
+    spendCents: v.number(),
+    impressions: v.number(),
+    clicks: v.number(),
+    metaConversations: v.optional(v.number()),
+    updatedAt: v.number(),
+  })
+    .index("by_siteId_and_dayStart", ["siteId", "dayStart"])
+    .index("by_siteId_and_adId_and_dayStart", ["siteId", "adId", "dayStart"])
+    .index("by_siteId_and_adAccountId_and_dayStart", [
+      "siteId",
+      "adAccountId",
+      "dayStart",
+    ]),
+
+  // `adSpendDaily` summed per site, ad account and day, kept by every write
+  // to it: the site totals read these few rows, so linking Meta never eats the
+  // lead report's row budget.
+  siteSpendDaily: defineTable({
+    siteId: v.id("sites"),
+    adAccountId: v.string(),
+    dayStart: v.number(),
+    spendCents: v.number(),
+    impressions: v.number(),
+    clicks: v.number(),
+  })
+    .index("by_siteId_and_dayStart", ["siteId", "dayStart"])
+    .index("by_siteId_and_adAccountId_and_dayStart", [
+      "siteId",
+      "adAccountId",
+      "dayStart",
+    ]),
+
+  // The site's Meta pixel (dataset) for the Conversions API.
+  sitePixels: defineTable(sitePixelFieldsValidator)
+    .index("by_siteId", ["siteId"])
+    // A connection's pixels go with it when it is removed.
+    .index("by_connectionId", ["connectionId"])
+    // The pixels an owner set, for the federated scope check.
+    .index("by_setBy", ["setBy"]),
+
+  // The host's board for a site (its columns in order) and which Meta event
+  // each column sends. One row per site; see pipeline.ts.
+  sitePipelines: defineTable({
+    siteId: v.id("sites"),
+    columns: v.array(
+      pipelineColumnValidator.extend({
+        event: v.optional(metaColumnEventValidator),
+      }),
+    ),
+    /** Whether a new lead is sent as `Lead` (mirrored on the pixel). */
+    sendLeadEvent: v.boolean(),
+    /** When the host first sent a lead's column: counts start there. */
+    columnsSince: v.optional(v.number()),
+    updatedAt: v.number(),
+  }).index("by_siteId", ["siteId"]),
+
+  // Leads whose latest entry into a column fell on each UTC day, for the
+  // event map's monthly counts of distinct leads. Lead volume is human
+  // scale, so a row is never a hot spot.
+  leadColumnDaily: defineTable({
+    siteId: v.id("sites"),
+    column: v.string(),
+    dayStart: v.number(),
+    entries: v.number(),
+  }).index("by_siteId_and_column_and_dayStart", [
+    "siteId",
+    "column",
+    "dayStart",
+  ]),
+
+  // One per lead and column: the UTC day the lead last entered it.
+  leadColumnMarks: defineTable({
+    siteId: v.id("sites"),
+    leadId: v.id("leads"),
+    column: v.string(),
+    dayStart: v.number(),
+  }).index("by_leadId_and_column", ["leadId", "column"]),
+
+  // Column moves the host sent alone, so a retried `eventId` is a no-op.
+  leadColumnMoves: defineTable({
+    siteId: v.id("sites"),
+    leadId: v.id("leads"),
+    eventId: v.string(),
+    column: v.string(),
+    at: v.number(),
+  })
+    .index("by_siteId_and_eventId", ["siteId", "eventId"])
+    .index("by_leadId", ["leadId"]),
+
+  // The single scheduler chain that syncs every linked ad account.
+  metaSyncLoop: defineTable({
+    key: v.literal("loop"),
+    scheduledFor: v.number(),
+    jobId: v.id("_scheduled_functions"),
+  }).index("by_key", ["key"]),
+
   visitorSecrets: defineTable({
     key: v.literal("current"),
     value: v.string(),
@@ -569,4 +696,73 @@ export default defineSchema({
       "windowStart",
     ])
     .index("by_siteId_and_updatedAt", ["siteId", "updatedAt"]),
+
+  // Conversions waiting for (or done with) the site's Meta pixel, sent by
+  // metaCapi.ts. `userData` holds the host's hashes only until the final
+  // attempt, then it is erased.
+  metaConversionOutbox: defineTable({
+    siteId: v.id("sites"),
+    leadId: v.id("leads"),
+    eventId: v.string(),
+    eventName: metaConversionEventNameValidator,
+    eventTime: v.number(),
+    valueCents: v.optional(v.number()),
+    currency: v.optional(v.string()),
+    userData: v.optional(metaUserDataValidator),
+    status: v.union(
+      v.literal("pending"),
+      v.literal("sent"),
+      v.literal("failed"),
+    ),
+    attempts: v.number(),
+    /** When the current attempt is due; the sweep reads it to find stalls. */
+    nextAttemptAt: v.number(),
+    /** When a send took the current attempt: its lease, for the sweep. */
+    attemptStartedAt: v.optional(v.number()),
+    lastError: v.optional(v.string()),
+    fbtraceId: v.optional(v.string()),
+    sentAt: v.optional(v.number()),
+    createdAt: v.number(),
+  })
+    .index("by_siteId_and_eventId", ["siteId", "eventId"])
+    .index("by_siteId_and_createdAt", ["siteId", "createdAt"])
+    .index("by_siteId_and_status_and_createdAt", [
+      "siteId",
+      "status",
+      "createdAt",
+    ])
+    .index("by_leadId", ["leadId"])
+    .index("by_status_and_nextAttemptAt", ["status", "nextAttemptAt"]),
+
+  // The single scheduler chain that re-sends stalled outbox rows; it stops
+  // by itself when nothing is pending.
+  metaCapiSweep: defineTable({
+    key: v.literal("sweep"),
+    scheduledFor: v.number(),
+    jobId: v.id("_scheduled_functions"),
+  }).index("by_key", ["key"]),
+
+  // The outbox counted per site and status, kept by every write to it, so
+  // `meta.status` never scans the history. Split in shards by event (see
+  // metaCapi.ts) so concurrent sends rarely write the same document.
+  metaConversionCounts: defineTable({
+    siteId: v.id("sites"),
+    shard: v.number(),
+    pending: v.number(),
+    sent: v.number(),
+    failed: v.number(),
+  }).index("by_siteId_and_shard", ["siteId", "shard"]),
+
+  // Conversions Meta accepted, per site, UTC day and event, in the same
+  // shards as the counts: `meta.status` reads the last 30 days through the
+  // index, at most 31 days times the shards.
+  metaConversionDaily: defineTable({
+    siteId: v.id("sites"),
+    dayStart: v.number(),
+    shard: v.number(),
+    Purchase: v.number(),
+    QualifiedLead: v.number(),
+    Lead: v.number(),
+    Schedule: v.number(),
+  }).index("by_siteId_and_dayStart_and_shard", ["siteId", "dayStart", "shard"]),
 });