@iann29/rastro 0.5.0 → 0.7.0

package/src/component/_generated/component.ts

@@ -241,6 +241,46 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         Name
       >;
     };
+    people: {
+      backfillSearch: FunctionReference<
+        "mutation",
+        "internal",
+        { siteId: string },
+        { isDone: boolean; updated: number },
+        Name
+      >;
+      list: FunctionReference<
+        "query",
+        "internal",
+        {
+          filter?:
+            | { field: "name" | "email" | "visitorId"; prefix: string }
+            | {
+                field: "attribute";
+                key: string;
+                value: string | number | boolean;
+              };
+          paginationOpts: { cursor: string | null; numItems: number };
+          siteId: string;
+        },
+        {
+          continueCursor: string;
+          isDone: boolean;
+          page: Array<{
+            _creationTime: number;
+            _id: string;
+            attributes?: Record<string, string | number | boolean>;
+            email?: string;
+            identifiedAt: number;
+            name?: string;
+            siteId: string;
+            updatedAt: number;
+            visitorId: string;
+          }>;
+        },
+        Name
+      >;
+    };
     reports: {
       affiliatesReport: FunctionReference<
         "query",
@@ -415,6 +455,14 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           os: string;
           pageviewCount: number;
           platform?: string;
+          profile?: {
+            attributes?: Record<string, string | number | boolean>;
+            email?: string;
+            identifiedAt: number;
+            name?: string;
+            updatedAt: number;
+            visitorId: string;
+          };
           referrer?: string;
           revenueCents: number;
           sessionId: string;
@@ -575,6 +623,14 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             os: string;
             pageviewCount: number;
             platform?: string;
+            profile?: {
+              attributes?: Record<string, string | number | boolean>;
+              email?: string;
+              identifiedAt: number;
+              name?: string;
+              updatedAt: number;
+              visitorId: string;
+            };
             referrer?: string;
             revenueCents: number;
             sessionId: string;
@@ -617,6 +673,14 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           pageviewCount: number;
           path: string;
           previousPath?: string;
+          profile?: {
+            attributes?: Record<string, string | number | boolean>;
+            email?: string;
+            identifiedAt: number;
+            name?: string;
+            updatedAt: number;
+            visitorId: string;
+          };
           returning?: boolean;
           sessionId: string;
           siteId: string;
@@ -878,6 +942,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             };
             visitors: { basis: "estimatedUnique"; exact: boolean };
           };
+          routePatterns?: Record<string, Array<string>>;
           routes: Array<{
             bounces: number;
             daily: Array<{ bucketStart: number; pageviews: number }>;
@@ -1139,6 +1204,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           name: string;
           networkId?: string;
           ownerId: string;
+          routePatterns?: Array<string>;
           timezone?: string;
         },
         string,
@@ -1158,6 +1224,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           name: string;
           networkId?: string;
           ownerId: string;
+          routePatterns?: Array<string>;
           timezone?: string;
           updatedAt: number;
         } | null,
@@ -1177,6 +1244,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           name: string;
           networkId?: string;
           ownerId: string;
+          routePatterns?: Array<string>;
           timezone?: string;
           updatedAt: number;
         }>,
@@ -1198,6 +1266,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           domains?: Array<string>;
           name?: string;
           networkId?: string | null;
+          routePatterns?: Array<string>;
           siteId: string;
           timezone?: string | null;
         },
@@ -1292,6 +1361,41 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
       >;
     };
     visitors: {
+      getProfile: FunctionReference<
+        "query",
+        "internal",
+        { siteId: string; visitorId: string },
+        {
+          attributes?: Record<string, string | number | boolean>;
+          email?: string;
+          identifiedAt: number;
+          name?: string;
+          updatedAt: number;
+          visitorId: string;
+        } | null,
+        Name
+      >;
+      identify: FunctionReference<
+        "mutation",
+        "internal",
+        {
+          attributes?: Record<string, string | number | boolean | null>;
+          email?: string | null;
+          name?: string | null;
+          previousVisitorId?: string;
+          siteId: string;
+          visitorId: string;
+        },
+        {
+          attributes?: Record<string, string | number | boolean>;
+          email?: string;
+          identifiedAt: number;
+          name?: string;
+          updatedAt: number;
+          visitorId: string;
+        },
+        Name
+      >;
       link: FunctionReference<
         "mutation",
         "internal",

package/src/component/ingest.ts

@@ -89,6 +89,7 @@ import {
 } from "./rollupStore.js";
 import { localDayStart, localDayTimezone } from "./localTime.js";
 import { classifyOrigin, platformDimensionValue } from "./origin.js";
+import { resolveVisitorIdentities } from "./visitors.js";
 
 type DimensionType =
   | "source"
@@ -217,6 +218,7 @@ type BatchState = {
     CachedRow<CachedDocument<"visitorAttributions">>
   >;
   funnelProgress: Map<string, CachedRow<CachedDocument<"funnelProgress">>>;
+  funnelIdentities: Map<string, string[]>;
   eventBatches: EventBatchBuffer;
   liveSweepSites: Set<string>;
 };
@@ -286,6 +288,7 @@ export const ingestBatch = mutation({
       liveSessions: new Map(),
       visitorAttributions: new Map(),
       funnelProgress: new Map(),
+      funnelIdentities: new Map(),
       eventBatches: new Map(),
       liveSweepSites: new Set(),
     };
@@ -579,6 +582,7 @@ export const ingestBatch = mutation({
         recordTelemetry: true,
         timezone,
         siteDomains: site.domains,
+        routePatterns: site.routePatterns,
         financialConversion: false,
         trustedGoalKey: undefined,
         now,
@@ -710,6 +714,7 @@ export const trackConversion = mutation({
       recordTelemetry: existingTelemetry === null && !telemetry.saturated,
       timezone: localDayTimezone(site.timezone),
       siteDomains: site.domains,
+      routePatterns: site.routePatterns,
       financialConversion: true,
       trustedGoalKey,
       now: Date.now(),
@@ -850,6 +855,7 @@ async function accountEvent(
     timezone?: string;
     /** The site's own domains: a referrer on them is not an origin. */
     siteDomains: string[];
+    routePatterns?: string[];
     financialConversion: boolean;
     trustedGoalKey: string | undefined;
     now: number;
@@ -879,6 +885,7 @@ async function accountEvent(
       event,
       session,
       input.existingSession,
+      input.routePatterns,
     );
   }
 
@@ -1927,46 +1934,62 @@ async function advanceFunnel(
   reportRollupDeltas: ReportRollupDeltas,
   batchState?: BatchState,
 ): Promise<FunnelReach | undefined> {
-  const cacheKey = JSON.stringify([funnel._id, event.visitorId]);
+  let identities = batchState?.funnelIdentities.get(event.visitorId);
+  if (!identities) {
+    identities = await resolveVisitorIdentities(ctx, siteId, event.visitorId);
+    for (const id of identities)
+      batchState?.funnelIdentities.set(id, identities);
+  }
+  const cacheKey = JSON.stringify([funnel._id, identities[0]]);
   let progressState = batchState?.funnelProgress.get(cacheKey);
-  if (batchState && !progressState) {
-    const current = await ctx.db
-      .query("funnelProgress")
-      .withIndex("by_siteId_and_funnelId_and_visitorId", (range) =>
-        range
-          .eq("siteId", siteId)
-          .eq("funnelId", funnel._id)
-          .eq("visitorId", event.visitorId),
-      )
-      .unique();
+  if (!progressState) {
+    const candidates = await Promise.all(
+      identities.map((visitorId) =>
+        ctx.db
+          .query("funnelProgress")
+          .withIndex("by_siteId_and_funnelId_and_visitorId", (range) =>
+            range
+              .eq("siteId", siteId)
+              .eq("funnelId", funnel._id)
+              .eq("visitorId", visitorId),
+          )
+          .unique(),
+      ),
+    );
+    // Reuse the furthest attempt still inside its window. Already-counted
+    // attempts are not replayed or merged into historical aggregates.
+    const current =
+      candidates
+        .filter((row) => row !== null)
+        .sort((a, b) => {
+          const activeA =
+            event.timestamp - a.startedAt <= funnel.conversionWindowMs;
+          const activeB =
+            event.timestamp - b.startedAt <= funnel.conversionWindowMs;
+          return (
+            Number(activeB) - Number(activeA) ||
+            (activeA ? b.currentStep - a.currentStep : 0) ||
+            b.lastStepAt - a.lastStepAt
+          );
+        })[0] ?? null;
     progressState = { current, dirty: false };
-    batchState.funnelProgress.set(cacheKey, progressState);
+    batchState?.funnelProgress.set(cacheKey, progressState);
   }
-  const progress = progressState
-    ? progressState.current
-    : await ctx.db
-        .query("funnelProgress")
-        .withIndex("by_siteId_and_funnelId_and_visitorId", (range) =>
-          range
-            .eq("siteId", siteId)
-            .eq("funnelId", funnel._id)
-            .eq("visitorId", event.visitorId),
-        )
-        .unique();
+  const progress = progressState.current;
   const matchesFirst = stepMatches(funnel.steps[0], event);
   if (!progress) {
     if (!matchesFirst) return;
     const fields: Omit<Doc<"funnelProgress">, "_id" | "_creationTime"> = {
       siteId,
       funnelId: funnel._id,
-      visitorId: event.visitorId,
+      visitorId: identities[0],
       sessionId: event.sessionId,
       currentStep: 1,
       startedAt: event.timestamp,
       lastStepAt: event.timestamp,
     };
     const progressId = await ctx.db.insert("funnelProgress", fields);
-    if (progressState) {
+    if (batchState) {
       progressState.current = {
         ...fields,
         _id: progressId,
@@ -1997,7 +2020,7 @@ async function advanceFunnel(
       lastStepAt: event.timestamp,
     };
     delete updated.completedAt;
-    if (progressState) {
+    if (batchState) {
       progressState.current = updated;
       progressState.dirty = true;
     } else {
@@ -2031,7 +2054,7 @@ async function advanceFunnel(
     lastStepAt: event.timestamp,
     ...(completed ? { completedAt: event.timestamp } : {}),
   };
-  if (progressState) {
+  if (batchState) {
     progressState.current = { ...progress, ...updates };
     progressState.dirty = true;
   } else {
@@ -2388,8 +2411,9 @@ function foldRouteRollups(
   event: TrackerEvent,
   session: SessionState,
   existing: CachedDocument<"sessions"> | null,
+  patterns?: string[],
 ) {
-  const route = normalizeRoute(event.path);
+  const route = normalizeRoute(event.path, patterns);
   const priorPageviews = existing?.pageviewCount ?? 0;
   const firstPageview = existing === null || priorPageviews === 0;
   foldRouteDelta(
@@ -2407,7 +2431,7 @@ function foldRouteRollups(
   // Out-of-order telemetry keeps its view but cannot say where it came from.
   if (existing === null || firstPageview) return;
   if (event.timestamp < existing.lastSeenAt) return;
-  const previous = normalizeRoute(existing.exitPath);
+  const previous = normalizeRoute(existing.exitPath, patterns);
   const lastPageviewAt = existing.lastPageviewAt;
   foldRouteDelta(
     deltas,
@@ -2427,7 +2451,7 @@ function foldRouteRollups(
     foldRouteDelta(
       deltas,
       siteId,
-      normalizeRoute(existing.entryPath),
+      normalizeRoute(existing.entryPath, patterns),
       existing.startedAt,
       event.visitorId,
       (delta) => {

package/src/component/people.ts

@@ -0,0 +1,321 @@
+import { v } from "convex/values";
+import { query, mutation, type MutationCtx } from "./_generated/server.js";
+import type { Doc, Id } from "./_generated/dataModel.js";
+import { fail } from "./errors.js";
+import {
+  personFilterValidator,
+  visitorProfileValidator,
+  type PersonAttributes,
+  type PersonFilter,
+} from "./validators.js";
+
+export const personDocumentValidator = visitorProfileValidator.extend({
+  _id: v.id("visitorProfiles"),
+  _creationTime: v.number(),
+  siteId: v.id("sites"),
+});
+
+export function profileSearchFields(name?: string, email?: string) {
+  return {
+    searchName: normalizeSearch(name ?? ""),
+    searchEmail: normalizeSearch(email ?? ""),
+  };
+}
+
+function normalizeSearch(value: string) {
+  return value.normalize("NFKC").trim().toLowerCase();
+}
+
+export function mergePersonAttributes(
+  previous: PersonAttributes = {},
+  patch?: Record<string, string | number | boolean | null>,
+): PersonAttributes {
+  const attributes = { ...previous };
+  for (const [key, value] of Object.entries(patch ?? {})) {
+    validateAttributeKey(key);
+    if (value === null) delete attributes[key];
+    else {
+      validateAttributeValue(value);
+      attributes[key] = value;
+    }
+  }
+  if (
+    Object.keys(attributes).length > 32 ||
+    new TextEncoder().encode(JSON.stringify(attributes)).length > 4096
+  )
+    fail(
+      "INVALID_ARGUMENT",
+      "attributes allow at most 32 keys and 4096 UTF-8 bytes",
+    );
+  return Object.fromEntries(
+    Object.entries(attributes).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)),
+  );
+}
+
+function validateAttributeKey(key: string) {
+  if (
+    !/^[A-Za-z][A-Za-z0-9_.-]{0,63}$/.test(key) ||
+    ["__proto__", "constructor", "prototype"].includes(key)
+  )
+    fail(
+      "INVALID_ARGUMENT",
+      "attribute keys must start with a letter and contain up to 64 ASCII letters, digits, dots, underscores or hyphens",
+    );
+}
+
+function validateAttributeValue(value: string | number | boolean) {
+  if (
+    (typeof value === "string" && value.length > 512) ||
+    (typeof value === "number" && !Number.isFinite(value))
+  )
+    fail(
+      "INVALID_ARGUMENT",
+      "attribute values require finite numbers or strings up to 512 characters",
+    );
+}
+
+/** Index each current attribute once; ordinary reads never scan all profiles. */
+export async function syncPersonAttributes(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  visitorId: string,
+  attributes: PersonAttributes,
+) {
+  const rows = await ctx.db
+    .query("visitorAttributes")
+    .withIndex("by_siteId_and_visitorId", (q) =>
+      q.eq("siteId", siteId).eq("visitorId", visitorId),
+    )
+    .take(33);
+  const present = new Set<string>();
+  for (const row of rows) {
+    present.add(row.key);
+    if (!Object.hasOwn(attributes, row.key))
+      await ctx.db.delete("visitorAttributes", row._id);
+    else if (attributes[row.key] !== row.value)
+      await ctx.db.patch("visitorAttributes", row._id, {
+        value: attributes[row.key],
+      });
+  }
+  for (const [key, value] of Object.entries(attributes)) {
+    if (!present.has(key))
+      await ctx.db.insert("visitorAttributes", {
+        siteId,
+        visitorId,
+        key,
+        value,
+      });
+  }
+}
+
+export function publicPerson(row: Doc<"visitorProfiles">) {
+  const { searchName: _name, searchEmail: _email, ...person } = row;
+  return person;
+}
+
+/** An optional, bounded upgrade for profiles created before the directory. */
+export const backfillSearch = mutation({
+  args: { siteId: v.id("sites") },
+  returns: v.object({ updated: v.number(), isDone: v.boolean() }),
+  handler: async (ctx, { siteId }) => {
+    if (!(await ctx.db.get("sites", siteId)))
+      fail("NOT_FOUND", "site not found");
+    const rows = await ctx.db
+      .query("visitorProfiles")
+      .withIndex("by_siteId_and_searchName_and_visitorId", (q) =>
+        q.eq("siteId", siteId).eq("searchName", undefined),
+      )
+      .take(65);
+    for (const row of rows.slice(0, 64)) {
+      await ctx.db.patch(
+        "visitorProfiles",
+        row._id,
+        profileSearchFields(row.name, row.email),
+      );
+      await syncPersonAttributes(
+        ctx,
+        siteId,
+        row.visitorId,
+        row.attributes ?? {},
+      );
+    }
+    return { updated: Math.min(64, rows.length), isDone: rows.length <= 64 };
+  },
+});
+
+/** Exclusive upper bound for a Unicode prefix, including supplementary characters. */
+function afterPrefix(prefix: string) {
+  const points = Array.from(prefix);
+  for (let i = points.length - 1; i >= 0; i--) {
+    const code = points[i].codePointAt(0)!;
+    if (code < 0x10ffff)
+      return (
+        points.slice(0, i).join("") +
+        String.fromCodePoint(code + 1 === 0xd800 ? 0xe000 : code + 1)
+      );
+  }
+  return undefined;
+}
+
+function normalizedFilter(filter?: PersonFilter): PersonFilter | undefined {
+  if (!filter) return undefined;
+  if (filter.field === "attribute") {
+    validateAttributeKey(filter.key);
+    validateAttributeValue(filter.value);
+    return { field: "attribute", key: filter.key, value: filter.value };
+  }
+  if (filter.prefix.length > 320)
+    fail("INVALID_ARGUMENT", "search prefix exceeds 320 characters");
+  const prefix =
+    filter.field === "visitorId"
+      ? filter.prefix.trim()
+      : normalizeSearch(filter.prefix);
+  return prefix ? { field: filter.field, prefix } : undefined;
+}
+
+export const list = query({
+  args: {
+    siteId: v.id("sites"),
+    filter: v.optional(personFilterValidator),
+    paginationOpts: v.object({
+      cursor: v.union(v.string(), v.null()),
+      numItems: v.number(),
+    }),
+  },
+  returns: v.object({
+    page: v.array(personDocumentValidator),
+    isDone: v.boolean(),
+    continueCursor: v.string(),
+  }),
+  handler: async (ctx, args) => {
+    const filter = normalizedFilter(args.filter);
+    const scope = JSON.stringify([args.siteId, filter ?? null]);
+    const limit = args.paginationOpts.numItems;
+    if (!Number.isSafeInteger(limit) || limit < 1 || limit > 50)
+      fail("INVALID_ARGUMENT", "numItems must be an integer between 1 and 50");
+    let after: { value: string; visitorId: string } | undefined;
+    if (args.paginationOpts.cursor !== null) {
+      try {
+        // NFKC may expand one character into several, in both scope and value.
+        if (args.paginationOpts.cursor.length > 16_384) throw new Error();
+        const cursor = JSON.parse(args.paginationOpts.cursor) as Record<
+          string,
+          unknown
+        >;
+        if (
+          cursor.v !== 1 ||
+          cursor.scope !== scope ||
+          typeof cursor.value !== "string" ||
+          typeof cursor.visitorId !== "string"
+        )
+          throw new Error();
+        after = { value: cursor.value, visitorId: cursor.visitorId };
+        if (
+          filter &&
+          filter.field !== "attribute" &&
+          !after.value.startsWith(filter.prefix)
+        )
+          throw new Error();
+      } catch {
+        fail("INVALID_ARGUMENT", "cursor does not belong to this people query");
+      }
+    }
+    let rows: Doc<"visitorProfiles">[];
+    const take = limit + 1;
+    if (filter?.field === "attribute") {
+      const matches = await ctx.db
+        .query("visitorAttributes")
+        .withIndex("by_siteId_and_key_and_value_and_visitorId", (q) =>
+          q
+            .eq("siteId", args.siteId)
+            .eq("key", filter.key)
+            .eq("value", filter.value)
+            .gt("visitorId", after?.visitorId ?? ""),
+        )
+        .take(take);
+      rows = await Promise.all(
+        matches.map(async (match) => {
+          const row = await ctx.db
+            .query("visitorProfiles")
+            .withIndex("by_siteId_and_visitorId", (q) =>
+              q.eq("siteId", args.siteId).eq("visitorId", match.visitorId),
+            )
+            .unique();
+          if (!row) fail("NOT_FOUND", "person attribute index has no profile");
+          return row;
+        }),
+      );
+    } else if (filter && filter.field !== "visitorId") {
+      const pending = await ctx.db
+        .query("visitorProfiles")
+        .withIndex("by_siteId_and_searchName_and_visitorId", (q) =>
+          q.eq("siteId", args.siteId).eq("searchName", undefined),
+        )
+        .first();
+      if (pending)
+        fail(
+          "CONFLICT",
+          "Run backfillVisitorProfiles for this site before searching existing profiles",
+        );
+      const field = filter.field === "name" ? "searchName" : "searchEmail";
+      const index =
+        filter.field === "name"
+          ? "by_siteId_and_searchName_and_visitorId"
+          : "by_siteId_and_searchEmail_and_visitorId";
+      rows = after
+        ? await ctx.db
+            .query("visitorProfiles")
+            .withIndex(index, (q) =>
+              q
+                .eq("siteId", args.siteId)
+                .eq(field, after.value)
+                .gt("visitorId", after.visitorId),
+            )
+            .take(take)
+        : [];
+      if (rows.length < take) {
+        const upper = afterPrefix(filter.prefix);
+        const rest = await ctx.db
+          .query("visitorProfiles")
+          .withIndex(index, (q) => {
+            const lower = after
+              ? q.eq("siteId", args.siteId).gt(field, after.value)
+              : q.eq("siteId", args.siteId).gte(field, filter.prefix);
+            return upper === undefined ? lower : lower.lt(field, upper);
+          })
+          .take(take - rows.length);
+        rows.push(...rest);
+      }
+    } else {
+      const prefix = filter?.prefix ?? "";
+      const upper = afterPrefix(prefix);
+      rows = await ctx.db
+        .query("visitorProfiles")
+        .withIndex("by_siteId_and_visitorId", (q) => {
+          const lower = after
+            ? q.eq("siteId", args.siteId).gt("visitorId", after.visitorId)
+            : q.eq("siteId", args.siteId).gte("visitorId", prefix);
+          return upper === undefined ? lower : lower.lt("visitorId", upper);
+        })
+        .take(take);
+    }
+    const page = rows.slice(0, limit);
+    const last = page.at(-1);
+    return {
+      page: page.map(publicPerson),
+      isDone: rows.length <= limit,
+      continueCursor: JSON.stringify({
+        v: 1,
+        scope,
+        visitorId: last?.visitorId ?? after?.visitorId ?? "",
+        value: last
+          ? filter?.field === "name"
+            ? last.searchName
+            : filter?.field === "email"
+              ? last.searchEmail
+              : last.visitorId
+          : (after?.value ?? ""),
+      }),
+    };
+  },
+});