@iann29/rastro 0.5.0 → 0.6.0

package/src/component/people.ts

@@ -0,0 +1,321 @@
+import { v } from "convex/values";
+import { query, mutation, type MutationCtx } from "./_generated/server.js";
+import type { Doc, Id } from "./_generated/dataModel.js";
+import { fail } from "./errors.js";
+import {
+  personFilterValidator,
+  visitorProfileValidator,
+  type PersonAttributes,
+  type PersonFilter,
+} from "./validators.js";
+
+export const personDocumentValidator = visitorProfileValidator.extend({
+  _id: v.id("visitorProfiles"),
+  _creationTime: v.number(),
+  siteId: v.id("sites"),
+});
+
+export function profileSearchFields(name?: string, email?: string) {
+  return {
+    searchName: normalizeSearch(name ?? ""),
+    searchEmail: normalizeSearch(email ?? ""),
+  };
+}
+
+function normalizeSearch(value: string) {
+  return value.normalize("NFKC").trim().toLowerCase();
+}
+
+export function mergePersonAttributes(
+  previous: PersonAttributes = {},
+  patch?: Record<string, string | number | boolean | null>,
+): PersonAttributes {
+  const attributes = { ...previous };
+  for (const [key, value] of Object.entries(patch ?? {})) {
+    validateAttributeKey(key);
+    if (value === null) delete attributes[key];
+    else {
+      validateAttributeValue(value);
+      attributes[key] = value;
+    }
+  }
+  if (
+    Object.keys(attributes).length > 32 ||
+    new TextEncoder().encode(JSON.stringify(attributes)).length > 4096
+  )
+    fail(
+      "INVALID_ARGUMENT",
+      "attributes allow at most 32 keys and 4096 UTF-8 bytes",
+    );
+  return Object.fromEntries(
+    Object.entries(attributes).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)),
+  );
+}
+
+function validateAttributeKey(key: string) {
+  if (
+    !/^[A-Za-z][A-Za-z0-9_.-]{0,63}$/.test(key) ||
+    ["__proto__", "constructor", "prototype"].includes(key)
+  )
+    fail(
+      "INVALID_ARGUMENT",
+      "attribute keys must start with a letter and contain up to 64 ASCII letters, digits, dots, underscores or hyphens",
+    );
+}
+
+function validateAttributeValue(value: string | number | boolean) {
+  if (
+    (typeof value === "string" && value.length > 512) ||
+    (typeof value === "number" && !Number.isFinite(value))
+  )
+    fail(
+      "INVALID_ARGUMENT",
+      "attribute values require finite numbers or strings up to 512 characters",
+    );
+}
+
+/** Index each current attribute once; ordinary reads never scan all profiles. */
+export async function syncPersonAttributes(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  visitorId: string,
+  attributes: PersonAttributes,
+) {
+  const rows = await ctx.db
+    .query("visitorAttributes")
+    .withIndex("by_siteId_and_visitorId", (q) =>
+      q.eq("siteId", siteId).eq("visitorId", visitorId),
+    )
+    .take(33);
+  const present = new Set<string>();
+  for (const row of rows) {
+    present.add(row.key);
+    if (!Object.hasOwn(attributes, row.key))
+      await ctx.db.delete("visitorAttributes", row._id);
+    else if (attributes[row.key] !== row.value)
+      await ctx.db.patch("visitorAttributes", row._id, {
+        value: attributes[row.key],
+      });
+  }
+  for (const [key, value] of Object.entries(attributes)) {
+    if (!present.has(key))
+      await ctx.db.insert("visitorAttributes", {
+        siteId,
+        visitorId,
+        key,
+        value,
+      });
+  }
+}
+
+export function publicPerson(row: Doc<"visitorProfiles">) {
+  const { searchName: _name, searchEmail: _email, ...person } = row;
+  return person;
+}
+
+/** An optional, bounded upgrade for profiles created before the directory. */
+export const backfillSearch = mutation({
+  args: { siteId: v.id("sites") },
+  returns: v.object({ updated: v.number(), isDone: v.boolean() }),
+  handler: async (ctx, { siteId }) => {
+    if (!(await ctx.db.get("sites", siteId)))
+      fail("NOT_FOUND", "site not found");
+    const rows = await ctx.db
+      .query("visitorProfiles")
+      .withIndex("by_siteId_and_searchName_and_visitorId", (q) =>
+        q.eq("siteId", siteId).eq("searchName", undefined),
+      )
+      .take(65);
+    for (const row of rows.slice(0, 64)) {
+      await ctx.db.patch(
+        "visitorProfiles",
+        row._id,
+        profileSearchFields(row.name, row.email),
+      );
+      await syncPersonAttributes(
+        ctx,
+        siteId,
+        row.visitorId,
+        row.attributes ?? {},
+      );
+    }
+    return { updated: Math.min(64, rows.length), isDone: rows.length <= 64 };
+  },
+});
+
+/** Exclusive upper bound for a Unicode prefix, including supplementary characters. */
+function afterPrefix(prefix: string) {
+  const points = Array.from(prefix);
+  for (let i = points.length - 1; i >= 0; i--) {
+    const code = points[i].codePointAt(0)!;
+    if (code < 0x10ffff)
+      return (
+        points.slice(0, i).join("") +
+        String.fromCodePoint(code + 1 === 0xd800 ? 0xe000 : code + 1)
+      );
+  }
+  return undefined;
+}
+
+function normalizedFilter(filter?: PersonFilter): PersonFilter | undefined {
+  if (!filter) return undefined;
+  if (filter.field === "attribute") {
+    validateAttributeKey(filter.key);
+    validateAttributeValue(filter.value);
+    return { field: "attribute", key: filter.key, value: filter.value };
+  }
+  if (filter.prefix.length > 320)
+    fail("INVALID_ARGUMENT", "search prefix exceeds 320 characters");
+  const prefix =
+    filter.field === "visitorId"
+      ? filter.prefix.trim()
+      : normalizeSearch(filter.prefix);
+  return prefix ? { field: filter.field, prefix } : undefined;
+}
+
+export const list = query({
+  args: {
+    siteId: v.id("sites"),
+    filter: v.optional(personFilterValidator),
+    paginationOpts: v.object({
+      cursor: v.union(v.string(), v.null()),
+      numItems: v.number(),
+    }),
+  },
+  returns: v.object({
+    page: v.array(personDocumentValidator),
+    isDone: v.boolean(),
+    continueCursor: v.string(),
+  }),
+  handler: async (ctx, args) => {
+    const filter = normalizedFilter(args.filter);
+    const scope = JSON.stringify([args.siteId, filter ?? null]);
+    const limit = args.paginationOpts.numItems;
+    if (!Number.isSafeInteger(limit) || limit < 1 || limit > 50)
+      fail("INVALID_ARGUMENT", "numItems must be an integer between 1 and 50");
+    let after: { value: string; visitorId: string } | undefined;
+    if (args.paginationOpts.cursor !== null) {
+      try {
+        // NFKC may expand one character into several, in both scope and value.
+        if (args.paginationOpts.cursor.length > 16_384) throw new Error();
+        const cursor = JSON.parse(args.paginationOpts.cursor) as Record<
+          string,
+          unknown
+        >;
+        if (
+          cursor.v !== 1 ||
+          cursor.scope !== scope ||
+          typeof cursor.value !== "string" ||
+          typeof cursor.visitorId !== "string"
+        )
+          throw new Error();
+        after = { value: cursor.value, visitorId: cursor.visitorId };
+        if (
+          filter &&
+          filter.field !== "attribute" &&
+          !after.value.startsWith(filter.prefix)
+        )
+          throw new Error();
+      } catch {
+        fail("INVALID_ARGUMENT", "cursor does not belong to this people query");
+      }
+    }
+    let rows: Doc<"visitorProfiles">[];
+    const take = limit + 1;
+    if (filter?.field === "attribute") {
+      const matches = await ctx.db
+        .query("visitorAttributes")
+        .withIndex("by_siteId_and_key_and_value_and_visitorId", (q) =>
+          q
+            .eq("siteId", args.siteId)
+            .eq("key", filter.key)
+            .eq("value", filter.value)
+            .gt("visitorId", after?.visitorId ?? ""),
+        )
+        .take(take);
+      rows = await Promise.all(
+        matches.map(async (match) => {
+          const row = await ctx.db
+            .query("visitorProfiles")
+            .withIndex("by_siteId_and_visitorId", (q) =>
+              q.eq("siteId", args.siteId).eq("visitorId", match.visitorId),
+            )
+            .unique();
+          if (!row) fail("NOT_FOUND", "person attribute index has no profile");
+          return row;
+        }),
+      );
+    } else if (filter && filter.field !== "visitorId") {
+      const pending = await ctx.db
+        .query("visitorProfiles")
+        .withIndex("by_siteId_and_searchName_and_visitorId", (q) =>
+          q.eq("siteId", args.siteId).eq("searchName", undefined),
+        )
+        .first();
+      if (pending)
+        fail(
+          "CONFLICT",
+          "Run backfillVisitorProfiles for this site before searching existing profiles",
+        );
+      const field = filter.field === "name" ? "searchName" : "searchEmail";
+      const index =
+        filter.field === "name"
+          ? "by_siteId_and_searchName_and_visitorId"
+          : "by_siteId_and_searchEmail_and_visitorId";
+      rows = after
+        ? await ctx.db
+            .query("visitorProfiles")
+            .withIndex(index, (q) =>
+              q
+                .eq("siteId", args.siteId)
+                .eq(field, after.value)
+                .gt("visitorId", after.visitorId),
+            )
+            .take(take)
+        : [];
+      if (rows.length < take) {
+        const upper = afterPrefix(filter.prefix);
+        const rest = await ctx.db
+          .query("visitorProfiles")
+          .withIndex(index, (q) => {
+            const lower = after
+              ? q.eq("siteId", args.siteId).gt(field, after.value)
+              : q.eq("siteId", args.siteId).gte(field, filter.prefix);
+            return upper === undefined ? lower : lower.lt(field, upper);
+          })
+          .take(take - rows.length);
+        rows.push(...rest);
+      }
+    } else {
+      const prefix = filter?.prefix ?? "";
+      const upper = afterPrefix(prefix);
+      rows = await ctx.db
+        .query("visitorProfiles")
+        .withIndex("by_siteId_and_visitorId", (q) => {
+          const lower = after
+            ? q.eq("siteId", args.siteId).gt("visitorId", after.visitorId)
+            : q.eq("siteId", args.siteId).gte("visitorId", prefix);
+          return upper === undefined ? lower : lower.lt("visitorId", upper);
+        })
+        .take(take);
+    }
+    const page = rows.slice(0, limit);
+    const last = page.at(-1);
+    return {
+      page: page.map(publicPerson),
+      isDone: rows.length <= limit,
+      continueCursor: JSON.stringify({
+        v: 1,
+        scope,
+        visitorId: last?.visitorId ?? after?.visitorId ?? "",
+        value: last
+          ? filter?.field === "name"
+            ? last.searchName
+            : filter?.field === "email"
+              ? last.searchEmail
+              : last.visitorId
+          : (after?.value ?? ""),
+      }),
+    };
+  },
+});

package/src/component/reports.ts

@@ -64,8 +64,9 @@ import {
   goalFieldsValidator,
   sessionFieldsValidator,
   vitalMetricValidator,
+  visitorProfileValidator,
 } from "./validators.js";
-import { resolveVisitorIdentities } from "./visitors.js";
+import { resolveVisitorIdentities, withVisitorProfiles } from "./visitors.js";
 import {
   mergeVitalHistograms,
   VITAL_ALL,
@@ -76,6 +77,7 @@ import {
 } from "./vitals.js";
 
 const sessionDocumentValidator = sessionFieldsValidator.extend({
+  profile: v.optional(visitorProfileValidator),
   _id: v.id("sessions"),
   _creationTime: v.number(),
 });
@@ -84,6 +86,7 @@ const eventDocumentValidator = eventFieldsValidator.extend({
   _creationTime: v.number(),
 });
 const liveSessionDocumentValidator = v.object({
+  profile: v.optional(visitorProfileValidator),
   _id: v.id("liveSessions"),
   _creationTime: v.number(),
   siteId: v.id("sites"),
@@ -628,7 +631,7 @@ export const liveVisitors = query({
           .take(limit)),
       );
     }
-    return rows
+    const visible = rows
       .sort((left, right) => right.lastSeenAt - left.lastSeenAt)
       .slice(0, limit)
       .map((row) => ({
@@ -662,6 +665,7 @@ export const liveVisitors = query({
           : {}),
         ...(row.funnel ? { funnel: row.funnel } : {}),
       }));
+    return withVisitorProfiles(ctx, visible);
   },
 });
 
@@ -1003,7 +1007,9 @@ export const getSession = query({
       ...result
     } = session;
     const site = await ctx.db.get("sites", args.siteId);
-    return withOrigin(result, site?.domains ?? []);
+    return (
+      await withVisitorProfiles(ctx, [withOrigin(result, site?.domains ?? [])])
+    )[0];
   },
 });
 
@@ -1150,10 +1156,11 @@ export const listSessions = query({
         ...row
       }) => withOrigin(row, site?.domains ?? []),
     );
-    return keysetPaginationResult(publicRows, pagination, (row) => ({
+    const result = keysetPaginationResult(publicRows, pagination, (row) => ({
       timestamp: row.lastSeenAt,
       creationTime: row._creationTime,
     }));
+    return { ...result, page: await withVisitorProfiles(ctx, result.page) };
   },
 });
 

package/src/component/schema.ts

@@ -10,6 +10,8 @@ import {
   siteFieldsValidator,
   trackedLinkFieldsValidator,
   visitorSketchValidator,
+  visitorProfileValidator,
+  personAttributeValueValidator,
   vitalMetricValidator,
 } from "./validators.js";
 import {
@@ -395,6 +397,39 @@ export default defineSchema({
     createdAt: v.number(),
   }).index("by_key", ["key"]),
 
+  visitorProfiles: defineTable(
+    visitorProfileValidator.extend({
+      siteId: v.id("sites"),
+      searchName: v.optional(v.string()),
+      searchEmail: v.optional(v.string()),
+    }),
+  )
+    .index("by_siteId_and_visitorId", ["siteId", "visitorId"])
+    .index("by_siteId_and_searchName_and_visitorId", [
+      "siteId",
+      "searchName",
+      "visitorId",
+    ])
+    .index("by_siteId_and_searchEmail_and_visitorId", [
+      "siteId",
+      "searchEmail",
+      "visitorId",
+    ]),
+
+  visitorAttributes: defineTable({
+    siteId: v.id("sites"),
+    visitorId: v.string(),
+    key: v.string(),
+    value: personAttributeValueValidator,
+  })
+    .index("by_siteId_and_key_and_value_and_visitorId", [
+      "siteId",
+      "key",
+      "value",
+      "visitorId",
+    ])
+    .index("by_siteId_and_visitorId", ["siteId", "visitorId"]),
+
   visitorAliases: defineTable({
     siteId: v.id("sites"),
     visitorId: v.string(),

package/src/component/validators.ts

@@ -1,5 +1,54 @@
 import { v, type Infer } from "convex/values";
 
+export const personAttributeValueValidator = v.union(
+  v.string(),
+  v.number(),
+  v.boolean(),
+);
+export const personAttributesValidator = v.record(
+  v.string(),
+  personAttributeValueValidator,
+);
+export type PersonAttributes = Infer<typeof personAttributesValidator>;
+export const personFilterValidator = v.union(
+  v.object({
+    field: v.union(
+      v.literal("name"),
+      v.literal("email"),
+      v.literal("visitorId"),
+    ),
+    prefix: v.string(),
+  }),
+  v.object({
+    field: v.literal("attribute"),
+    key: v.string(),
+    value: personAttributeValueValidator,
+  }),
+);
+export type PersonFilter = Infer<typeof personFilterValidator>;
+
+/** A profile supplied by the product's trusted backend, shared by its visits. */
+export const visitorProfileValidator = v.object({
+  visitorId: v.string(),
+  name: v.optional(v.string()),
+  email: v.optional(v.string()),
+  attributes: v.optional(personAttributesValidator),
+  identifiedAt: v.number(),
+  updatedAt: v.number(),
+});
+export type VisitorProfile = Infer<typeof visitorProfileValidator>;
+
+export const identifyVisitorFields = {
+  visitorId: v.string(),
+  previousVisitorId: v.optional(v.string()),
+  // Omission preserves the value; null explicitly removes it.
+  name: v.optional(v.union(v.string(), v.null())),
+  email: v.optional(v.union(v.string(), v.null())),
+  attributes: v.optional(
+    v.record(v.string(), v.union(personAttributeValueValidator, v.null())),
+  ),
+};
+
 export const eventTypeValidator = v.union(
   v.literal("pageview"),
   v.literal("click"),