@missionmind/sync-api 0.0.0-stage → 0.1.0

package/src/pull.ts

@@ -0,0 +1,319 @@
+import {
+  and,
+  asc,
+  eq,
+  getTableColumns,
+  inArray,
+  type SQL,
+  sql,
+} from "drizzle-orm";
+import type { PgColumn, PgDatabase, PgTable } from "drizzle-orm/pg-core";
+import z from "zod";
+import { syncTombstones } from "./schema.js";
+
+/** Drizzle on Postgres, whatever the driver. */
+// biome-ignore lint/suspicious/noExplicitAny: any driver and schema
+export type SyncDatabase = PgDatabase<any, any, any>;
+
+/**
+ * A table the app pulls by cursor: the user's rows, with a string `id`, a
+ * `userId` and an `updatedAt` kept current by the sync triggers
+ * (`syncTriggersSql`). Give it a (userId, updatedAt, id) index.
+ */
+export type PulledServerTable = PgTable & {
+  id: PgColumn;
+  updatedAt: PgColumn;
+  userId: PgColumn;
+};
+
+/** A row as a pull sends it: without its `userId`. */
+export type PulledRow<TTable extends PulledServerTable> = Omit<
+  TTable["$inferSelect"],
+  "userId"
+>;
+
+/** A page of the changes made to a table since a cursor. */
+export interface ChangesPage<TRow> {
+  /** Where the next pull of the table starts. */
+  cursor: string;
+  deletedIds: string[];
+  /** Whether another page is ready: ask again with `cursor`. */
+  hasMore: boolean;
+  rows: TRow[];
+}
+
+/** The user's rows of a small table, sent whole on every pull. */
+export type WholeTableReader = (userId: string) => Promise<unknown[]>;
+
+export interface SyncPullOptions<
+  TPulled extends Record<string, PulledServerTable>,
+  TWhole extends Record<string, WholeTableReader>,
+> {
+  db: SyncDatabase;
+  /** Rows per table and page. Defaults to 500. */
+  pageSize?: number;
+  /** Tables pulled by cursor, under the names the app asks for them by. */
+  pulled: TPulled;
+  /**
+   * How far behind "now" a cursor stays, in seconds. Defaults to 60: longer
+   * than any transaction that writes the pulled tables.
+   */
+  safetyMarginSeconds?: number;
+  /** Small tables the app fetches whole on every pull, with their readers. */
+  whole: TWhole;
+}
+
+/**
+ * Position in the two streams of a table: rows ordered by (updatedAt, id),
+ * tombstones by (deletedAt, id). Timestamps are kept as text with their
+ * microseconds, which JavaScript dates would drop and stall the cursor.
+ */
+const cursorSchema = z.object({
+  rows: z.tuple([z.string(), z.string()]).nullable(),
+  tombstones: z.tuple([z.string(), z.number()]).nullable(),
+});
+type Cursor = z.infer<typeof cursorSchema>;
+
+function decodeCursor(cursor: string | null): Cursor {
+  if (!cursor) {
+    return { rows: null, tombstones: null };
+  }
+  const parsed = cursorSchema.safeParse(
+    JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"))
+  );
+  if (!parsed.success) {
+    throw new Error("SYNC_ERROR.INVALID_CURSOR");
+  }
+  return parsed.data;
+}
+
+function encodeCursor(cursor: Cursor) {
+  return Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url");
+}
+
+/** Fixed-width UTC text: compares like the timestamp it encodes. */
+const cursorTime = (value: SQL | SQL.Aliased | unknown) =>
+  sql<string>`to_char((${value}) at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"')`;
+
+/** The rows of a raw query, whether the driver returns them alone or in `rows`. */
+function rowsOf<T>(result: unknown): T[] {
+  return (
+    Array.isArray(result) ? result : (result as { rows: T[] }).rows
+  ) as T[];
+}
+
+/**
+ * What the app's sync fetches: the small tables whole, and the others a page
+ * at a time from a cursor, with the ids deleted since (the tombstones the
+ * sync triggers write).
+ */
+export function createSyncPull<
+  TPulled extends Record<string, PulledServerTable>,
+  TWhole extends Record<string, WholeTableReader>,
+>(options: SyncPullOptions<TPulled, TWhole>) {
+  type PulledName = Extract<keyof TPulled, string>;
+  type WholeName = Extract<keyof TWhole, string>;
+
+  const { db, pulled } = options;
+  const pageSize = options.pageSize ?? 500;
+  /**
+   * A row is stamped with its transaction's start time but only visible once
+   * committed, so a transaction still running when a page is read may commit
+   * rows older than that page's newest one. Staying behind re-sends the last
+   * minute of changes on every pull (the app applies them idempotently)
+   * instead of missing those rows.
+   */
+  const safetyMargin = sql.raw(
+    `interval '${Math.floor(options.safetyMarginSeconds ?? 60)} seconds'`
+  );
+
+  const pulledTableSchema = z.enum(
+    Object.keys(pulled) as [PulledName, ...PulledName[]]
+  );
+  const wholeTableSchema = z.enum(
+    Object.keys(options.whole) as [WholeName, ...WholeName[]]
+  );
+
+  /** The user's rows among `ids`, as `pull` returns them. */
+  async function getRows<TName extends PulledName>(
+    userId: string,
+    tableName: TName,
+    ids: string[]
+  ): Promise<PulledRow<TPulled[TName]>[]> {
+    if (ids.length === 0) {
+      return [];
+    }
+    const table: PulledServerTable = pulled[tableName];
+    const rows = (await db
+      .select()
+      .from(table)
+      .where(and(eq(table.userId, userId), inArray(table.id, ids)))) as Record<
+      string,
+      unknown
+    >[];
+    return rows.map(
+      ({ userId: _user, ...row }) => row as PulledRow<TPulled[TName]>
+    );
+  }
+
+  /** One page of the changes to `tableName` since `cursor`. */
+  async function pullChanges<TName extends PulledName>(
+    userId: string,
+    tableName: TName,
+    cursor: string | null,
+    safeUntil: string
+  ): Promise<ChangesPage<PulledRow<TPulled[TName]>>> {
+    const table: PulledServerTable = pulled[tableName];
+    const position = decodeCursor(cursor);
+
+    const rows = (await db
+      .select({
+        ...getTableColumns(table),
+        cursorAt: cursorTime(table.updatedAt),
+        isRecent: sql<boolean>`${table.updatedAt} > ${safeUntil}::timestamptz`,
+      })
+      .from(table)
+      .where(
+        and(
+          eq(table.userId, userId),
+          position.rows
+            ? sql`(${table.updatedAt}, ${table.id}) > (${position.rows[0]}::timestamptz, ${position.rows[1]})`
+            : undefined
+        )
+      )
+      .orderBy(asc(table.updatedAt), asc(table.id))
+      .limit(pageSize)) as (Record<string, unknown> & {
+      cursorAt: string;
+      id: string;
+      isRecent: boolean;
+    })[];
+
+    const tombstones = await db
+      .select({
+        id: syncTombstones.id,
+        rowId: syncTombstones.rowId,
+        cursorAt: cursorTime(syncTombstones.deletedAt),
+        isRecent: sql<boolean>`${syncTombstones.deletedAt} > ${safeUntil}::timestamptz`,
+      })
+      .from(syncTombstones)
+      .where(
+        and(
+          eq(syncTombstones.userId, userId),
+          eq(syncTombstones.tableName, tableName),
+          position.tombstones
+            ? sql`(${syncTombstones.deletedAt}, ${syncTombstones.id}) > (${position.tombstones[0]}::timestamptz, ${position.tombstones[1]})`
+            : undefined
+        )
+      )
+      .orderBy(asc(syncTombstones.deletedAt), asc(syncTombstones.id))
+      .limit(pageSize);
+
+    const rowsStream = advance(
+      rows.map((row) => ({
+        position: [row.cursorAt, row.id] as [string, string],
+        isRecent: row.isRecent,
+      })),
+      position.rows,
+      [safeUntil, ""],
+      pageSize
+    );
+    const tombstonesStream = advance(
+      tombstones.map((row) => ({
+        position: [row.cursorAt, row.id] as [string, number],
+        isRecent: row.isRecent,
+      })),
+      position.tombstones,
+      [safeUntil, 0],
+      pageSize
+    );
+
+    return {
+      rows: rows.map(
+        ({ cursorAt: _at, isRecent: _recent, userId: _user, ...row }) =>
+          row as PulledRow<TPulled[TName]>
+      ),
+      deletedIds: tombstones.map((tombstone) => tombstone.rowId),
+      cursor: encodeCursor({
+        rows: rowsStream.position,
+        tombstones: tombstonesStream.position,
+      }),
+      hasMore: rowsStream.hasMore || tombstonesStream.hasMore,
+    };
+  }
+
+  /**
+   * Everything a sync fetches, in one request: the `whole` tables in full,
+   * and a page of the changes to each table of `changes` since its cursor (a
+   * null cursor starts from the beginning). The app calls again, for the
+   * tables whose page `hasMore`, with the returned cursors.
+   */
+  async function pull(
+    userId: string,
+    request: {
+      changes: Partial<Record<PulledName, string | null>>;
+      whole: WholeName[];
+    }
+  ) {
+    // One reference time for every table of the request.
+    const [{ safeUntil }] = rowsOf<{ safeUntil: string }>(
+      await db.execute(
+        sql`select ${cursorTime(sql`now() - ${safetyMargin}`)} as "safeUntil"`
+      )
+    );
+
+    const changes: {
+      [TName in PulledName]?: ChangesPage<PulledRow<TPulled[TName]>>;
+    } = {};
+    for (const [tableName, cursor] of Object.entries(request.changes) as [
+      PulledName,
+      string | null,
+    ][]) {
+      changes[tableName] = await pullChanges(
+        userId,
+        tableName,
+        cursor,
+        safeUntil
+      );
+    }
+
+    const requested = new Set<string>(request.whole);
+    const whole: {
+      [TName in WholeName]?: Awaited<ReturnType<TWhole[TName]>>;
+    } = {};
+    for (const [tableName, read] of Object.entries(options.whole) as [
+      WholeName,
+      WholeTableReader,
+    ][]) {
+      if (requested.has(tableName)) {
+        whole[tableName] = (await read(userId)) as Awaited<
+          ReturnType<TWhole[typeof tableName]>
+        >;
+      }
+    }
+    return { changes, whole };
+  }
+
+  return { getRows, pull, pulledTableSchema, wholeTableSchema };
+}
+
+/**
+ * Where a stream resumes: after its last item, unless that item is inside the
+ * safety margin, in which case at the margin (so the recent items come again).
+ */
+function advance<T extends [string, string | number]>(
+  items: { position: T; isRecent: boolean }[],
+  previous: T | null,
+  margin: T,
+  pageSize: number
+): { position: T | null; hasMore: boolean } {
+  const last = items.at(-1);
+  if (!last) {
+    return { position: previous, hasMore: false };
+  }
+  if (last.isRecent) {
+    // Never move backwards past where a previous pull already got to.
+    const isAheadOfPrevious = !previous || margin[0] > previous[0];
+    return { position: isAheadOfPrevious ? margin : previous, hasMore: false };
+  }
+  return { position: last.position, hasMore: items.length === pageSize };
+}

package/src/push.ts

@@ -0,0 +1,36 @@
+/**
+ * A sync push: a headless background notification (data only, no title or
+ * body, `content-available` on iOS) telling the user's device that these
+ * tables changed on the server, so it pulls them now rather than at its next
+ * sync. `@missionmind/sync-front/push` handles it on the device, with the same
+ * `type`.
+ */
+export interface SyncPushMessage {
+  _contentAvailable: true;
+  data: { tables: string[]; type: string };
+  priority: "high";
+  to: string;
+  ttl: number;
+}
+
+/** The Expo push message of a sync push, for Expo's push API. */
+export function createSyncPushMessage(options: {
+  tables: string[];
+  /** The Expo push token of the device. */
+  to: string;
+  /** Tells a sync push from the app's other notifications, e.g. "myapp.sync". */
+  type: string;
+  /**
+   * Seconds the push service keeps trying to deliver it. Defaults to an hour:
+   * a sync asked for long ago is no longer worth waking the app for.
+   */
+  ttl?: number;
+}): SyncPushMessage {
+  return {
+    to: options.to,
+    data: { type: options.type, tables: options.tables },
+    _contentAvailable: true,
+    priority: "high",
+    ttl: options.ttl ?? 60 * 60,
+  };
+}

package/src/schema.ts

@@ -0,0 +1,33 @@
+import { sql } from "drizzle-orm";
+import {
+  bigserial,
+  index,
+  pgTable,
+  timestamp,
+  varchar,
+} from "drizzle-orm/pg-core";
+
+/**
+ * Rows deleted from the tables the app pulls by cursor, written by a trigger
+ * (`syncTriggersSql`) so that cascades are recorded too. No foreign key to
+ * the users table, which is the app's: the trigger writes none for a row
+ * deleted with its account.
+ */
+export const syncTombstones = pgTable(
+  "sync_tombstones",
+  {
+    id: bigserial({ mode: "number" }).primaryKey(),
+    userId: varchar({ length: 255 }).notNull(),
+    tableName: varchar({ length: 64 }).notNull(),
+    rowId: varchar({ length: 255 }).notNull(),
+    deletedAt: timestamp({ withTimezone: true }).notNull().default(sql`now()`),
+  },
+  (table) => [
+    index("syncTombstones_sync_idx").on(
+      table.userId,
+      table.tableName,
+      table.deletedAt,
+      table.id
+    ),
+  ]
+);

package/src/triggers.ts

@@ -0,0 +1,80 @@
+/**
+ * The SQL that keeps the tables an app pulls by cursor ready for it: each
+ * write stamps the row's "updatedAt", and each deletion records a tombstone
+ * in `sync_tombstones`. Run it in a migration, after the tables and
+ * `sync_tombstones` exist, and again whenever a pulled table is added; it can
+ * be run more than once.
+ *
+ * Inserts are stamped too: a row restored or imported with an old
+ * "updatedAt" would otherwise sit behind every cursor.
+ *
+ * Triggers rather than application code, so that every write counts: foreign
+ * keys' ON DELETE SET NULL / CASCADE, scripts, and upserts whose conflict
+ * branch Drizzle does not stamp.
+ */
+export function syncTriggersSql(options: {
+  /** The pulled tables, by their SQL names. */
+  tables: string[];
+  /**
+   * The table of users, by its SQL name (Better Auth's is "user"). A row
+   * deleted with its user leaves no tombstone: nobody will pull it.
+   */
+  usersTable: string;
+}) {
+  const tables = options.tables.map((table) => quoteLiteral(table)).join(", ");
+  const usersTable = quoteIdentifier(options.usersTable);
+  return `BEGIN;
+
+-- A row inserted again under a deleted id (a restore) must not be deleted
+-- again by its old tombstone on a device pulling both for the first time.
+CREATE OR REPLACE FUNCTION sync_touch_updated_at() RETURNS trigger AS $$
+BEGIN
+  NEW."updatedAt" := now();
+  IF TG_OP = 'INSERT' THEN
+    DELETE FROM sync_tombstones
+    WHERE "tableName" = TG_TABLE_NAME AND "rowId" = NEW.id;
+  END IF;
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Rows removed with their user leave no tombstone: nobody will pull them.
+CREATE OR REPLACE FUNCTION sync_record_tombstone() RETURNS trigger AS $$
+BEGIN
+  IF EXISTS (SELECT 1 FROM ${usersTable} WHERE id = OLD."userId") THEN
+    INSERT INTO sync_tombstones ("userId", "tableName", "rowId")
+    VALUES (OLD."userId", TG_TABLE_NAME, OLD.id);
+  END IF;
+  RETURN OLD;
+END;
+$$ LANGUAGE plpgsql;
+
+DO $$
+DECLARE
+  t text;
+BEGIN
+  FOREACH t IN ARRAY ARRAY[${tables}]
+  LOOP
+    EXECUTE format('DROP TRIGGER IF EXISTS sync_touch_updated_at ON %I', t);
+    EXECUTE format(
+      'CREATE TRIGGER sync_touch_updated_at BEFORE INSERT OR UPDATE ON %I
+         FOR EACH ROW EXECUTE FUNCTION sync_touch_updated_at()', t);
+    EXECUTE format('DROP TRIGGER IF EXISTS sync_record_tombstone ON %I', t);
+    EXECUTE format(
+      'CREATE TRIGGER sync_record_tombstone AFTER DELETE ON %I
+         FOR EACH ROW EXECUTE FUNCTION sync_record_tombstone()', t);
+  END LOOP;
+END;
+$$;
+
+COMMIT;
+`;
+}
+
+function quoteLiteral(value: string) {
+  return `'${value.replaceAll("'", "''")}'`;
+}
+
+function quoteIdentifier(value: string) {
+  return `"${value.replaceAll('"', '""')}"`;
+}