@fatagnus/dink-convex 1.0.0

package/convex/outbox.ts

@@ -0,0 +1,198 @@
+/**
+ * Outbox processing utilities for @fatagnus/dink-convex component.
+ *
+ * Provides helper functions for processing the _sync_outbox table
+ * and pushing deltas to dinkd.
+ *
+ * @module convex/outbox
+ */
+
+/**
+ * Status constants for outbox items.
+ */
+export const OUTBOX_STATUS = {
+  PENDING: "pending",
+  SENT: "sent",
+  FAILED: "failed",
+} as const;
+
+/**
+ * Maximum number of retry attempts before marking as failed.
+ */
+export const MAX_RETRIES = 5;
+
+/**
+ * Base delay for exponential backoff (1 second).
+ */
+const BASE_DELAY_MS = 1000;
+
+/**
+ * Maximum delay cap for backoff (5 minutes).
+ */
+const MAX_DELAY_MS = 5 * 60 * 1000;
+
+/**
+ * Outbox item structure from _sync_outbox table.
+ */
+export interface OutboxItem {
+  _id: string;
+  collection: string;
+  docId: string;
+  delta: ArrayBuffer;
+  seq: number;
+  status: string;
+  retries: number;
+  createdAt: number;
+  lastAttempt?: number;
+}
+
+/**
+ * Payload structure for POSTing to dinkd /api/sync/convex-push.
+ */
+export interface PushPayload {
+  collection: string;
+  docId: string;
+  seq: number;
+  bytes: number[];
+}
+
+/**
+ * Configuration for outbox processing.
+ */
+export interface OutboxConfig {
+  dinkHttpUrl: string | undefined;
+  appSyncKey: string | undefined;
+}
+
+/**
+ * Item identifier for deduplication.
+ */
+export interface ItemIdentifier {
+  docId: string;
+  seq: number;
+}
+
+/**
+ * Calculate exponential backoff delay based on retry count.
+ *
+ * Uses the formula: min(BASE_DELAY * 2^retries, MAX_DELAY)
+ *
+ * @param retries - Number of previous retry attempts
+ * @returns Delay in milliseconds before next retry
+ */
+export function calculateBackoffDelay(retries: number): number {
+  const delay = BASE_DELAY_MS * Math.pow(2, retries);
+  return Math.min(delay, MAX_DELAY_MS);
+}
+
+/**
+ * Check if an item should be retried based on retry count.
+ *
+ * @param retries - Current number of retry attempts
+ * @returns True if retries < MAX_RETRIES
+ */
+export function shouldRetry(retries: number): boolean {
+  return retries < MAX_RETRIES;
+}
+
+/**
+ * Check if an item is a duplicate based on docId and seq.
+ *
+ * Used to prevent duplicate pushes when the same delta
+ * might be processed multiple times.
+ *
+ * @param item - Item to check
+ * @param sentItems - List of already sent items
+ * @returns True if the item is already in the sent list
+ */
+export function isDuplicate(
+  item: ItemIdentifier,
+  sentItems: ItemIdentifier[]
+): boolean {
+  return sentItems.some(
+    (sent) => sent.docId === item.docId && sent.seq === item.seq
+  );
+}
+
+/**
+ * Create the payload structure for POSTing to dinkd.
+ *
+ * Converts the delta bytes from Uint8Array to number array
+ * for JSON serialization.
+ *
+ * @param item - Outbox item with delta bytes
+ * @returns Payload ready for JSON serialization
+ */
+export function createPushPayload(item: {
+  collection: string;
+  docId: string;
+  delta: Uint8Array | ArrayBuffer;
+  seq: number;
+}): PushPayload {
+  // Convert delta to number array for JSON
+  const deltaArray =
+    item.delta instanceof Uint8Array
+      ? item.delta
+      : new Uint8Array(item.delta);
+
+  return {
+    collection: item.collection,
+    docId: item.docId,
+    seq: item.seq,
+    bytes: Array.from(deltaArray),
+  };
+}
+
+/**
+ * Get the outbox configuration from environment variables.
+ *
+ * Reads DINK_HTTP_URL and DINK_APP_SYNC_KEY from the environment.
+ * In Convex, these are set via the dashboard environment variables.
+ *
+ * @returns Configuration object with dinkHttpUrl and appSyncKey
+ */
+export function getOutboxConfig(): OutboxConfig {
+  return {
+    dinkHttpUrl: process.env.DINK_HTTP_URL,
+    appSyncKey: process.env.DINK_APP_SYNC_KEY,
+  };
+}
+
+/**
+ * Determine the new status for an outbox item based on push result.
+ *
+ * - Success: "sent"
+ * - Failure with retries remaining: "pending" (will be retried)
+ * - Failure at max retries: "failed" (permanent failure)
+ *
+ * @param success - Whether the push succeeded
+ * @param currentRetries - Current retry count
+ * @returns New status string
+ */
+export function getNewStatus(
+  success: boolean,
+  currentRetries: number
+): string {
+  if (success) {
+    return OUTBOX_STATUS.SENT;
+  }
+
+  // If we've hit max retries, mark as failed
+  if (currentRetries >= MAX_RETRIES) {
+    return OUTBOX_STATUS.FAILED;
+  }
+
+  // Otherwise, keep as pending for retry
+  return OUTBOX_STATUS.PENDING;
+}
+
+/**
+ * Build the full URL for the convex-push endpoint.
+ *
+ * @param baseUrl - Base dinkd HTTP URL
+ * @returns Full URL for the convex-push endpoint
+ */
+export function buildPushUrl(baseUrl: string): string {
+  const url = new URL("/api/sync/convex-push", baseUrl);
+  return url.toString();
+}

package/convex/outboxProcessor.ts

@@ -0,0 +1,240 @@
+/**
+ * Outbox processor for @fatagnus/dink-convex component.
+ *
+ * Processes pending items from sync_outbox and pushes them to dinkd.
+ * Called by the scheduled cron job and immediately after mutations.
+ *
+ * Re-export these in your app's convex/outboxProcessor.ts:
+ * ```typescript
+ * export {
+ *   processOutboxBatch,
+ *   queryPendingItems,
+ *   updateItemStatus,
+ *   scheduleOutboxProcessing,
+ * } from "@fatagnus/dink-convex/outbox";
+ * ```
+ *
+ * @module convex/outboxProcessor
+ */
+
+import { v } from "convex/values";
+import { internalAction, internalMutation, internalQuery } from "./_generated/server";
+import { internal } from "./_generated/api";
+
+import {
+  OUTBOX_STATUS,
+  calculateBackoffDelay,
+  isDuplicate,
+  createPushPayload,
+  getNewStatus,
+  buildPushUrl,
+} from "./outbox";
+
+/**
+ * Maximum number of items to process per batch.
+ */
+const BATCH_SIZE = 50;
+
+/**
+ * Process a batch of pending outbox items.
+ *
+ * This action:
+ * 1. Queries pending items from _sync_outbox
+ * 2. POSTs each to dinkd /api/sync/convex-push with appSyncKey
+ * 3. Updates status to 'sent' on success, handles retries on error
+ * 4. Deduplicates by (docId, seq) to prevent duplicate pushes
+ */
+export const processOutboxBatch = internalAction({
+  args: {},
+  returns: v.object({
+    processed: v.number(),
+    sent: v.number(),
+    failed: v.number(),
+    skipped: v.number(),
+  }),
+  handler: async (ctx) => {
+    // Get configuration from environment
+    const dinkHttpUrl = process.env.DINK_HTTP_URL;
+    const appSyncKey = process.env.DINK_APP_SYNC_KEY;
+
+    // If not configured, skip processing
+    if (!dinkHttpUrl || !appSyncKey) {
+      console.log(
+        "Outbox processor: DINK_HTTP_URL or DINK_APP_SYNC_KEY not configured, skipping"
+      );
+      return { processed: 0, sent: 0, failed: 0, skipped: 0 };
+    }
+
+    // Query pending items
+    const pendingItems = await ctx.runQuery(
+      internal.outboxProcessor.queryPendingItems,
+      { limit: BATCH_SIZE }
+    );
+
+    if (pendingItems.length === 0) {
+      return { processed: 0, sent: 0, failed: 0, skipped: 0 };
+    }
+
+    const pushUrl = buildPushUrl(dinkHttpUrl);
+    const sentItems: Array<{ docId: string; seq: number }> = [];
+    let sent = 0;
+    let failed = 0;
+    let skipped = 0;
+
+    for (const item of pendingItems) {
+      // Check for duplicates within this batch
+      if (isDuplicate({ docId: item.docId, seq: item.seq }, sentItems)) {
+        skipped++;
+        continue;
+      }
+
+      // Check if we should skip based on backoff timing
+      if (item.lastAttempt && item.retries > 0) {
+        const backoffDelay = calculateBackoffDelay(item.retries - 1);
+        const timeSinceLastAttempt = Date.now() - item.lastAttempt;
+        if (timeSinceLastAttempt < backoffDelay) {
+          skipped++;
+          continue;
+        }
+      }
+
+      // Create the payload
+      const payload = createPushPayload({
+        collection: item.collection,
+        docId: item.docId,
+        delta: item.delta,
+        seq: item.seq,
+      });
+
+      let success = false;
+      try {
+        // POST to dinkd
+        const response = await fetch(pushUrl, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${appSyncKey}`,
+          },
+          body: JSON.stringify(payload),
+        });
+
+        success = response.ok;
+
+        if (!success) {
+          console.error(
+            `Outbox push failed for ${item.docId}:${item.seq}: ${response.status} ${response.statusText}`
+          );
+        }
+      } catch (error) {
+        console.error(
+          `Outbox push error for ${item.docId}:${item.seq}:`,
+          error
+        );
+        success = false;
+      }
+
+      // Determine new status
+      const newRetries = success ? item.retries : item.retries + 1;
+      const newStatus = getNewStatus(success, newRetries);
+
+      // Update the item status
+      await ctx.runMutation(
+        internal.outboxProcessor.updateItemStatus,
+        {
+          itemId: item._id,
+          status: newStatus,
+          retries: newRetries,
+          lastAttempt: Date.now(),
+        }
+      );
+
+      if (success) {
+        sent++;
+        sentItems.push({ docId: item.docId, seq: item.seq });
+      } else if (newStatus === OUTBOX_STATUS.FAILED) {
+        failed++;
+      }
+    }
+
+    return {
+      processed: pendingItems.length,
+      sent,
+      failed,
+      skipped,
+    };
+  },
+});
+
+/**
+ * Query pending items from _sync_outbox.
+ * Internal query used by the processor action.
+ */
+export const queryPendingItems = internalQuery({
+  args: {
+    limit: v.number(),
+  },
+  returns: v.array(
+    v.object({
+      _id: v.id("sync_outbox"),
+      collection: v.string(),
+      docId: v.string(),
+      delta: v.bytes(),
+      seq: v.number(),
+      status: v.string(),
+      retries: v.number(),
+      createdAt: v.number(),
+      lastAttempt: v.optional(v.number()),
+    })
+  ),
+  handler: async (ctx, args) => {
+    const items = await ctx.db
+      .query("sync_outbox")
+      .withIndex("by_status", (q) => q.eq("status", OUTBOX_STATUS.PENDING))
+      .take(args.limit);
+
+    return items;
+  },
+});
+
+/**
+ * Update the status of an outbox item.
+ * Internal mutation used by the processor action.
+ */
+export const updateItemStatus = internalMutation({
+  args: {
+    itemId: v.id("sync_outbox"),
+    status: v.string(),
+    retries: v.number(),
+    lastAttempt: v.number(),
+  },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    await ctx.db.patch(args.itemId, {
+      status: args.status,
+      retries: args.retries,
+      lastAttempt: args.lastAttempt,
+    });
+    return null;
+  },
+});
+
+/**
+ * Schedule immediate outbox processing.
+ *
+ * This mutation uses ctx.scheduler.runAfter(0, ...) to trigger
+ * the processOutboxBatch action immediately after the current
+ * transaction commits. This provides near real-time sync instead
+ * of waiting for the cron job fallback.
+ *
+ * Called by trigger handlers after inserting to _sync_outbox.
+ */
+export const scheduleOutboxProcessing = internalMutation({
+  args: {},
+  returns: v.null(),
+  handler: async (ctx) => {
+    // Schedule immediate processing with 0 delay
+    // This runs right after the current transaction commits
+    await ctx.scheduler.runAfter(0, internal.outboxProcessor.processOutboxBatch, {});
+    return null;
+  },
+});

package/convex/schema.ts

@@ -0,0 +1,97 @@
+/**
+ * Internal sync tables schema for @fatagnus/dink-convex component.
+ *
+ * These tables are component-private and not visible to developers.
+ * They manage sync state, deltas, and configuration for bidirectional sync.
+ *
+ * @module convex/schema
+ */
+
+import { defineSchema, defineTable } from "convex/server";
+import { v } from "convex/values";
+
+export default defineSchema({
+  /**
+   * _sync_deltas - Stores CRDT delta updates for sync operations.
+   *
+   * Each delta represents a change to a document that needs to be
+   * synchronized with edge clients via dinkd.
+   */
+  sync_deltas: defineTable({
+    /** Collection name (user table) */
+    collection: v.string(),
+    /** Document sync ID (syncId from user table) */
+    docId: v.string(),
+    /** CRDT delta bytes (Yjs update) */
+    bytes: v.bytes(),
+    /** Sequence number for ordering */
+    seq: v.number(),
+    /** Timestamp when delta was created */
+    timestamp: v.number(),
+    /** Edge ID that originated this delta (optional) */
+    edgeId: v.optional(v.string()),
+  })
+    .index("by_collection", ["collection"])
+    .index("by_docId", ["collection", "docId"])
+    .index("by_seq", ["collection", "seq"]),
+
+  /**
+   * _sync_outbox - Queue of deltas pending push to dinkd.
+   *
+   * Deltas are queued here when triggers fire, then processed by
+   * the scheduled outbox processor which pushes them to dinkd.
+   */
+  sync_outbox: defineTable({
+    /** Collection name (user table) */
+    collection: v.string(),
+    /** Document sync ID (syncId from user table) */
+    docId: v.string(),
+    /** CRDT delta bytes to send */
+    delta: v.bytes(),
+    /** Sequence number for deduplication */
+    seq: v.number(),
+    /** Status: "pending" | "sent" | "failed" */
+    status: v.string(),
+    /** Number of retry attempts */
+    retries: v.number(),
+    /** Timestamp when item was created */
+    createdAt: v.number(),
+    /** Timestamp of last send attempt (optional) */
+    lastAttempt: v.optional(v.number()),
+  })
+    .index("by_status", ["status"])
+    .index("by_collection", ["collection"])
+    .index("by_docId_seq", ["docId", "seq"]),
+
+  /**
+   * _sync_sessions - Tracks connected sessions and their sync state.
+   *
+   * Each session represents a connected client that is syncing
+   * documents from a specific collection.
+   */
+  sync_sessions: defineTable({
+    /** Unique session identifier */
+    sessionId: v.string(),
+    /** Collection being synced */
+    collection: v.string(),
+    /** Last sequence number acknowledged by this session */
+    lastSeq: v.number(),
+    /** Timestamp when session was last active */
+    lastSeen: v.number(),
+  })
+    .index("by_sessionId", ["sessionId"])
+    .index("by_collection", ["collection"]),
+
+  /**
+   * _sync_config - Configuration for which tables have sync enabled.
+   *
+   * Triggers check this table to determine whether to process
+   * writes to a given collection.
+   */
+  sync_config: defineTable({
+    /** Collection name (user table) */
+    collection: v.string(),
+    /** Whether sync is enabled for this collection */
+    enabled: v.boolean(),
+  }).index("by_collection", ["collection"]),
+});