@tanstack/offline-transactions 0.0.0

package/src/executor/TransactionExecutor.ts

@@ -0,0 +1,330 @@
+import { createTraceState } from "@opentelemetry/api"
+import { DefaultRetryPolicy } from "../retry/RetryPolicy"
+import { NonRetriableError } from "../types"
+import { withNestedSpan } from "../telemetry/tracer"
+import type { SpanContext } from "@opentelemetry/api"
+import type { KeyScheduler } from "./KeyScheduler"
+import type { OutboxManager } from "../outbox/OutboxManager"
+import type {
+  OfflineConfig,
+  OfflineTransaction,
+  SerializedSpanContext,
+} from "../types"
+
+const HANDLED_EXECUTION_ERROR = Symbol(`HandledExecutionError`)
+
+function toSpanContext(
+  serialized?: SerializedSpanContext
+): SpanContext | undefined {
+  if (!serialized) {
+    return undefined
+  }
+
+  return {
+    traceId: serialized.traceId,
+    spanId: serialized.spanId,
+    traceFlags: serialized.traceFlags,
+    traceState: serialized.traceState
+      ? createTraceState(serialized.traceState)
+      : undefined,
+  }
+}
+
+export class TransactionExecutor {
+  private scheduler: KeyScheduler
+  private outbox: OutboxManager
+  private config: OfflineConfig
+  private retryPolicy: DefaultRetryPolicy
+  private isExecuting = false
+  private executionPromise: Promise<void> | null = null
+  private offlineExecutor: any // Reference to OfflineExecutor for signaling
+  private retryTimer: ReturnType<typeof setTimeout> | null = null
+
+  constructor(
+    scheduler: KeyScheduler,
+    outbox: OutboxManager,
+    config: OfflineConfig,
+    offlineExecutor: any
+  ) {
+    this.scheduler = scheduler
+    this.outbox = outbox
+    this.config = config
+    this.retryPolicy = new DefaultRetryPolicy(10, config.jitter ?? true)
+    this.offlineExecutor = offlineExecutor
+  }
+
+  async execute(transaction: OfflineTransaction): Promise<void> {
+    this.scheduler.schedule(transaction)
+    await this.executeAll()
+  }
+
+  async executeAll(): Promise<void> {
+    if (this.isExecuting) {
+      return this.executionPromise!
+    }
+
+    this.isExecuting = true
+    this.executionPromise = this.runExecution()
+
+    try {
+      await this.executionPromise
+    } finally {
+      this.isExecuting = false
+      this.executionPromise = null
+    }
+  }
+
+  private async runExecution(): Promise<void> {
+    const maxConcurrency = this.config.maxConcurrency ?? 3
+
+    while (this.scheduler.getPendingCount() > 0) {
+      const batch = this.scheduler.getNextBatch(maxConcurrency)
+
+      if (batch.length === 0) {
+        break
+      }
+
+      const executions = batch.map((transaction) =>
+        this.executeTransaction(transaction)
+      )
+      await Promise.allSettled(executions)
+    }
+
+    // Schedule next retry after execution completes
+    this.scheduleNextRetry()
+  }
+
+  private async executeTransaction(
+    transaction: OfflineTransaction
+  ): Promise<void> {
+    try {
+      await withNestedSpan(
+        `transaction.execute`,
+        {
+          "transaction.id": transaction.id,
+          "transaction.mutationFnName": transaction.mutationFnName,
+          "transaction.retryCount": transaction.retryCount,
+          "transaction.keyCount": transaction.keys.length,
+        },
+        async (span) => {
+          this.scheduler.markStarted(transaction)
+
+          if (transaction.retryCount > 0) {
+            span.setAttribute(`retry.attempt`, transaction.retryCount)
+          }
+
+          try {
+            const result = await this.runMutationFn(transaction)
+
+            this.scheduler.markCompleted(transaction)
+            await this.outbox.remove(transaction.id)
+
+            span.setAttribute(`result`, `success`)
+            this.offlineExecutor.resolveTransaction(transaction.id, result)
+          } catch (error) {
+            const err =
+              error instanceof Error ? error : new Error(String(error))
+
+            span.setAttribute(`result`, `error`)
+
+            await this.handleError(transaction, err)
+            ;(err as any)[HANDLED_EXECUTION_ERROR] = true
+            throw err
+          }
+        },
+        {
+          parentContext: toSpanContext(transaction.spanContext),
+        }
+      )
+    } catch (error) {
+      if (
+        error instanceof Error &&
+        (error as any)[HANDLED_EXECUTION_ERROR] === true
+      ) {
+        return
+      }
+
+      throw error
+    }
+  }
+
+  private async runMutationFn(transaction: OfflineTransaction): Promise<void> {
+    const mutationFn = this.config.mutationFns[transaction.mutationFnName]
+
+    if (!mutationFn) {
+      const errorMessage = `Unknown mutation function: ${transaction.mutationFnName}`
+
+      if (this.config.onUnknownMutationFn) {
+        this.config.onUnknownMutationFn(transaction.mutationFnName, transaction)
+      }
+
+      throw new NonRetriableError(errorMessage)
+    }
+
+    // Mutations are already PendingMutation objects with collections attached
+    // from the deserializer, so we can use them directly
+    const transactionWithMutations = {
+      id: transaction.id,
+      mutations: transaction.mutations,
+      metadata: transaction.metadata ?? {},
+    }
+
+    await mutationFn({
+      transaction: transactionWithMutations as any,
+      idempotencyKey: transaction.idempotencyKey,
+    })
+  }
+
+  private async handleError(
+    transaction: OfflineTransaction,
+    error: Error
+  ): Promise<void> {
+    return withNestedSpan(
+      `transaction.handleError`,
+      {
+        "transaction.id": transaction.id,
+        "error.name": error.name,
+        "error.message": error.message,
+      },
+      async (span) => {
+        const shouldRetry = this.retryPolicy.shouldRetry(
+          error,
+          transaction.retryCount
+        )
+
+        span.setAttribute(`shouldRetry`, shouldRetry)
+
+        if (!shouldRetry) {
+          this.scheduler.markCompleted(transaction)
+          await this.outbox.remove(transaction.id)
+          console.warn(
+            `Transaction ${transaction.id} failed permanently:`,
+            error
+          )
+
+          span.setAttribute(`result`, `permanent_failure`)
+          // Signal permanent failure to the waiting transaction
+          this.offlineExecutor.rejectTransaction(transaction.id, error)
+          return
+        }
+
+        const delay = this.retryPolicy.calculateDelay(transaction.retryCount)
+        const updatedTransaction: OfflineTransaction = {
+          ...transaction,
+          retryCount: transaction.retryCount + 1,
+          nextAttemptAt: Date.now() + delay,
+          lastError: {
+            name: error.name,
+            message: error.message,
+            stack: error.stack,
+          },
+        }
+
+        span.setAttribute(`retryDelay`, delay)
+        span.setAttribute(`nextRetryCount`, updatedTransaction.retryCount)
+
+        this.scheduler.markFailed(transaction)
+        this.scheduler.updateTransaction(updatedTransaction)
+
+        try {
+          await this.outbox.update(transaction.id, updatedTransaction)
+          span.setAttribute(`result`, `scheduled_retry`)
+        } catch (persistError) {
+          span.recordException(persistError as Error)
+          span.setAttribute(`result`, `persist_failed`)
+          throw persistError
+        }
+
+        // Schedule retry timer
+        this.scheduleNextRetry()
+      }
+    )
+  }
+
+  async loadPendingTransactions(): Promise<void> {
+    const transactions = await this.outbox.getAll()
+    let filteredTransactions = transactions
+
+    if (this.config.beforeRetry) {
+      filteredTransactions = this.config.beforeRetry(transactions)
+    }
+
+    for (const transaction of filteredTransactions) {
+      this.scheduler.schedule(transaction)
+    }
+
+    // Reset retry delays for all loaded transactions so they can run immediately
+    this.resetRetryDelays()
+
+    // Schedule retry timer for loaded transactions
+    this.scheduleNextRetry()
+
+    const removedTransactions = transactions.filter(
+      (tx) => !filteredTransactions.some((filtered) => filtered.id === tx.id)
+    )
+
+    if (removedTransactions.length > 0) {
+      await this.outbox.removeMany(removedTransactions.map((tx) => tx.id))
+    }
+  }
+
+  clear(): void {
+    this.scheduler.clear()
+    this.clearRetryTimer()
+  }
+
+  getPendingCount(): number {
+    return this.scheduler.getPendingCount()
+  }
+
+  private scheduleNextRetry(): void {
+    // Clear existing timer
+    this.clearRetryTimer()
+
+    // Find the earliest retry time among pending transactions
+    const earliestRetryTime = this.getEarliestRetryTime()
+
+    if (earliestRetryTime === null) {
+      return // No transactions pending retry
+    }
+
+    const delay = Math.max(0, earliestRetryTime - Date.now())
+
+    this.retryTimer = setTimeout(() => {
+      this.executeAll().catch((error) => {
+        console.warn(`Failed to execute retry batch:`, error)
+      })
+    }, delay)
+  }
+
+  private getEarliestRetryTime(): number | null {
+    const allTransactions = this.scheduler.getAllPendingTransactions()
+
+    if (allTransactions.length === 0) {
+      return null
+    }
+
+    return Math.min(...allTransactions.map((tx) => tx.nextAttemptAt))
+  }
+
+  private clearRetryTimer(): void {
+    if (this.retryTimer) {
+      clearTimeout(this.retryTimer)
+      this.retryTimer = null
+    }
+  }
+
+  getRunningCount(): number {
+    return this.scheduler.getRunningCount()
+  }
+
+  resetRetryDelays(): void {
+    const allTransactions = this.scheduler.getAllPendingTransactions()
+    const updatedTransactions = allTransactions.map((transaction) => ({
+      ...transaction,
+      nextAttemptAt: Date.now(),
+    }))
+
+    this.scheduler.updateTransactions(updatedTransactions)
+  }
+}

package/src/index.ts

@@ -0,0 +1,47 @@
+// Main API
+export { OfflineExecutor, startOfflineExecutor } from "./OfflineExecutor"
+
+// Types
+export type {
+  OfflineTransaction,
+  OfflineConfig,
+  StorageAdapter,
+  RetryPolicy,
+  LeaderElection,
+  OnlineDetector,
+  CreateOfflineTransactionOptions,
+  CreateOfflineActionOptions,
+  SerializedError,
+  SerializedMutation,
+} from "./types"
+
+export { NonRetriableError } from "./types"
+
+// Storage adapters
+export { IndexedDBAdapter } from "./storage/IndexedDBAdapter"
+export { LocalStorageAdapter } from "./storage/LocalStorageAdapter"
+
+// Retry policies
+export { DefaultRetryPolicy } from "./retry/RetryPolicy"
+export { BackoffCalculator } from "./retry/BackoffCalculator"
+
+// Coordination
+export { WebLocksLeader } from "./coordination/WebLocksLeader"
+export { BroadcastChannelLeader } from "./coordination/BroadcastChannelLeader"
+
+// Connectivity
+export { DefaultOnlineDetector } from "./connectivity/OnlineDetector"
+
+// API components
+export { OfflineTransaction as OfflineTransactionAPI } from "./api/OfflineTransaction"
+export { createOfflineAction } from "./api/OfflineAction"
+
+// Outbox management
+export { OutboxManager } from "./outbox/OutboxManager"
+export { TransactionSerializer } from "./outbox/TransactionSerializer"
+
+// Execution engine
+export { KeyScheduler } from "./executor/KeyScheduler"
+export { TransactionExecutor } from "./executor/TransactionExecutor"
+
+// Replay

package/src/outbox/OutboxManager.ts

@@ -0,0 +1,141 @@
+import { withSpan } from "../telemetry/tracer"
+import { TransactionSerializer } from "./TransactionSerializer"
+import type { OfflineTransaction, StorageAdapter } from "../types"
+import type { Collection } from "@tanstack/db"
+
+export class OutboxManager {
+  private storage: StorageAdapter
+  private serializer: TransactionSerializer
+  private keyPrefix = `tx:`
+
+  constructor(
+    storage: StorageAdapter,
+    collections: Record<string, Collection>
+  ) {
+    this.storage = storage
+    this.serializer = new TransactionSerializer(collections)
+  }
+
+  private getStorageKey(id: string): string {
+    return `${this.keyPrefix}${id}`
+  }
+
+  async add(transaction: OfflineTransaction): Promise<void> {
+    return withSpan(
+      `outbox.add`,
+      {
+        "transaction.id": transaction.id,
+        "transaction.mutationFnName": transaction.mutationFnName,
+        "transaction.keyCount": transaction.keys.length,
+      },
+      async () => {
+        const key = this.getStorageKey(transaction.id)
+        const serialized = this.serializer.serialize(transaction)
+        await this.storage.set(key, serialized)
+      }
+    )
+  }
+
+  async get(id: string): Promise<OfflineTransaction | null> {
+    return withSpan(`outbox.get`, { "transaction.id": id }, async (span) => {
+      const key = this.getStorageKey(id)
+      const data = await this.storage.get(key)
+
+      if (!data) {
+        span.setAttribute(`result`, `not_found`)
+        return null
+      }
+
+      try {
+        const transaction = this.serializer.deserialize(data)
+        span.setAttribute(`result`, `found`)
+        return transaction
+      } catch (error) {
+        console.warn(`Failed to deserialize transaction ${id}:`, error)
+        span.setAttribute(`result`, `deserialize_error`)
+        return null
+      }
+    })
+  }
+
+  async getAll(): Promise<Array<OfflineTransaction>> {
+    return withSpan(`outbox.getAll`, {}, async (span) => {
+      const keys = await this.storage.keys()
+      const transactionKeys = keys.filter((key) =>
+        key.startsWith(this.keyPrefix)
+      )
+
+      span.setAttribute(`transactionCount`, transactionKeys.length)
+
+      const transactions: Array<OfflineTransaction> = []
+
+      for (const key of transactionKeys) {
+        const data = await this.storage.get(key)
+        if (data) {
+          try {
+            const transaction = this.serializer.deserialize(data)
+            transactions.push(transaction)
+          } catch (error) {
+            console.warn(
+              `Failed to deserialize transaction from key ${key}:`,
+              error
+            )
+          }
+        }
+      }
+
+      return transactions.sort(
+        (a, b) => a.createdAt.getTime() - b.createdAt.getTime()
+      )
+    })
+  }
+
+  async getByKeys(keys: Array<string>): Promise<Array<OfflineTransaction>> {
+    const allTransactions = await this.getAll()
+    const keySet = new Set(keys)
+
+    return allTransactions.filter((transaction) =>
+      transaction.keys.some((key) => keySet.has(key))
+    )
+  }
+
+  async update(
+    id: string,
+    updates: Partial<OfflineTransaction>
+  ): Promise<void> {
+    return withSpan(`outbox.update`, { "transaction.id": id }, async () => {
+      const existing = await this.get(id)
+      if (!existing) {
+        throw new Error(`Transaction ${id} not found`)
+      }
+
+      const updated = { ...existing, ...updates }
+      await this.add(updated)
+    })
+  }
+
+  async remove(id: string): Promise<void> {
+    return withSpan(`outbox.remove`, { "transaction.id": id }, async () => {
+      const key = this.getStorageKey(id)
+      await this.storage.delete(key)
+    })
+  }
+
+  async removeMany(ids: Array<string>): Promise<void> {
+    return withSpan(`outbox.removeMany`, { count: ids.length }, async () => {
+      await Promise.all(ids.map((id) => this.remove(id)))
+    })
+  }
+
+  async clear(): Promise<void> {
+    const keys = await this.storage.keys()
+    const transactionKeys = keys.filter((key) => key.startsWith(this.keyPrefix))
+
+    await Promise.all(transactionKeys.map((key) => this.storage.delete(key)))
+  }
+
+  async count(): Promise<number> {
+    const keys = await this.storage.keys()
+    return keys.filter((key) => key.startsWith(this.keyPrefix)).length
+  }
+}

package/src/outbox/TransactionSerializer.ts

@@ -0,0 +1,163 @@
+import type {
+  OfflineTransaction,
+  SerializedError,
+  SerializedMutation,
+  SerializedOfflineTransaction,
+} from "../types"
+import type { Collection, PendingMutation } from "@tanstack/db"
+
+export class TransactionSerializer {
+  private collections: Record<string, Collection>
+  private collectionIdToKey: Map<string, string>
+
+  constructor(collections: Record<string, Collection>) {
+    this.collections = collections
+    // Create reverse lookup from collection.id to registry key
+    this.collectionIdToKey = new Map()
+    for (const [key, collection] of Object.entries(collections)) {
+      this.collectionIdToKey.set(collection.id, key)
+    }
+  }
+
+  serialize(transaction: OfflineTransaction): string {
+    const serialized: SerializedOfflineTransaction = {
+      ...transaction,
+      createdAt: transaction.createdAt,
+      mutations: transaction.mutations.map((mutation) =>
+        this.serializeMutation(mutation)
+      ),
+    }
+    // Convert the whole object to JSON, handling dates
+    return JSON.stringify(serialized, (key, value) => {
+      if (value instanceof Date) {
+        return value.toISOString()
+      }
+      return value
+    })
+  }
+
+  deserialize(data: string): OfflineTransaction {
+    const parsed: SerializedOfflineTransaction = JSON.parse(
+      data,
+      (key, value) => {
+        // Parse ISO date strings back to Date objects
+        if (
+          typeof value === `string` &&
+          /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/.test(value)
+        ) {
+          return new Date(value)
+        }
+        return value
+      }
+    )
+
+    return {
+      ...parsed,
+      mutations: parsed.mutations.map((mutationData) =>
+        this.deserializeMutation(mutationData)
+      ),
+    }
+  }
+
+  private serializeMutation(mutation: PendingMutation): SerializedMutation {
+    const registryKey = this.collectionIdToKey.get(mutation.collection.id)
+    if (!registryKey) {
+      throw new Error(
+        `Collection with id ${mutation.collection.id} not found in registry`
+      )
+    }
+
+    return {
+      globalKey: mutation.globalKey,
+      type: mutation.type,
+      modified: this.serializeValue(mutation.modified),
+      original: this.serializeValue(mutation.original),
+      collectionId: registryKey, // Store registry key instead of collection.id
+    }
+  }
+
+  private deserializeMutation(data: SerializedMutation): PendingMutation {
+    const collection = this.collections[data.collectionId]
+    if (!collection) {
+      throw new Error(`Collection with id ${data.collectionId} not found`)
+    }
+
+    // Create a partial PendingMutation - we can't fully reconstruct it but
+    // we provide what we can. The executor will need to handle the rest.
+    return {
+      globalKey: data.globalKey,
+      type: data.type as any,
+      modified: this.deserializeValue(data.modified),
+      original: this.deserializeValue(data.original),
+      collection,
+      // These fields would need to be reconstructed by the executor
+      mutationId: ``, // Will be regenerated
+      key: null, // Will be extracted from the data
+      changes: {}, // Will be recalculated
+      metadata: undefined,
+      syncMetadata: {},
+      optimistic: true,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    } as PendingMutation
+  }
+
+  private serializeValue(value: any): any {
+    if (value === null || value === undefined) {
+      return value
+    }
+
+    if (value instanceof Date) {
+      return { __type: `Date`, value: value.toISOString() }
+    }
+
+    if (typeof value === `object`) {
+      const result: any = Array.isArray(value) ? [] : {}
+      for (const key in value) {
+        if (value.hasOwnProperty(key)) {
+          result[key] = this.serializeValue(value[key])
+        }
+      }
+      return result
+    }
+
+    return value
+  }
+
+  private deserializeValue(value: any): any {
+    if (value === null || value === undefined) {
+      return value
+    }
+
+    if (typeof value === `object` && value.__type === `Date`) {
+      return new Date(value.value)
+    }
+
+    if (typeof value === `object`) {
+      const result: any = Array.isArray(value) ? [] : {}
+      for (const key in value) {
+        if (value.hasOwnProperty(key)) {
+          result[key] = this.deserializeValue(value[key])
+        }
+      }
+      return result
+    }
+
+    return value
+  }
+
+  serializeError(error: Error): SerializedError {
+    return {
+      name: error.name,
+      message: error.message,
+      stack: error.stack,
+    }
+  }
+
+  deserializeError(data: SerializedError): Error {
+    const error = new Error(data.message)
+    error.name = data.name
+    error.stack = data.stack
+    return error
+  }
+}

package/src/retry/BackoffCalculator.ts

@@ -0,0 +1,13 @@
+export class BackoffCalculator {
+  private jitter: boolean
+
+  constructor(jitter = true) {
+    this.jitter = jitter
+  }
+
+  calculate(retryCount: number): number {
+    const baseDelay = Math.min(1000 * Math.pow(2, retryCount), 60000)
+    const jitterMultiplier = this.jitter ? Math.random() * 0.3 : 0
+    return Math.floor(baseDelay * (1 + jitterMultiplier))
+  }
+}

package/src/retry/NonRetriableError.ts

@@ -0,0 +1 @@
+export { NonRetriableError } from "../types"