@abloatai/ablo 0.3.0

package/dist/transactions/OptimisticEchoTracker.js

@@ -0,0 +1,104 @@
+/**
+ * OptimisticEchoTracker — receive-layer reconciliation primitive.
+ *
+ * Tracks the set of transaction ids the client has applied locally
+ * but the server has not yet confirmed. When a sync delta arrives
+ * carrying a `transactionId` the tracker recognizes, the pool
+ * mutation is suppressed (the optimistic write already reflects it).
+ * Drained when the matching delta echo lands or when the originating
+ * transaction is rolled back.
+ *
+ * Architectural framing: see
+ * `apps/sync-server/docs/OPTIMISTIC_RECONCILIATION.md`. This is the
+ * "discriminator at the apply layer" the doc names — without it the
+ * authoritative-layer apply path blindly re-applies confirmations on
+ * top of whatever optimistic state has since diverged, producing the
+ * chart-delete flicker.
+ *
+ * Bounded by `maxSize` to defend against runaway growth from a
+ * pathological "transactions never confirmed and never rolled back"
+ * loop. When the bound is hit, the OLDEST id is evicted (FIFO via
+ * insertion-ordered Map). Eviction means a future echo of that
+ * transaction will be applied as a foreign mutation — no correctness
+ * risk for the originating tab (the pool already reflects the local
+ * write); the worst case is a single redundant pool re-set.
+ */
+const DEFAULT_MAX_SIZE = 10_000;
+export class OptimisticEchoTracker {
+    // Map (not Set) for O(1) FIFO eviction via insertion order.
+    // Value is unused; Map.keys() iterates in insertion order so
+    // `keys().next()` yields the oldest id.
+    ids = new Map();
+    maxSize;
+    // Metrics — internal counters; exposed via `getMetrics()`. Kept
+    // numeric (not BigInt) since cumulative-since-page-load fits well
+    // under Number.MAX_SAFE_INTEGER for any realistic session.
+    _totalAdded = 0;
+    _hits = 0;
+    _rollbacks = 0;
+    _evictions = 0;
+    constructor(options = {}) {
+        this.maxSize = options.maxSize ?? DEFAULT_MAX_SIZE;
+    }
+    /**
+     * Mark a transaction as locally-applied. The next sync delta with a
+     * matching `transactionId` will be recognized as the server's
+     * confirmation of this same write. Idempotent — repeated calls with
+     * the same id are no-ops.
+     */
+    markPending(transactionId) {
+        if (this.ids.has(transactionId))
+            return;
+        if (this.ids.size >= this.maxSize) {
+            const oldest = this.ids.keys().next().value;
+            if (oldest !== undefined) {
+                this.ids.delete(oldest);
+                this._evictions += 1;
+            }
+        }
+        this.ids.set(transactionId, true);
+        this._totalAdded += 1;
+    }
+    /**
+     * If the id is currently tracked, drain it and return true (signal
+     * to the caller: this is an echo, skip the pool mutation).
+     * Otherwise return false (foreign mutation, apply normally).
+     *
+     * Combined check+drain into one method to make the receive-path
+     * idiom hard to misuse: a separate `has` then `drain` would race
+     * if multiple deltas with the same id arrived in the same batch.
+     */
+    consumeEcho(transactionId) {
+        if (!transactionId)
+            return false;
+        if (!this.ids.has(transactionId))
+            return false;
+        this.ids.delete(transactionId);
+        this._hits += 1;
+        return true;
+    }
+    /**
+     * Drain on rollback. The transaction was cancelled before a server
+     * confirmation arrived — no echo will ever come, so the pending
+     * entry would otherwise leak. Counts as a separate metric category
+     * so a spike of `rollbacks` (vs `hits`) signals network or
+     * server-side health issues.
+     */
+    drainOnRollback(transactionId) {
+        if (this.ids.delete(transactionId)) {
+            this._rollbacks += 1;
+        }
+    }
+    getMetrics() {
+        return {
+            size: this.ids.size,
+            totalAdded: this._totalAdded,
+            hits: this._hits,
+            rollbacks: this._rollbacks,
+            evictions: this._evictions,
+        };
+    }
+    clear() {
+        this.ids.clear();
+    }
+}

package/dist/transactions/TransactionQueue.d.ts

@@ -0,0 +1,499 @@
+/**
+ * TransactionQueue - Production-ready transaction management
+ *
+ * Key features:
+ * - Optimistic updates with rollback
+ * - Conflict resolution strategies
+ * - LINEAR-style microtask batching (transactions in same event loop share batchId)
+ * - Proper dependency injection (no singleton)
+ */
+import { EventEmitter } from 'events';
+import type { Database } from '../Database.js';
+import { Model } from '../Model.js';
+import type { MutationOptions } from '../interfaces/index.js';
+export interface UserContext {
+    userId: string;
+    organizationId: string;
+    role?: string;
+    teamIds?: string[];
+}
+/** Wire-format mutation payload (post-projection). */
+type MutationInput = Record<string, unknown>;
+type TransactionWriteOptions = Pick<MutationOptions, 'readAt' | 'onStale'>;
+export interface Transaction {
+    id: string;
+    type: 'create' | 'update' | 'delete' | 'archive' | 'unarchive';
+    modelName: string;
+    modelId: string;
+    modelKey: string;
+    data?: MutationInput;
+    previousData?: MutationInput | null;
+    context: UserContext;
+    status: 'pending' | 'executing' | 'awaiting_delta' | 'completed' | 'failed' | 'rolled_back';
+    createdAt: number;
+    attempts: number;
+    priority: 'normal' | 'high';
+    priorityScore: number;
+    writeOptions?: TransactionWriteOptions;
+    batchId?: string;
+    /** LINEAR PATTERN: syncId threshold - transaction confirms when delta.id >= this value */
+    syncIdNeededForCompletion?: number;
+    /**
+     * Resolves when the server has confirmed this transaction (delta arrived
+     * or HTTP ack). Rejects with the originating error if the transaction is
+     * permanently rolled back. Name matches the queue's existing `'confirmed'`
+     * status vocabulary (`commits.create({wait:'confirmed'})`,
+     * `waitForConfirmation`) — gives call sites a single `await` point for
+     * "did my write land?", so failures surface at the source instead of
+     * leaking via silent pool rollback. The rejection error is the same
+     * `AbloError` recorded on the queue's `transaction:failed` event.
+     */
+    confirmation?: Promise<void>;
+}
+/**
+ * A raw multi-op commit transaction queued via `ablo.commits.create()`.
+ *
+ * Distinct from the per-model `Transaction` above: operations are
+ * pre-built by the caller and the envelope is atomic — no coalescing,
+ * no FK reordering, no optimistic local apply. The lane shares the
+ * same `mutationExecutor.commit()` underneath as the model-proxy
+ * batch path, so reconnect-retry behavior is identical.
+ */
+interface CommitTransaction {
+    id: string;
+    kind: 'commit';
+    operations: Array<{
+        type: string;
+        model: string;
+        id: string;
+        input?: Record<string, unknown>;
+        transactionId?: string;
+        readAt?: number | null;
+        onStale?: 'reject' | 'force' | 'flag' | 'merge' | null;
+    }>;
+    causedByTaskId?: string | null;
+    status: 'pending' | 'executing' | 'completed' | 'failed';
+    createdAt: number;
+    attempts: number;
+    lastSyncId?: number;
+    error?: Error;
+}
+interface ConflictResolution {
+    strategy: 'last-write-wins' | 'merge' | 'reject' | 'custom';
+    resolver?: (local: MutationInput | undefined, remote: MutationInput) => MutationInput;
+}
+interface TransactionQueueConfig {
+    maxBatchSize: number;
+    batchDelay: number;
+    maxRetries: number;
+    conflictResolution: ConflictResolution;
+    enablePersistence: boolean;
+    enableOptimistic: boolean;
+    maxExecutingTransactions: number;
+    deltaConfirmationTimeout: number;
+    /**
+     * Exponential backoff for retryable server responses (HTTP 429/503).
+     * `baseMs` is the first retry delay; each subsequent attempt doubles
+     * up to `capMs`. Final delay = min(capMs, baseMs * 2^(attempt-1)) +
+     * up to 100ms of jitter. Defaults: 200ms / 1500ms.
+     */
+    retryBackoff: {
+        baseMs: number;
+        capMs: number;
+    };
+    /**
+     * Grace window in ms before in-flight commit-lane transactions are
+     * failed with `AbloConnectionError` after the WebSocket transitions
+     * to `'disconnected'`. Brief disconnects (deploy rotations, mobile
+     * jitter) are absorbed transparently; only persistent disconnects
+     * surface as failures. Aligned with the 30s convention from the
+     * WebSocket reconnection guidance (websocket.org). Set lower for
+     * human-interactive consumers (e.g. 10s for chat) or higher for
+     * batch workers (e.g. 60s for agent-worker).
+     *
+     * Without this deadline, `commits.create({wait:'confirmed'})` waits
+     * forever when the WS dies mid-flight — see the 2026-05-15 wedge.
+     */
+    commitOfflineGraceMs: number;
+}
+export declare class TransactionQueue extends EventEmitter {
+    private store;
+    private _mutationExecutor;
+    private get mutationExecutor();
+    private executionQueue;
+    private isProcessing;
+    private processTimer?;
+    private processScheduled;
+    private createdTransactions;
+    private commitScheduled;
+    private inFlightByModel;
+    private pendingMergeByModel;
+    private commitLane;
+    private commitStore;
+    private commitProcessing;
+    private computePriorityScore;
+    private ensureDerivedFields;
+    private mergeUpdateData;
+    private config;
+    private executingCount;
+    private optimisticUpdates;
+    private deltaConfirmationTimeouts;
+    private deltaConfirmationRetries;
+    private isConnectedFn;
+    private commitOfflineGraceTimer;
+    private lastSeenSyncId;
+    private static readonly DELTA_MAX_RETRIES;
+    private static readonly DELTA_INITIAL_TIMEOUT_MS;
+    private static readonly DELTA_MAX_TIMEOUT_MS;
+    private batchIndex;
+    /**
+     * Resolvers for per-transaction `confirmation` promises. Populated in
+     * `attachConfirmation` at staging time, consumed by the constructor-time
+     * listeners on `transaction:completed` / `transaction:failed`. Kept off
+     * the Transaction row so the store's iteration order stays plain-data
+     * and serialization-friendly.
+     */
+    private confirmationResolvers;
+    constructor(config?: Partial<TransactionQueueConfig>);
+    /**
+     * Look up the in-flight `confirmation` promise for a (model, id) pair.
+     * Returns the promise from the most-recent live transaction matching
+     * the given model+id, or `Promise.resolve()` if none is open (which
+     * means either "already confirmed" or "never staged" — both safe
+     * outcomes for the routing-helper grace-window use case).
+     *
+     * Looks across `pending`, `executing`, and `awaiting_delta` — these
+     * are the three non-terminal statuses where rollback is still
+     * possible. Skips `completed` (already settled) and `failed` /
+     * `rolled_back` (already rejected; the call site missed the
+     * `confirmation` window and should rely on `onMutationFailure` toast
+     * instead).
+     *
+     * Distinct from `tx.confirmation` on a known transaction — used by
+     * call sites that hold a Model reference (returned by
+     * `ablo.<model>.create()`) but never see the underlying transaction.
+     */
+    confirmationFor(modelName: string, modelId: string): Promise<void>;
+    /**
+     * Attach a hot `confirmation` promise to a freshly created transaction.
+     * Must be called BEFORE the transaction is staged so the call site can
+     * `await tx.confirmation` synchronously after the create/update/delete
+     * call returns. Idempotent: returns early if the tx already has one.
+     *
+     * The unhandled-rejection trap is mandatory — most call sites won't
+     * `await confirmation`, and Node/browser would otherwise crash on the
+     * rejection. Consumers who *do* want failure visibility just attach a
+     * `.then`/`.catch` and the trap becomes a no-op.
+     */
+    private attachConfirmation;
+    /**
+     * Set connection state checker - prevents rollbacks during disconnection.
+     * When disconnected, timeouts re-schedule instead of rolling back.
+     */
+    setConnectionChecker(fn: () => boolean): void;
+    /**
+     * Drive the offline-grace timer for in-flight commit-lane transactions.
+     *
+     * On `'disconnected'`: start a one-shot timer of
+     * `config.commitOfflineGraceMs`. If the timer fires (disconnect
+     * persisted past grace), iterate every commit-lane transaction with
+     * `status ∈ {'pending', 'executing'}` and emit
+     * `transaction:failed:${id}` with an `AbloConnectionError`. That
+     * lets `waitForCommitReceipt` reject in seconds instead of hanging
+     * forever — which is what wedged the 2026-05-15 subagent run.
+     *
+     * On `'connected'`: clear any pending grace timer. Brief blips are
+     * absorbed transparently; the existing reconnect-retry path in
+     * `processCommitLane` / `flushOfflineQueue` handles the resumption.
+     *
+     * Called from SyncClient's `setConnectionState` after the
+     * `'connection:disconnected'` / `'connection:established'` events.
+     */
+    setConnectionState(state: 'connected' | 'disconnected'): void;
+    private failInFlightCommitsOnOffline;
+    /**
+     * Bind the executor for this queue instance. Called by the owning Ablo
+     * right after `BaseSyncedStore` is constructed so the executor's
+     * `storeHolder.store` closure resolves to *this* Ablo's WS — not whichever
+     * Ablo most recently called `initSyncEngine()`.
+     */
+    setMutationExecutor(executor: import('../interfaces/index.js').MutationExecutor): void;
+    /**
+     * Stage a transaction for commit (Linear pattern)
+     * Transactions staged in the same event loop tick will be committed together
+     */
+    private stageTransaction;
+    /**
+     * Schedule commit of staged transactions via microtask
+     * This ensures all synchronous transaction creates are batched together
+     */
+    private scheduleCommit;
+    /**
+     * Commit all staged transactions to the execution queue (Linear pattern)
+     * All transactions get the same batchIndex for efficient batching
+     */
+    private commitCreatedTransactions;
+    flushOfflineQueue(): Promise<void>;
+    /**
+     * Create operation with optimistic update
+     */
+    create(model: Model, context: UserContext, writeOptions?: TransactionWriteOptions): Promise<Transaction>;
+    /**
+     * Update operation with conflict detection
+     * @param precomputedChanges - Optional pre-captured changes (avoids re-reading from model)
+     */
+    update(model: Model, context: UserContext, precomputedChanges?: Record<string, unknown>, writeOptions?: TransactionWriteOptions): Promise<Transaction>;
+    /**
+     * Delete operation with cascade handling
+     */
+    delete(model: Model, context: UserContext, writeOptions?: TransactionWriteOptions): Promise<Transaction>;
+    /**
+     * Upload attachment — delegates to attachment-uploader.ts
+     */
+    uploadAttachment(_file: File, options: {
+        id: string;
+        [key: string]: unknown;
+    }, _context: UserContext): Promise<{
+        url: string;
+    } | null>;
+    /**
+     * Batch upload attachments — delegates to MutationExecutor
+     */
+    batchUploadAttachments(_files: File[], items: Array<{
+        id: string;
+        [key: string]: unknown;
+    }>, _context: UserContext): Promise<Array<{
+        id: string;
+        url: string;
+    }>>;
+    /**
+     * Archive operation
+     */
+    archive(model: Model, context: UserContext, writeOptions?: TransactionWriteOptions): Promise<Transaction>;
+    /**
+     * Unarchive operation
+     */
+    unarchive(model: Model, context: UserContext): Promise<Transaction>;
+    /**
+     * Enqueue transaction for execution
+     */
+    private enqueue;
+    private scheduleProcessing;
+    /**
+     * Process batch of transactions using LINEAR-style unified batch execution.
+     *
+     * Key optimization: Instead of making separate calls per operation type/model,
+     * we collect ALL batchable operations and send them in a SINGLE commit call.
+     * The sync-server handles mixed types atomically inside one transaction.
+     *
+     * This reduces N round-trips to 1, dramatically improving batch latency.
+     */
+    private processBatch;
+    /**
+     * LINEAR PATTERN: Confirm all awaiting transactions when delta with syncId >= threshold arrives.
+     * This replaces clientMutationId echoing - transactions are confirmed by sync ID threshold.
+     * @param syncId - The sync ID of the received delta
+     */
+    onDeltaReceived(syncId: number): void;
+    private scheduleDeltaConfirmationTimeout;
+    private cancelDeltaConfirmationTimeout;
+    /**
+     * Wait for a transaction to be confirmed via delta echo (Linear pattern)
+     * Reuses existing timeout mechanism from scheduleDeltaConfirmationTimeout
+     */
+    waitForConfirmation(transactionId: string): Promise<void>;
+    hasClientMutationId(id: string): boolean;
+    /**
+     * Enqueue a raw multi-op atomic commit envelope (the `ablo.commits.create`
+     * path). Operations are pre-built by the caller; the queue's job is
+     * retry-on-reconnect + idempotent dedup, NOT optimistic apply or FK
+     * ordering. Same idempotency key (clientTxId) is dropped on the floor
+     * if already in flight — server-side `mutation_log` handles cross-session
+     * dedup; this guard handles same-session double-enqueue.
+     */
+    enqueueCommit(clientTxId: string, operations: CommitTransaction['operations'], options?: {
+        causedByTaskId?: string | null;
+    }): void;
+    /**
+     * Drain pending commit-lane envelopes serially. Transient failures
+     * (network, ws_not_ready) leave the head-of-queue tx in `pending` and
+     * break — reconnect handler re-kicks via `flushOfflineQueue`.
+     * Permanent failures emit `transaction:failed:<id>` and drop the tx.
+     */
+    private processCommitLane;
+    /**
+     * Promise-based confirmation for a commit-lane transaction. Resolves
+     * with the server-side `lastSyncId` once `mutation_result` lands;
+     * rejects on permanent failure. Backs the `wait: 'confirmed'` semantics
+     * of `ablo.commits.create()`.
+     */
+    waitForCommitReceipt(clientTxId: string): Promise<{
+        lastSyncId: number;
+    }>;
+    private isReorderPayload;
+    /**
+     * Determine if an error is transient (retryable) vs permanent (non-retryable).
+     *
+     * IMPORTANT: Uses a BLOCKLIST approach for safety - only retry on known transient errors.
+     * Any unknown error type defaults to permanent (don't retry) to prevent infinite loops.
+     *
+     * Transient errors (will retry):
+     * - Network failures, connection errors, timeouts
+     * - Server errors (5xx status codes)
+     * - Rate limiting (429)
+     *
+     * Permanent errors (won't retry - includes but not limited to):
+     * - Validation errors, constraint violations
+     * - Not found, unauthorized, forbidden
+     * - Any other business logic error from the server
+     */
+    private isPermanentError;
+    /**
+     * Handle transaction failure
+     */
+    private handleFailure;
+    /**
+     * Conflict resolution
+     */
+    handleConflict(transaction: Transaction, serverData: MutationInput): Promise<void>;
+    /**
+     * Optimistic updates
+     */
+    private applyOptimisticCreate;
+    private applyOptimisticUpdate;
+    private applyOptimisticDelete;
+    private rollbackOptimistic;
+    /**
+     * Execute individual transaction via the unified commit path
+     */
+    private executeTransaction;
+    /**
+     * Persistence
+     */
+    loadPersistedTransactions(database: Database): Promise<void>;
+    private deserializeTransaction;
+    /**
+     * Cancel transactions for a specific model
+     */
+    cancelTransactionsForModel(modelId: string, transactionType?: string): Transaction[];
+    /**
+     * LINEAR PATTERN: Cancel transactions for child entities by foreign key
+     *
+     * Used by SyncedStore for cascade cancellation when a parent is deleted.
+     * This keeps FK relationship knowledge in ModelRegistry/SyncedStore,
+     * while TransactionQueue just handles the cancellation mechanics.
+     *
+     * @param childModelName - The child model type (e.g., 'SlideLayer')
+     * @param foreignKey - The FK property name (e.g., 'slideId')
+     * @param parentId - The deleted parent's ID
+     * @returns Number of transactions cancelled
+     */
+    cancelTransactionsByForeignKey(childModelName: string, foreignKey: string, parentId: string): number;
+    /**
+     * Get count of outstanding transactions
+     */
+    getOutstandingTransactionCount(): number;
+    /**
+     * Utilities
+     */
+    private generateId;
+    private mergeData;
+    private extractCreateData;
+    private mapChangesToInput;
+    private extractUpdateData;
+    private buildUpdateInput;
+    private extractPreviousData;
+    /**
+     * Public API
+     */
+    getStats(): {
+        pending: number;
+        executing: number;
+        completed: number;
+        failed: number;
+        optimistic: number;
+        totalTransactions: number;
+        batchIndex: number;
+        config: {
+            maxBatchSize: number;
+            batchDelay: number;
+            maxRetries: number;
+            conflictResolution: ConflictResolution;
+            enablePersistence: boolean;
+            enableOptimistic: boolean;
+            maxExecutingTransactions: number;
+            deltaConfirmationTimeout: number;
+            /**
+             * Exponential backoff for retryable server responses (HTTP 429/503).
+             * `baseMs` is the first retry delay; each subsequent attempt doubles
+             * up to `capMs`. Final delay = min(capMs, baseMs * 2^(attempt-1)) +
+             * up to 100ms of jitter. Defaults: 200ms / 1500ms.
+             */
+            retryBackoff: {
+                baseMs: number;
+                capMs: number;
+            };
+            /**
+             * Grace window in ms before in-flight commit-lane transactions are
+             * failed with `AbloConnectionError` after the WebSocket transitions
+             * to `'disconnected'`. Brief disconnects (deploy rotations, mobile
+             * jitter) are absorbed transparently; only persistent disconnects
+             * surface as failures. Aligned with the 30s convention from the
+             * WebSocket reconnection guidance (websocket.org). Set lower for
+             * human-interactive consumers (e.g. 10s for chat) or higher for
+             * batch workers (e.g. 60s for agent-worker).
+             *
+             * Without this deadline, `commits.create({wait:'confirmed'})` waits
+             * forever when the WS dies mid-flight — see the 2026-05-15 wedge.
+             */
+            commitOfflineGraceMs: number;
+        };
+    };
+    /**
+     * Get detailed debug info for the sync debug page
+     * Exposes internal state that helps diagnose delta confirmation issues
+     */
+    getDebugInfo(): {
+        lastSeenSyncId: number;
+        awaitingDeltaCount: number;
+        awaitingDeltaTransactions: {
+            id: string;
+            type: "update" | "create" | "delete" | "archive" | "unarchive";
+            modelName: string;
+            modelId: string;
+            syncIdNeeded: number | undefined;
+            createdAt: number;
+            age: number;
+        }[];
+        pendingTransactions: {
+            id: string;
+            type: "update" | "create" | "delete" | "archive" | "unarchive";
+            modelName: string;
+            modelId: string;
+        }[];
+        executingTransactions: {
+            id: string;
+            type: "update" | "create" | "delete" | "archive" | "unarchive";
+            modelName: string;
+            modelId: string;
+        }[];
+    };
+    /**
+     * Set configuration
+     */
+    setConfig(config: Partial<TransactionQueueConfig>): void;
+    /**
+     * Handle incoming sync delta - simplified for permanent IDs
+     */
+    handleSyncDelta(delta: {
+        id: string;
+        modelName: string;
+        action: string;
+        data: any;
+    }): boolean;
+    /**
+     * Cleanup and dispose resources
+     */
+    dispose(): void;
+}
+export {};