@syncular/client 0.0.6-184 → 0.0.6-188

package/dist/sync-loop.js

@@ -5,8 +5,8 @@
  */
 import { upsertConflictsForRejectedCommit } from './conflicts.js';
 import { getNextSendableOutboxCommit, markOutboxCommitAcked, markOutboxCommitFailed, markOutboxCommitPending, } from './outbox.js';
+import { INCREMENTING_VERSION_PLUGIN_KIND } from './plugins/incrementing-version.js';
 import { applyPullResponse, buildPullRequest, createFollowupPullState, syncPullOnce, } from './pull-engine.js';
-import { syncPushOnce } from './push-engine.js';
 function firstPushErrorCode(response) {
     const firstError = response.results.find((result) => result.status === 'error');
     if (firstError &&
@@ -29,21 +29,239 @@ function buildPushResult(args) {
         timestamp: Date.now(),
     };
 }
+function normalizeError(error) {
+    return error instanceof Error ? error : new Error(String(error));
+}
+function createPushRequest(clientId, outboxCommit) {
+    return {
+        clientId,
+        clientCommitId: outboxCommit.client_commit_id,
+        operations: outboxCommit.operations,
+        schemaVersion: outboxCommit.schema_version,
+    };
+}
+function toSyncPushResponse(response) {
+    return {
+        ok: true,
+        status: response.status,
+        commitSeq: response.commitSeq,
+        results: response.results,
+    };
+}
+function isRetriablePushResponse(response) {
+    const errorResults = response.results.filter((result) => result.status === 'error');
+    return (errorResults.length > 0 &&
+        errorResults.every((result) => result.retriable === true));
+}
+async function claimSendableOutboxCommits(db, maxCommits) {
+    const claimed = [];
+    for (let i = 0; i < maxCommits; i++) {
+        const next = await getNextSendableOutboxCommit(db);
+        if (!next)
+            break;
+        claimed.push(next);
+    }
+    return claimed;
+}
+async function markClaimedOutboxCommitsPending(db, outboxCommits, error) {
+    for (const outboxCommit of outboxCommits) {
+        await markOutboxCommitPending(db, {
+            id: outboxCommit.id,
+            error,
+        });
+    }
+}
+function hasIncrementingVersionPlugin(plugins) {
+    return plugins.some((plugin) => plugin.kind === INCREMENTING_VERSION_PLUGIN_KIND ||
+        plugin.name === INCREMENTING_VERSION_PLUGIN_KIND);
+}
+function makeSequentialRowKey(operation) {
+    return `${operation.table}\u001f${operation.row_id}`;
+}
+function advanceSequentialBaseVersionsInBatch(preparedCommits) {
+    const nextExpectedBaseVersionByRow = new Map();
+    return preparedCommits.map((preparedCommit) => {
+        let requestChanged = false;
+        const operations = preparedCommit.request.operations.map((operation) => {
+            const key = makeSequentialRowKey(operation);
+            const nextExpected = nextExpectedBaseVersionByRow.get(key);
+            const baseVersion = typeof operation.base_version === 'number' &&
+                typeof nextExpected === 'number' &&
+                nextExpected > operation.base_version
+                ? nextExpected
+                : operation.base_version;
+            const rewrittenOperation = baseVersion === operation.base_version
+                ? operation
+                : { ...operation, base_version: baseVersion };
+            if (rewrittenOperation !== operation) {
+                requestChanged = true;
+            }
+            if (operation.op === 'delete') {
+                nextExpectedBaseVersionByRow.delete(key);
+                return rewrittenOperation;
+            }
+            if (typeof baseVersion === 'number') {
+                nextExpectedBaseVersionByRow.set(key, baseVersion + 1);
+                return rewrittenOperation;
+            }
+            nextExpectedBaseVersionByRow.set(key, 1);
+            return rewrittenOperation;
+        });
+        if (!requestChanged)
+            return preparedCommit;
+        return {
+            ...preparedCommit,
+            request: {
+                ...preparedCommit.request,
+                operations,
+            },
+        };
+    });
+}
+async function preparePushCommits(outboxCommits, options) {
+    const plugins = options.plugins ?? [];
+    const ctx = {
+        actorId: options.actorId ?? 'unknown',
+        clientId: options.clientId,
+    };
+    const prepared = [];
+    for (const outboxCommit of outboxCommits) {
+        let request = createPushRequest(options.clientId, outboxCommit);
+        for (const plugin of plugins) {
+            if (!plugin.beforePush)
+                continue;
+            request = await plugin.beforePush(ctx, request);
+        }
+        prepared.push({ outboxCommit, request });
+    }
+    if (!hasIncrementingVersionPlugin(plugins)) {
+        return prepared;
+    }
+    return advanceSequentialBaseVersionsInBatch(prepared);
+}
+async function finalizePushCommit(db, options, preparedCommit, rawResponse) {
+    const plugins = options.plugins ?? [];
+    const ctx = {
+        actorId: options.actorId ?? 'unknown',
+        clientId: options.clientId,
+    };
+    let response = rawResponse;
+    let afterPushError = null;
+    try {
+        for (const plugin of plugins) {
+            if (!plugin.afterPush)
+                continue;
+            response = await plugin.afterPush(ctx, {
+                request: preparedCommit.request,
+                response,
+            });
+        }
+    }
+    catch (error) {
+        afterPushError = normalizeError(error);
+        response = rawResponse;
+    }
+    const responseJson = JSON.stringify(response);
+    let status;
+    if (response.status === 'applied' || response.status === 'cached') {
+        await markOutboxCommitAcked(db, {
+            id: preparedCommit.outboxCommit.id,
+            commitSeq: response.commitSeq ?? null,
+            responseJson,
+        });
+        status = response.status;
+    }
+    else if (isRetriablePushResponse(response)) {
+        const errorMessages = response.results
+            .filter((result) => result.status === 'error')
+            .map((result) => result.error ?? 'Unknown error')
+            .join('; ');
+        await markOutboxCommitPending(db, {
+            id: preparedCommit.outboxCommit.id,
+            error: `Retriable: ${errorMessages}`,
+            responseJson,
+        });
+        status = 'retriable';
+    }
+    else {
+        await upsertConflictsForRejectedCommit(db, {
+            outboxCommitId: preparedCommit.outboxCommit.id,
+            clientCommitId: preparedCommit.outboxCommit.client_commit_id,
+            response,
+        });
+        await markOutboxCommitFailed(db, {
+            id: preparedCommit.outboxCommit.id,
+            error: 'REJECTED',
+            responseJson,
+        });
+        status = 'rejected';
+    }
+    return {
+        pushResult: buildPushResult({
+            outboxCommitId: preparedCommit.outboxCommit.id,
+            clientCommitId: preparedCommit.outboxCommit.client_commit_id,
+            status,
+            response,
+        }),
+        afterPushError,
+    };
+}
 async function syncPushUntilSettled(db, transport, options) {
     const maxCommits = Math.max(1, Math.min(1000, options.maxCommits ?? 20));
     let pushedCount = 0;
     const pushResults = [];
-    for (let i = 0; i < maxCommits; i++) {
-        const res = await syncPushOnce(db, transport, {
-            clientId: options.clientId,
-            actorId: options.actorId,
-            plugins: options.plugins,
-        });
-        if (!res.pushed)
+    while (pushedCount < maxCommits) {
+        const claimedCommits = await claimSendableOutboxCommits(db, maxCommits - pushedCount);
+        if (claimedCommits.length === 0)
             break;
-        pushedCount += 1;
-        if (res.pushResult) {
-            pushResults.push(res.pushResult);
+        let preparedCommits;
+        try {
+            preparedCommits = await preparePushCommits(claimedCommits, options);
+        }
+        catch (error) {
+            const normalizedError = normalizeError(error);
+            await markClaimedOutboxCommitsPending(db, claimedCommits, normalizedError.message);
+            throw normalizedError;
+        }
+        let combined;
+        try {
+            combined = await transport.sync({
+                clientId: options.clientId,
+                push: {
+                    commits: preparedCommits.map(({ request }) => ({
+                        clientCommitId: request.clientCommitId,
+                        operations: request.operations,
+                        schemaVersion: request.schemaVersion,
+                    })),
+                },
+            });
+        }
+        catch (error) {
+            const normalizedError = normalizeError(error);
+            await markClaimedOutboxCommitsPending(db, claimedCommits, normalizedError.message);
+            throw normalizedError;
+        }
+        const batchResponses = combined.push?.commits ?? [];
+        const responsesByClientCommitId = new Map(batchResponses.map((response) => [response.clientCommitId, response]));
+        if (!combined.push ||
+            preparedCommits.some(({ request }) => !responsesByClientCommitId.has(request.clientCommitId))) {
+            await markClaimedOutboxCommitsPending(db, claimedCommits, 'MISSING_PUSH_RESPONSE');
+            throw new Error('Server returned incomplete push response');
+        }
+        let deferredError = null;
+        for (const preparedCommit of preparedCommits) {
+            const batchResponse = responsesByClientCommitId.get(preparedCommit.request.clientCommitId);
+            if (!batchResponse)
+                continue;
+            const finalized = await finalizePushCommit(db, options, preparedCommit, toSyncPushResponse(batchResponse));
+            pushResults.push(finalized.pushResult);
+            pushedCount += 1;
+            if (!deferredError && finalized.afterPushError) {
+                deferredError = finalized.afterPushError;
+            }
+        }
+        if (deferredError) {
+            throw deferredError;
         }
     }
     return { pushedCount, pushResults };
@@ -51,16 +269,16 @@ async function syncPushUntilSettled(db, transport, options) {
 function hasPushViaWs(transport) {
     return 'pushViaWs' in transport && typeof transport.pushViaWs === 'function';
 }
-function needsAnotherPull(res) {
+function needsAnotherPull(res, limitCommits) {
+    let totalCommits = 0;
     for (const sub of res.subscriptions ?? []) {
         if (sub.status !== 'active')
             continue;
         if (sub.bootstrap)
             return true;
-        if ((sub.commits?.length ?? 0) > 0)
-            return true;
+        totalCommits += sub.commits?.length ?? 0;
     }
-    return false;
+    return totalCommits >= limitCommits;
 }
 function mergePullResponse(targetBySubId, res) {
     for (const sub of res.subscriptions ?? []) {
@@ -83,6 +301,30 @@ function mergePullResponse(targetBySubId, res) {
         targetBySubId.set(sub.id, merged);
     }
 }
+function canSkipPullAfterLocalWsPush(pullState, pushResponse, options) {
+    if (!options.allowSkipPullOnLocalWsPush)
+        return false;
+    if (options.trigger !== 'local')
+        return false;
+    if (!pushResponse)
+        return false;
+    if (pushResponse.status !== 'applied' && pushResponse.status !== 'cached') {
+        return false;
+    }
+    if ((options.plugins ?? []).some((plugin) => typeof plugin.afterPull === 'function')) {
+        return false;
+    }
+    for (const subscription of pullState.request.subscriptions ?? []) {
+        if (subscription.bootstrapState != null) {
+            return false;
+        }
+        const cursor = subscription.cursor ?? -1;
+        if (cursor < 0) {
+            return false;
+        }
+    }
+    return true;
+}
 async function syncPullUntilSettled(db, transport, handlers, options) {
     const maxRounds = Math.max(1, Math.min(1000, options.maxRounds ?? 20));
     const aggregatedBySubId = new Map();
@@ -92,7 +334,7 @@ async function syncPullUntilSettled(db, transport, handlers, options) {
         rounds += 1;
         const res = await syncPullOnce(db, transport, handlers, options, pullState);
         mergePullResponse(aggregatedBySubId, res);
-        if (!needsAnotherPull(res))
+        if (!needsAnotherPull(res, pullState.request.limitCommits))
             break;
         pullState = createFollowupPullState(pullState, res);
     }
@@ -131,161 +373,162 @@ async function syncOnceCombined(db, transport, handlers, options) {
     // Build pull request (reads subscription state)
     const pullState = await buildPullRequest(db, pullOpts);
     const { clientId } = pullState.request;
-    // Grab at most one outbox commit
-    const outbox = await getNextSendableOutboxCommit(db);
     const plugins = options.plugins ?? [];
-    const ctx = {
-        actorId: options.actorId ?? 'unknown',
-        clientId,
-    };
-    // Build push request, running beforePush plugins
-    let pushRequest;
+    const outbox = await getNextSendableOutboxCommit(db);
+    let preparedFirstCommit = null;
     if (outbox) {
-        pushRequest = {
-            clientId,
-            clientCommitId: outbox.client_commit_id,
-            operations: outbox.operations,
-            schemaVersion: outbox.schema_version,
-        };
-        for (const plugin of plugins) {
-            if (!plugin.beforePush)
-                continue;
-            pushRequest = await plugin.beforePush(ctx, pushRequest);
+        try {
+            const preparedCommits = await preparePushCommits([outbox], {
+                clientId: options.clientId,
+                actorId: options.actorId,
+                plugins,
+            });
+            preparedFirstCommit = preparedCommits[0] ?? null;
+        }
+        catch (error) {
+            const normalizedError = normalizeError(error);
+            await markClaimedOutboxCommitsPending(db, [outbox], normalizedError.message);
+            throw normalizedError;
         }
     }
     // Try WS push first for the first outbox commit (if realtime transport supports it).
     // Fall back to HTTP push in the combined request when WS is unavailable or fails.
     let wsPushResponse = null;
-    if (pushRequest && hasPushViaWs(transport)) {
+    if (preparedFirstCommit && hasPushViaWs(transport)) {
         try {
-            wsPushResponse = await transport.pushViaWs(pushRequest);
+            wsPushResponse = await transport.pushViaWs(preparedFirstCommit.request);
         }
         catch {
             wsPushResponse = null;
         }
     }
-    let combined;
-    try {
-        combined = await transport.sync({
-            clientId,
-            ...(pushRequest && !wsPushResponse
-                ? {
-                    push: {
-                        clientCommitId: pushRequest.clientCommitId,
-                        operations: pushRequest.operations,
-                        schemaVersion: pushRequest.schemaVersion,
-                    },
-                }
-                : {}),
-            pull: {
-                limitCommits: pullState.request.limitCommits,
-                limitSnapshotRows: pullState.request.limitSnapshotRows,
-                maxSnapshotPages: pullState.request.maxSnapshotPages,
-                dedupeRows: pullState.request.dedupeRows,
-                subscriptions: pullState.request.subscriptions,
-            },
-        });
-    }
-    catch (err) {
-        if (outbox) {
-            const message = err instanceof Error ? err.message : 'Unknown error';
-            await markOutboxCommitPending(db, { id: outbox.id, error: message });
+    const skipPullAfterWsPush = canSkipPullAfterLocalWsPush(pullState, wsPushResponse, options);
+    let combined = null;
+    let combinedPushCommits = [];
+    if (!skipPullAfterWsPush) {
+        combinedPushCommits =
+            preparedFirstCommit && !wsPushResponse
+                ? [
+                    preparedFirstCommit,
+                    ...(await (async () => {
+                        const additionalOutboxCommits = await claimSendableOutboxCommits(db, Math.max(0, (options.maxPushCommits ?? 20) - 1));
+                        if (additionalOutboxCommits.length === 0)
+                            return [];
+                        try {
+                            return await preparePushCommits(additionalOutboxCommits, {
+                                clientId: options.clientId,
+                                actorId: options.actorId,
+                                plugins,
+                            });
+                        }
+                        catch (error) {
+                            const normalizedError = normalizeError(error);
+                            await markClaimedOutboxCommitsPending(db, additionalOutboxCommits, normalizedError.message);
+                            throw normalizedError;
+                        }
+                    })()),
+                ]
+                : [];
+        if (hasIncrementingVersionPlugin(plugins)) {
+            combinedPushCommits =
+                advanceSequentialBaseVersionsInBatch(combinedPushCommits);
+        }
+        try {
+            combined = await transport.sync({
+                clientId,
+                ...(combinedPushCommits.length > 0
+                    ? {
+                        push: {
+                            commits: combinedPushCommits.map(({ request }) => ({
+                                clientCommitId: request.clientCommitId,
+                                operations: request.operations,
+                                schemaVersion: request.schemaVersion,
+                            })),
+                        },
+                    }
+                    : {}),
+                pull: {
+                    limitCommits: pullState.request.limitCommits,
+                    limitSnapshotRows: pullState.request.limitSnapshotRows,
+                    maxSnapshotPages: pullState.request.maxSnapshotPages,
+                    dedupeRows: pullState.request.dedupeRows,
+                    subscriptions: pullState.request.subscriptions,
+                },
+            });
+        }
+        catch (err) {
+            if (combinedPushCommits.length > 0) {
+                const message = err instanceof Error ? err.message : 'Unknown error';
+                await markClaimedOutboxCommitsPending(db, combinedPushCommits.map(({ outboxCommit }) => outboxCommit), message);
+            }
+            throw err;
         }
-        throw err;
     }
     // Process push response
     let pushedCommits = 0;
     const pushResults = [];
-    if (outbox && pushRequest) {
-        let pushRes = wsPushResponse ?? combined.push;
-        if (!pushRes) {
-            await markOutboxCommitPending(db, {
-                id: outbox.id,
-                error: 'MISSING_PUSH_RESPONSE',
-            });
-            throw new Error('Server returned no push response');
-        }
-        // Run afterPush plugins
-        for (const plugin of plugins) {
-            if (!plugin.afterPush)
-                continue;
-            pushRes = await plugin.afterPush(ctx, {
-                request: pushRequest,
-                response: pushRes,
-            });
-        }
-        const responseJson = JSON.stringify(pushRes);
-        if (pushRes.status === 'applied' || pushRes.status === 'cached') {
-            await markOutboxCommitAcked(db, {
-                id: outbox.id,
-                commitSeq: pushRes.commitSeq ?? null,
-                responseJson,
-            });
-            pushResults.push(buildPushResult({
-                outboxCommitId: outbox.id,
-                clientCommitId: outbox.client_commit_id,
-                status: pushRes.status,
-                response: pushRes,
-            }));
+    if (preparedFirstCommit) {
+        if (wsPushResponse) {
+            const finalizedFirstCommit = await finalizePushCommit(db, {
+                clientId: options.clientId,
+                actorId: options.actorId,
+                plugins,
+            }, preparedFirstCommit, wsPushResponse);
+            pushResults.push(finalizedFirstCommit.pushResult);
             pushedCommits = 1;
+            if (finalizedFirstCommit.afterPushError) {
+                throw finalizedFirstCommit.afterPushError;
+            }
         }
         else {
-            // Check if all errors are retriable
-            const errorResults = pushRes.results.filter((r) => r.status === 'error');
-            const allRetriable = errorResults.length > 0 &&
-                errorResults.every((r) => r.retriable === true);
-            if (allRetriable) {
-                await markOutboxCommitPending(db, {
-                    id: outbox.id,
-                    error: 'Retriable',
-                    responseJson,
-                });
-                pushResults.push(buildPushResult({
-                    outboxCommitId: outbox.id,
-                    clientCommitId: outbox.client_commit_id,
-                    status: 'retriable',
-                    response: pushRes,
-                }));
-                pushedCommits = 1;
+            const batchResponses = combined?.push?.commits ?? [];
+            const responsesByClientCommitId = new Map(batchResponses.map((response) => [response.clientCommitId, response]));
+            if (!combined?.push ||
+                combinedPushCommits.some(({ request }) => !responsesByClientCommitId.has(request.clientCommitId))) {
+                await markClaimedOutboxCommitsPending(db, combinedPushCommits.map(({ outboxCommit }) => outboxCommit), 'MISSING_PUSH_RESPONSE');
+                throw new Error('Server returned incomplete push response');
+            }
+            let deferredError = null;
+            for (const preparedCommit of combinedPushCommits) {
+                const batchResponse = responsesByClientCommitId.get(preparedCommit.request.clientCommitId);
+                if (!batchResponse)
+                    continue;
+                const finalizedCommit = await finalizePushCommit(db, {
+                    clientId: options.clientId,
+                    actorId: options.actorId,
+                    plugins,
+                }, preparedCommit, toSyncPushResponse(batchResponse));
+                pushResults.push(finalizedCommit.pushResult);
+                pushedCommits += 1;
+                if (!deferredError && finalizedCommit.afterPushError) {
+                    deferredError = finalizedCommit.afterPushError;
+                }
             }
-            else {
-                await upsertConflictsForRejectedCommit(db, {
-                    outboxCommitId: outbox.id,
-                    clientCommitId: outbox.client_commit_id,
-                    response: pushRes,
-                });
-                await markOutboxCommitFailed(db, {
-                    id: outbox.id,
-                    error: 'REJECTED',
-                    responseJson,
-                });
-                pushResults.push(buildPushResult({
-                    outboxCommitId: outbox.id,
-                    clientCommitId: outbox.client_commit_id,
-                    status: 'rejected',
-                    response: pushRes,
-                }));
-                pushedCommits = 1;
+            if (deferredError) {
+                throw deferredError;
             }
         }
         // Settle remaining outbox commits
-        const remaining = await syncPushUntilSettled(db, transport, {
-            clientId: options.clientId,
-            actorId: options.actorId,
-            plugins: options.plugins,
-            maxCommits: (options.maxPushCommits ?? 20) - 1,
-        });
-        pushedCommits += remaining.pushedCount;
-        pushResults.push(...remaining.pushResults);
+        const remainingMaxCommits = Math.max(0, (options.maxPushCommits ?? 20) - pushedCommits);
+        if (remainingMaxCommits > 0) {
+            const remaining = await syncPushUntilSettled(db, transport, {
+                clientId: options.clientId,
+                actorId: options.actorId,
+                plugins: options.plugins,
+                maxCommits: remainingMaxCommits,
+            });
+            pushedCommits += remaining.pushedCount;
+            pushResults.push(...remaining.pushResults);
+        }
     }
     // Process pull response
     let pullResponse = { ok: true, subscriptions: [] };
     let pullRounds = 0;
-    if (combined.pull) {
+    if (combined?.pull) {
         pullResponse = await applyPullResponse(db, transport, handlers, pullOpts, pullState, combined.pull);
         pullRounds = 1;
         // Continue pulling if more data
-        if (needsAnotherPull(pullResponse)) {
+        if (needsAnotherPull(pullResponse, pullState.request.limitCommits)) {
             const aggregatedBySubId = new Map();
             mergePullResponse(aggregatedBySubId, pullResponse);
             const more = await syncPullUntilSettled(db, transport, handlers, {