@rebasepro/client 0.13.0 → 0.13.1-canary.g06dbe5b

package/src/offline.test.ts

@@ -14,9 +14,9 @@ type Row = Record<string, unknown>;
  * online/offline decision logic rather than a mock of it.
  */
 function createFakeServer() {
-    const state = { online: true };
+    const state: { online: boolean; rejectNextCreate?: Error; rejectEveryCreate?: Error } = { online: true };
     const tables = new Map<string, Map<string, Row>>();
-    const log: { collection: string; op: string; id?: unknown; upsert?: boolean }[] = [];
+    const log: { collection: string; op: string; id?: unknown; ids?: unknown[]; upsert?: boolean }[] = [];
 
     function table(slug: string): Map<string, Row> {
         if (!tables.has(slug)) tables.set(slug, new Map());
@@ -41,6 +41,13 @@ function createFakeServer() {
             },
             async create(data: Partial<Row>, id?: string | number) {
                 assertOnline();
+                // Lets a test answer the first replay the way a server does
+                // when it is still holding this write's idempotency key.
+                const injected = state.rejectNextCreate ?? state.rejectEveryCreate;
+                if (injected) {
+                    state.rejectNextCreate = undefined;
+                    throw injected;
+                }
                 const rowId = id ?? data.id ?? `srv-${table(slug).size + 1}`;
                 log.push({ collection: slug, op: "create", id: rowId });
                 const row = { ...data, id: rowId };
@@ -67,11 +74,30 @@ function createFakeServer() {
                 table(slug).set(String(id), row);
                 return row;
             },
+            async updateMany(updates: { id: string | number; data: Partial<Row> }[]) {
+                assertOnline();
+                log.push({ collection: slug, op: "updateMany", ids: updates.map(u => u.id) });
+                return updates.map(({ id, data }) => {
+                    const existing = table(slug).get(String(id));
+                    if (!existing) throw new RebaseApiError("Not found", { status: 404 });
+                    const row = { ...existing, ...data };
+                    table(slug).set(String(id), row);
+                    return row;
+                });
+            },
             async delete(id: string | number) {
                 assertOnline();
                 log.push({ collection: slug, op: "delete", id });
                 table(slug).delete(String(id));
             },
+            async deleteMany(ids: (string | number)[]) {
+                assertOnline();
+                log.push({ collection: slug, op: "deleteMany", ids });
+                for (const id of ids) {
+                    if (!table(slug).has(String(id))) throw new RebaseApiError("Not found", { status: 404 });
+                }
+                for (const id of ids) table(slug).delete(String(id));
+            },
             async count(_params?: FindParams) {
                 assertOnline();
                 return table(slug).size;
@@ -292,6 +318,64 @@ describe("OfflineManager", () => {
             expect(server.table("posts").get("p9")).toMatchObject({ title: "kept" });
         });
 
+        it("keeps a write the server is still answering, and lands it on the next flush", async () => {
+            // `IDEMPOTENCY_KEY_IN_PROGRESS` means the server holds this write's
+            // key and has not answered yet — the row may not exist at all. Read
+            // as any other 409 it looked like "your row is already there": the
+            // queue looked the row up, found nothing, decided there was nothing
+            // left to do and deleted the write. The record survived only in the
+            // local store and disappeared at the next reconciliation, silently.
+            const server = createFakeServer();
+            const onSyncError = jest.fn<(error: Error, mutation: PendingMutation) => void>();
+            const { manager, wrap } = createManager(server, { onSyncError });
+
+            server.state.online = false;
+            await wrap("posts").create({ title: "queued" });
+
+            server.state.online = true;
+            server.state.rejectNextCreate = new RebaseApiError("still in progress", {
+                status: 409,
+                code: "IDEMPOTENCY_KEY_IN_PROGRESS"
+            });
+
+            const refused = await manager.sync();
+            expect(refused).toEqual({ flushed: 0, remaining: 1 });
+            expect(onSyncError).not.toHaveBeenCalled();
+            expect(server.table("posts").size).toBe(0);
+
+            // The claim expired (or its request finally answered): the same
+            // queued write goes through untouched.
+            const settled = await manager.sync();
+            expect(settled).toEqual({ flushed: 1, remaining: 0 });
+            expect([...server.table("posts").values()]).toMatchObject([{ title: "queued" }]);
+        });
+
+        it("outlasts the claim's lease rather than giving up on the default retry budget", async () => {
+            // Retries double from a second, so the five a busy server gets are
+            // spent inside half a minute — less than the lease the server gives
+            // a claim whose process died. Spending them here would roll back
+            // the write a few seconds before the key became free again.
+            const server = createFakeServer();
+            const onSyncError = jest.fn<(error: Error, mutation: PendingMutation) => void>();
+            const { manager, wrap } = createManager(server, { onSyncError });
+
+            server.state.online = false;
+            await wrap("posts").create({ title: "queued" });
+
+            server.state.online = true;
+            server.state.rejectEveryCreate = new RebaseApiError("still in progress", {
+                status: 409,
+                code: "IDEMPOTENCY_KEY_IN_PROGRESS"
+            });
+            for (let attempt = 0; attempt < 8; attempt++) await manager.sync();
+
+            expect(onSyncError).not.toHaveBeenCalled();
+            expect(await manager.api.pending()).toHaveLength(1);
+
+            server.state.rejectEveryCreate = undefined;
+            expect(await manager.sync()).toEqual({ flushed: 1, remaining: 0 });
+        });
+
         it("notifies queue-size listeners as writes queue and drain", async () => {
             const server = createFakeServer();
             const { manager, wrap } = createManager(server);
@@ -503,6 +587,100 @@ describe("OfflineManager", () => {
         });
     });
 
+    describe("updateMany / deleteMany", () => {
+        it("queues a bulk update offline, overlays it, and replays it once", async () => {
+            const server = createFakeServer();
+            const { manager, wrap } = createManager(server);
+            const posts = wrap("posts");
+            await posts.create({ title: "a" });
+            await posts.create({ title: "b" });
+            const seeded = (await posts.find()).data;
+
+            server.state.online = false;
+            const updated = await posts.updateMany(
+                seeded.map((r) => ({ id: r.id as string, data: { title: `${r.title}!` } }))
+            );
+            expect(updated.map((r) => r.title)).toEqual(["a!", "b!"]);
+
+            // The overlay is what a UI renders while the write is queued.
+            const overlaid = await posts.find();
+            expect(overlaid.data.map((r) => r.title).sort()).toEqual(["a!", "b!"]);
+
+            server.state.online = true;
+            await manager.sync();
+            const bulk = server.log.filter((e) => e.op === "updateMany");
+            expect(bulk).toHaveLength(1);
+            expect([...server.table("posts").values()].map((r) => r.title).sort()).toEqual(["a!", "b!"]);
+        });
+
+        it("queues a bulk delete offline, hides the rows, and replays it once", async () => {
+            const server = createFakeServer();
+            const { manager, wrap } = createManager(server);
+            const posts = wrap("posts");
+            await posts.create({ title: "a" });
+            await posts.create({ title: "b" });
+            await posts.create({ title: "c" });
+            const seeded = (await posts.find()).data;
+
+            server.state.online = false;
+            await posts.deleteMany(seeded.slice(0, 2).map((r) => r.id as string));
+
+            const overlaid = await posts.find();
+            expect(overlaid.data.map((r) => r.title)).toEqual(["c"]);
+
+            server.state.online = true;
+            await manager.sync();
+            const bulk = server.log.filter((e) => e.op === "deleteMany");
+            expect(bulk).toHaveLength(1);
+            expect([...server.table("posts").values()].map((r) => r.title)).toEqual(["c"]);
+        });
+
+        it("queues a bulk update whose rows have writes still pending", async () => {
+            // Splitting the batch — some rows now, some later — would break the
+            // one guarantee a batch makes, and would reorder a write against a
+            // row whose own create has not landed yet.
+            const server = createFakeServer();
+            const { manager, wrap } = createManager(server);
+            const posts = wrap("posts");
+            await posts.find();
+
+            server.state.online = false;
+            const created = await posts.create({ title: "fresh" });
+            await posts.updateMany([{ id: created.id as string, data: { title: "edited" } }]);
+
+            server.state.online = true;
+            await manager.sync();
+
+            // The create ran first, then the batch — order preserved.
+            expect(server.log.map((e) => e.op)).toEqual(["create", "updateMany"]);
+            expect([...server.table("posts").values()][0].title).toBe("edited");
+        });
+
+        it("goes straight to the server when online", async () => {
+            const server = createFakeServer();
+            const { wrap } = createManager(server);
+            const posts = wrap("posts");
+            const a = await posts.create({ title: "a" });
+
+            await posts.updateMany([{ id: a.id as string, data: { title: "A" } }]);
+            expect(server.table("posts").get(String(a.id))!.title).toBe("A");
+
+            await posts.deleteMany([a.id as string]);
+            expect(server.table("posts").size).toBe(0);
+        });
+
+        it("treats empty input as a no-op that touches neither queue nor server", async () => {
+            const server = createFakeServer();
+            const { wrap } = createManager(server);
+            const posts = wrap("posts");
+            await posts.find();
+
+            expect(await posts.updateMany([])).toEqual([]);
+            await posts.deleteMany([]);
+            expect(server.log.filter((e) => e.op.endsWith("Many"))).toHaveLength(0);
+        });
+    });
+
     describe("concurrency", () => {
         it("gives concurrent offline writes distinct, ordered mutation ids", async () => {
             const server = createFakeServer();

package/src/offline.ts

@@ -1,10 +1,16 @@
 import { buildQueryString, FindParams, RebaseApiError } from "./transport";
-import { FindAllParams, FindResult, IterateParams, LogicalCondition, SDKCollectionClient, WhereFilterOp, WhereValue } from "@rebasepro/types";
+import { FindAllParams, FindResult, IterateParams, LogicalCondition, SDKCollectionClient, WhereFilterOp, WhereValueFor, WriteOptions } from "@rebasepro/types";
 import { collectAllPages, paginateFind } from "@rebasepro/common";
 import { CollectionClient, LiveResult, ObserveOptions, RowSnapshotMeta } from "./collection";
 import { SDKQueryBuilder } from "./sdk_query_builder";
 import { dehydrateRow, hydrateRow } from "./offline-codec";
-import { ConnectivityMonitor, isDuplicateKeyError, isNetworkError, isRetryableError } from "./offline-connectivity";
+import {
+    ConnectivityMonitor,
+    isDuplicateKeyError,
+    isIdempotencyInProgressError,
+    isNetworkError,
+    isRetryableError
+} from "./offline-connectivity";
 import {
     IndexedDBOfflineStore,
     MemoryOfflineStore,
@@ -14,7 +20,9 @@ import {
 } from "./offline-store";
 import {
     isExactlyEvaluable,
+    isLocallySortable,
     matchesParams,
+    resolvePagination,
     runLocalQuery,
     sortRows
 } from "./offline-query";
@@ -217,6 +225,18 @@ interface Observer {
 // all 1,700 lines of it in silence.
 const MISSING = "\u0000missing";
 
+/**
+ * Replays to spend on a mutation whose idempotency key the server is still
+ * holding, when the app has not asked for more.
+ *
+ * Retries double from a second and cap at the sync interval, so the default
+ * budget of five covers about half a minute — less than the lease a server
+ * gives a claim nobody came back for. This many outlast it with room for a slow
+ * batch, and the count is what stops a server that never releases the key from
+ * blocking the queue behind it indefinitely.
+ */
+const IN_PROGRESS_MIN_RETRIES = 12;
+
 export class OfflineManager {
     private readonly store: OfflineStore;
     private readonly maxCachedQueries: number;
@@ -545,6 +565,83 @@ export class OfflineManager {
                 return rows;
             },
 
+            updateMany: async (updates: { id: string | number; data: Partial<M> }[], options?: WriteOptions) => {
+                await this.ensureCollection(slug);
+                if (!Array.isArray(updates)) {
+                    throw new TypeError("updateMany expects an array of { id, data } entries.");
+                }
+                if (updates.length === 0) return [];
+                // Any row in the batch with a write already queued sends the
+                // whole batch to the queue. Splitting it — some rows now, some
+                // later — would break the one guarantee a batch makes, that its
+                // rows land together, and would reorder writes against a row
+                // whose own create has not landed yet.
+                const anyPending = updates.some((u) => this.hasPending(slug, u.id));
+                if (this.connectivity.shouldAttempt() && !anyPending) {
+                    try {
+                        const rows = await inner.updateMany(updates, options);
+                        this.connectivity.markSuccess();
+                        await this.ingest(slug, rows);
+                        this.notifyCollection(slug);
+                        return rows;
+                    } catch (error) {
+                        if (!isNetworkError(error)) throw error;
+                        this.connectivity.markFailure();
+                    }
+                }
+                const rollback: Record<string, AnyRow | null> = {};
+                const optimistic: M[] = [];
+                for (const { id, data } of updates) {
+                    const base = this.rawLocalRow(slug, id);
+                    rollback[String(id)] = base ?? null;
+                    optimistic.push({ ...(base ?? {}), ...(data as AnyRow), id } as unknown as M);
+                }
+                await this.enqueue({
+                    collection: slug,
+                    type: "updateMany",
+                    updates: updates.map((u) => ({ id: u.id,
+data: u.data as AnyRow })),
+                    rollback: { rows: rollback }
+                });
+                for (const row of optimistic) this.setLocalRow(slug, row.id as string | number, row);
+                this.notifyCollection(slug);
+                return optimistic;
+            },
+
+            deleteMany: async (ids: (string | number)[], options?: WriteOptions) => {
+                await this.ensureCollection(slug);
+                if (!Array.isArray(ids)) {
+                    throw new TypeError("deleteMany expects an array of ids.");
+                }
+                if (ids.length === 0) return;
+                const anyPending = ids.some((id) => this.hasPending(slug, id));
+                if (this.connectivity.shouldAttempt() && !anyPending) {
+                    try {
+                        await inner.deleteMany(ids, options);
+                        this.connectivity.markSuccess();
+                        for (const id of ids) this.removeLocalRow(slug, id, true);
+                        this.notifyCollection(slug);
+                        this.scheduleRefresh(slug);
+                        return;
+                    } catch (error) {
+                        if (!isNetworkError(error)) throw error;
+                        this.connectivity.markFailure();
+                    }
+                }
+                const rollback: Record<string, AnyRow | null> = {};
+                for (const id of ids) {
+                    rollback[String(id)] = this.rawLocalRow(slug, id) ?? null;
+                }
+                await this.enqueue({
+                    collection: slug,
+                    type: "deleteMany",
+                    ids,
+                    rollback: { rows: rollback }
+                });
+                for (const id of ids) this.removeLocalRow(slug, id, false);
+                this.notifyCollection(slug);
+            },
+
             update: async (id: string | number, data: Partial<M>) => {
                 await this.ensureCollection(slug);
                 // Never overtake a write already queued for this row. The
@@ -651,13 +748,14 @@ export class OfflineManager {
                 return builder.where(
                     columnOrCondition as keyof M & string,
                     operator!,
-                    value as WhereValue<M[keyof M & string]>
+                    value as WhereValueFor<WhereFilterOp, M[keyof M & string]>
                 );
             },
             orderBy: (column, direction) => new SDKQueryBuilder<M>(wrapped).orderBy(column, direction),
             limit: (count) => new SDKQueryBuilder<M>(wrapped).limit(count),
             offset: (count) => new SDKQueryBuilder<M>(wrapped).offset(count),
-            search: (searchString) => new SDKQueryBuilder<M>(wrapped).search(searchString),
+            search: (searchString, options) => new SDKQueryBuilder<M>(wrapped).search(searchString, options),
+            vectorSearch: (property, vector, options) => new SDKQueryBuilder<M>(wrapped).vectorSearch(property, vector, options),
             include: (...relations) => new SDKQueryBuilder<M>(wrapped).include(...relations)
         };
 
@@ -942,7 +1040,7 @@ export class OfflineManager {
         if (!state) {
             return {
                 data: [],
-                meta: { total: 0, limit: params?.limit ?? 20, offset: params?.offset ?? 0, hasMore: false },
+                meta: { ...resolvePagination(params), total: 0, hasMore: false },
                 fromCache: true,
                 hasPendingWrites: false,
                 partial: true
@@ -996,9 +1094,28 @@ export class OfflineManager {
                 rows.push(entry.row as M);
                 added++;
             }
-            if (added > 0 && params?.orderBy) sortRows(rows, params.orderBy);
         }
 
+        // Order is part of the query, not a detail of how the rows were
+        // obtained. This used to sort only when a locally-created row had been
+        // injected — every other read handed back cache order, which is
+        // insertion order, and a caller that asked for `orderBy` got whatever
+        // the store happened to hold. In the admin that is the collection's
+        // `sort` being silently ignored on every list backed by this overlay:
+        // the query carries it, the server honours it, and the answer served
+        // from here did not.
+        //
+        // …but only when the local sort would land where the server's did.
+        // `snapshot.ids` already arrived in the server's order, so re-sorting a
+        // text column with `Intl.Collator` *replaces* a correct order with a
+        // possibly different one — under the C collation Postgres puts
+        // `Banana` before `apple` and the collator does not. When the column
+        // cannot be ordered locally the snapshot's order is the better answer,
+        // and the result says so rather than presenting it as the sorted page
+        // that was asked for.
+        const orderIsLocal = isLocallySortable(rows, params?.orderBy);
+        if (params?.orderBy && orderIsLocal) sortRows(rows, params.orderBy);
+
         const total = Math.max(rows.length, snapshot.total - removed + added);
         return {
             data: rows,
@@ -1010,7 +1127,9 @@ export class OfflineManager {
             },
             fromCache,
             hasPendingWrites: rows.some((row) => this.hasPending(slug, row.id as string | number)),
-            partial: !exact
+            // Not the page that was asked for if either the membership
+            // decision or the order could not be reproduced here.
+            partial: !exact || !orderIsLocal
         };
     }
 
@@ -1150,12 +1269,13 @@ export class OfflineManager {
     }
 
     private recordSnapshot(slug: string, params: FindParams | undefined, result: FindResult<AnyRow>): QuerySnapshot {
-        const meta = result.meta ?? { total: result.data?.length ?? 0, limit: 20, offset: 0, hasMore: false };
+        const window = resolvePagination(params);
+        const meta = result.meta ?? { total: result.data?.length ?? 0, ...window, hasMore: false };
         const snapshot: QuerySnapshot = {
             ids: (result.data ?? []).map((row) => row.id as string | number).filter((id) => id !== undefined),
             total: meta.total ?? result.data?.length ?? 0,
-            limit: meta.limit ?? params?.limit ?? 20,
-            offset: meta.offset ?? params?.offset ?? 0,
+            limit: meta.limit ?? window.limit,
+            offset: meta.offset ?? window.offset,
             hasMore: meta.hasMore ?? false
         };
         const state = this.collectionState(slug);
@@ -1390,7 +1510,18 @@ export class OfflineManager {
                         }
                         op.attempts = (op.attempts ?? 0) + 1;
                         op.lastError = (error as Error)?.message ?? String(error);
-                        if (isRetryableError(error) && op.attempts < this.maxRetries) {
+                        // A key the server is still holding gets a longer
+                        // budget than a busy server does. The claim outlives
+                        // the request that took it — the process was killed
+                        // between the write and its answer — so it is refused
+                        // until the claim's lease runs out, which is longer
+                        // than the default five retries reach. Giving up on
+                        // that schedule rolls back precisely the write the key
+                        // exists to save.
+                        const limit = isIdempotencyInProgressError(error)
+                            ? Math.max(this.maxRetries, IN_PROGRESS_MIN_RETRIES)
+                            : this.maxRetries;
+                        if (isRetryableError(error) && op.attempts < limit) {
                             // The server is busy, not unhappy. Keep the op — and
                             // its place in line, since later writes may depend on
                             // it — and come back after a backoff.
@@ -1466,13 +1597,39 @@ export class OfflineManager {
             await this.adoptServerRow(op, op.id, row);
         } else if (op.type === "createMany") {
             const queued = (op.data as AnyRow[]) ?? [];
-            const rows = await inner.createMany(queued, op.upsert ? { upsert: true } : undefined);
+            // The mutation id names this batch, exactly as it names a single
+            // `create` above — and it matters more here. Without it, a batch
+            // whose ACK went missing replays as a second genuine import and
+            // duplicates every row it holds, not one. `upsert` masked that for
+            // the callers who set it; nothing covered the ones who did not.
+            const rows = await inner.createMany(queued, {
+                ...(op.upsert ? { upsert: true } : {}),
+                idempotencyKey: op.mutationId
+            });
             for (let i = 0; i < rows.length; i++) {
                 await this.adoptServerRow(op, queued[i]?.id as string | number | undefined, rows[i]);
             }
+        } else if (op.type === "updateMany") {
+            const queued = op.updates ?? [];
+            // Keyed like every other replay: an update re-applied in full is
+            // naturally idempotent, but one interleaved with another writer's is
+            // not, and a lost ACK would otherwise re-apply a stale batch over
+            // newer data.
+            const rows = await inner.updateMany(
+                queued.map(u => ({ id: u.id,
+data: u.data as AnyRow })),
+                { idempotencyKey: op.mutationId }
+            );
+            for (let i = 0; i < rows.length; i++) {
+                await this.ingestReplaced(op, queued[i].id, rows[i]);
+            }
         } else if (op.type === "update") {
             const row = await inner.update(op.id!, op.data as AnyRow);
             await this.ingestReplaced(op, op.id!, row);
+        } else if (op.type === "deleteMany") {
+            const ids = op.ids ?? [];
+            await inner.deleteMany(ids, { idempotencyKey: op.mutationId });
+            for (const id of ids) this.removeLocalRow(op.collection, id, true);
         } else if (op.type === "delete") {
             await inner.delete(op.id!);
             this.removeLocalRow(op.collection, op.id!, true);