liwi-mongo 13.4.0 → 14.0.0

package/src/MongoStore.ts

@@ -1,15 +1,20 @@
+import { NotFoundError, ProtectedFieldUpdateError } from "liwi-store";
 import type {
   AllowedKeyValue,
   CreateQueryOptions,
   Criteria,
+  DeleteResult,
   Fields,
   OptionalBaseModelKeysForInsert,
+  PartialUpdateOptions,
+  PartialUpdateResult,
   QueryParams,
   Sort,
   SubscribableStore,
   Update,
   UpsertPartialObject,
   UpsertResult,
+  WriteResult,
 } from "liwi-store";
 import type {
   Collection,
@@ -38,6 +43,20 @@ import type {
   MongoIndexSyncResult,
   SyncIndexesOptions,
 } from "./indexes/types.ts";
+import { createReplacePipeline } from "./writes/createReplacePipeline.ts";
+import { stampPartialUpdate } from "./writes/stampPartialUpdate.ts";
+
+const resolveWriteTime = (options?: PartialUpdateOptions): Date | undefined =>
+  options?.touchUpdated === false ? undefined : new Date();
+
+// the driver stores `undefined` as `null` (`ignoreUndefined` is off)
+const toStoredValues = <T extends object>(values: T): T =>
+  Object.fromEntries(
+    Object.entries(values).map(([field, value]: [string, unknown]) => [
+      field,
+      value === undefined ? null : value,
+    ]),
+  ) as T;
 
 export type MongoUpsertResult<
   KeyValue extends AllowedKeyValue,
@@ -197,12 +216,42 @@ export default class MongoStore<
     return object as Model;
   }
 
+  /**
+   * Always sets `updated` to the write time and keeps the stored `created`: the
+   * object's own `created` and `updated` are ignored. The object is not mutated.
+   */
   async replaceOne(object: Model): Promise<Model> {
-    if (!object.updated) object.updated = new Date();
+    const collection = await this.collection;
+    const next = await collection.findOneAndUpdate(
+      { _id: object._id } as Filter<Model>,
+      createReplacePipeline({ ...object, updated: new Date() }),
+      { returnDocument: "after" },
+    );
+    if (!next) {
+      throw new NotFoundError(`Document not found: ${String(object._id)}`);
+    }
+    return next as Model;
+  }
+
+  /** Same rules as `replaceOne`; `next` is derived from the atomic `prev`, no read back. */
+  async replaceOneWithInfo(
+    object: Model,
+  ): Promise<WriteResult<Model> | undefined> {
+    const replacement: Model = { ...object, updated: new Date() };
 
     const collection = await this.collection;
-    await collection.replaceOne({ _id: object._id } as Filter<Model>, object);
-    return object;
+    const prev = await collection.findOneAndUpdate(
+      { _id: object._id } as Filter<Model>,
+      createReplacePipeline(replacement),
+      { returnDocument: "before" },
+    );
+    if (!prev) return undefined;
+
+    const storedCreated = prev.created as Date | null | undefined;
+    return {
+      prev: prev as Model,
+      next: { ...replacement, created: storedCreated ?? replacement.created },
+    };
   }
 
   async upsertOne<
@@ -230,6 +279,12 @@ export default class MongoStore<
     object: UpsertPartialObject<MongoKeyPath, KeyValue, Model, K>,
     setOnInsertPartialObject?: Update<Model>["$setOnInsert"],
   ): Promise<MongoUpsertResult<KeyValue, Model>> {
+    if (setOnInsertPartialObject && "updated" in setOnInsertPartialObject) {
+      throw new ProtectedFieldUpdateError(
+        'Cannot use $setOnInsert on "updated": upsert always sets it with $set, pass it in the object instead',
+      );
+    }
+
     const $setOnInsert: Update<Model>["$setOnInsert"] = {
       // @ts-expect-error -- created is Date as set in BaseModel
       created: object.created || new Date(),
@@ -260,7 +315,7 @@ export default class MongoStore<
       };
     }
 
-    const next = { ...prev, ...$set };
+    const next = { ...prev, ...toStoredValues($set) };
 
     return {
       resolvedAs: "updated",
@@ -278,45 +333,86 @@ export default class MongoStore<
     key: KeyValue,
     partialUpdate: Update<Model>,
     criteria?: Criteria<Model>,
+    options?: PartialUpdateOptions,
   ): Promise<Model> {
+    const update = stampPartialUpdate(partialUpdate, resolveWriteTime(options));
     const collection = await this.collection;
-    const commandResult = await collection.updateOne(
+    const next = await collection.findOneAndUpdate(
       { _id: key, ...criteria } as Filter<Model>,
-      partialUpdate as UpdateFilter<Model>,
+      update as UpdateFilter<Model>,
+      { returnDocument: "after" },
     );
-    if (!commandResult.acknowledged) {
-      console.error(commandResult);
-      throw new Error("Update failed");
-    }
-    const object = await this.findByKey(key);
-    return object!;
+    if (!next) throw new NotFoundError(`Document not found: ${String(key)}`);
+    return next as Model;
+  }
+
+  /**
+   * `prev` is read atomically with the write. `next` is read right after it,
+   * so it may already include a later concurrent write, which emits its own change.
+   * Resolves `undefined` when no document matches.
+   */
+  async partialUpdateByKeyWithInfo(
+    key: KeyValue,
+    partialUpdate: Update<Model>,
+    criteria?: Criteria<Model>,
+    options?: PartialUpdateOptions,
+  ): Promise<PartialUpdateResult<Model> | undefined> {
+    const update = stampPartialUpdate(partialUpdate, resolveWriteTime(options));
+    const collection = await this.collection;
+    const prev = await collection.findOneAndUpdate(
+      { _id: key, ...criteria } as Filter<Model>,
+      update as UpdateFilter<Model>,
+      { returnDocument: "before" },
+    );
+    if (!prev) return undefined;
+
+    return { prev: prev as Model, next: await this.findByKey(key) };
   }
 
   partialUpdateOne(
     object: Model,
     partialUpdate: Update<Model>,
+    options?: PartialUpdateOptions,
   ): Promise<Model> {
-    return this.partialUpdateByKey(object._id, partialUpdate);
+    return this.partialUpdateByKey(
+      object._id,
+      partialUpdate,
+      undefined,
+      options,
+    );
   }
 
-  partialUpdateMany(
+  /** One `updateMany` command; every document gets the same `updated`. */
+  async partialUpdateMany(
     criteria: Criteria<Model>,
     partialUpdate: Update<Model>,
+    options?: PartialUpdateOptions,
   ): Promise<void> {
-    return this.collection
-      .then((collection) =>
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
-        collection.updateMany(criteria as Filter<Model>, partialUpdate as any),
-      )
-      .then((res) => undefined); // TODO return updated object
+    const update = stampPartialUpdate(partialUpdate, resolveWriteTime(options));
+    const collection = await this.collection;
+    await collection.updateMany(
+      criteria as Filter<Model>,
+      update as UpdateFilter<Model>,
+    );
   }
 
-  deleteByKey(key: KeyValue, criteria?: Criteria<Model>): Promise<void> {
-    return this.collection
-      .then((collection) =>
-        collection.deleteOne({ _id: key, ...criteria } as Filter<Model>),
-      )
-      .then(() => undefined);
+  async deleteByKey(key: KeyValue, criteria?: Criteria<Model>): Promise<void> {
+    const collection = await this.collection;
+    await collection.deleteOne({ _id: key, ...criteria } as Filter<Model>);
+  }
+
+  async deleteByKeyWithInfo(
+    key: KeyValue,
+    criteria?: Criteria<Model>,
+  ): Promise<DeleteResult<Model> | undefined> {
+    const collection = await this.collection;
+    const prev = await collection.findOneAndDelete({
+      _id: key,
+      ...criteria,
+    } as Filter<Model>);
+    if (!prev) return undefined;
+
+    return { prev: prev as Model };
   }
 
   deleteOne(object: Model): Promise<void> {

package/src/SubscribeStore.integration.test.ts

@@ -0,0 +1,414 @@
+import assert from "node:assert/strict";
+import { after, before, beforeEach, describe, it } from "node:test";
+import type { Changes } from "liwi-store";
+import { NotFoundError, ProtectedFieldUpdateError } from "liwi-store";
+import type { Actions } from "liwi-subscribe-store";
+import type { MongoMemoryServer } from "mongodb-memory-server-core";
+import type { MongoBaseModel } from "./MongoBaseModel.ts";
+import MongoConnection from "./MongoConnection.ts";
+import MongoStore from "./MongoStore.ts";
+import createMongoSubscribeStore from "./createMongoSubscribeStore.ts";
+
+interface Item extends MongoBaseModel {
+  label: string;
+  flag: boolean;
+  note?: string | null;
+  __v?: number;
+}
+
+describe(
+  "SubscribeStore over MongoStore (integration)",
+  {
+    skip: !process.env.LIWI_MONGO_INTEGRATION && "set LIWI_MONGO_INTEGRATION=1",
+    timeout: 120_000,
+  },
+  () => {
+    let mongod: MongoMemoryServer;
+    let connection: MongoConnection;
+    let mongoStore: MongoStore<Item>;
+    let subscribeStore: ReturnType<typeof createMongoSubscribeStore<Item>>;
+    let actions: Actions<Item>[];
+
+    before(async () => {
+      const { MongoMemoryServer: MemoryServer } =
+        await import("mongodb-memory-server-core");
+      mongod = await MemoryServer.create();
+      connection = new MongoConnection({
+        host: "127.0.0.1",
+        port: mongod.instanceInfo!.port,
+        database: "liwi-mongo-subscribe-store-test",
+      });
+      await connection.getConnection();
+      mongoStore = new MongoStore<Item>(connection, "items");
+    });
+
+    after(async () => {
+      await connection.close();
+      await mongod.stop();
+    });
+
+    beforeEach(async () => {
+      await mongoStore.deleteMany({});
+      subscribeStore = createMongoSubscribeStore(mongoStore);
+      await subscribeStore.insertOne({ _id: "1", label: "a", flag: true });
+      actions = [];
+      subscribeStore.subscribe((action) => actions.push(action));
+    });
+
+    const loadStored = async (): Promise<Item> =>
+      (await mongoStore.findByKey("1"))!;
+
+    it("emits the stored prev on partialUpdateOne of a mutated copy", async () => {
+      const stored = await loadStored();
+      const copy: Item = { ...stored, label: "mutated locally" };
+
+      const next = await subscribeStore.partialUpdateOne(copy, {
+        $set: { flag: false },
+      });
+
+      assert.deepEqual(next, { ...stored, flag: false, updated: next.updated });
+      assert.deepEqual(actions, [
+        { type: "updated", changes: [[stored, next]] },
+      ]);
+    });
+
+    it("emits the stored prev on replaceOne of a stale copy", async () => {
+      const stale = await loadStored();
+      await mongoStore.partialUpdateByKey("1", { $set: { label: "b" } });
+      const stored = await loadStored();
+
+      const next = await subscribeStore.replaceOne({ ...stale, flag: false });
+
+      assert.notEqual(next, stale);
+      assert.deepEqual(actions, [
+        { type: "updated", changes: [[stored, next]] },
+      ]);
+      assert.deepEqual(await loadStored(), next);
+    });
+
+    it("emits the stored prev on deleteOne of a stale copy", async () => {
+      const stale = await loadStored();
+      await mongoStore.partialUpdateByKey("1", { $set: { label: "b" } });
+      const stored = await loadStored();
+
+      await subscribeStore.deleteOne(stale);
+
+      assert.deepEqual(actions, [{ type: "deleted", prev: [stored] }]);
+      assert.equal(await mongoStore.findByKey("1"), undefined);
+    });
+
+    it("removes a document from a query collection when replaceOne makes it leave the criteria", async () => {
+      const received: Changes<string, Item[]>[] = [];
+      const query = subscribeStore.createQueryCollection<Item, never>({
+        criteria: { flag: true },
+      });
+      query.subscribe((error, changes) => {
+        if (error) throw error;
+        received.push(changes);
+      });
+
+      const copy = await loadStored();
+      copy.flag = false;
+      await subscribeStore.replaceOne(copy);
+
+      assert.deepEqual(received, [[{ type: "deleted", keys: ["1"] }]]);
+    });
+
+    it("emits nothing for writes on a missing key", async () => {
+      await assert.rejects(
+        subscribeStore.partialUpdateByKey("missing", { $set: { label: "x" } }),
+        NotFoundError,
+      );
+      await assert.rejects(
+        subscribeStore.replaceOne({ ...(await loadStored()), _id: "missing" }),
+        NotFoundError,
+      );
+      await subscribeStore.deleteByKey("missing");
+
+      assert.deepEqual(actions, []);
+    });
+
+    it("treats a key that no longer matches the criteria as missing", async () => {
+      await assert.rejects(
+        subscribeStore.partialUpdateByKey(
+          "1",
+          { $set: { label: "x" } },
+          { flag: false },
+        ),
+        NotFoundError,
+      );
+      await subscribeStore.deleteByKey("1", { flag: false });
+
+      const stored = await loadStored();
+      assert.deepEqual(actions, []);
+      assert.equal(stored.label, "a");
+    });
+
+    describe("bulk writes with a concurrent write between the key read and the writes", () => {
+      // runs `inject` right before the first per-document write, after the keys were read
+      const subscribeInjecting = (
+        inject: () => Promise<unknown>,
+      ): {
+        injectingSubscribeStore: ReturnType<
+          typeof createMongoSubscribeStore<Item>
+        >;
+        emitted: Actions<Item>[];
+      } => {
+        let isInjected = false;
+        const injectOnce = async (): Promise<void> => {
+          if (isInjected) return;
+          isInjected = true;
+          await inject();
+        };
+        const injectingStore: MongoStore<Item> = Object.assign(
+          Object.create(mongoStore) as MongoStore<Item>,
+          {
+            partialUpdateByKeyWithInfo: async (
+              ...args: Parameters<
+                MongoStore<Item>["partialUpdateByKeyWithInfo"]
+              >
+            ) => {
+              await injectOnce();
+              return mongoStore.partialUpdateByKeyWithInfo(...args);
+            },
+            deleteByKeyWithInfo: async (
+              ...args: Parameters<MongoStore<Item>["deleteByKeyWithInfo"]>
+            ) => {
+              await injectOnce();
+              return mongoStore.deleteByKeyWithInfo(...args);
+            },
+          },
+        );
+        const injectingSubscribeStore =
+          createMongoSubscribeStore(injectingStore);
+        const emitted: Actions<Item>[] = [];
+        injectingSubscribeStore.subscribe((action) => emitted.push(action));
+        return { injectingSubscribeStore, emitted };
+      };
+
+      const injectConcurrentWrites = async (): Promise<void> => {
+        await mongoStore.partialUpdateByKey("2", { $set: { flag: false } });
+        await mongoStore.insertOne({ _id: "4", label: "late", flag: true });
+      };
+
+      beforeEach(async () => {
+        await mongoStore.insertOne({ _id: "2", label: "b", flag: true });
+        await mongoStore.insertOne({ _id: "3", label: "c", flag: false });
+      });
+
+      it("deleteMany emits exactly the documents it deleted", async () => {
+        const stored1 = await loadStored();
+        const { injectingSubscribeStore, emitted } = subscribeInjecting(
+          injectConcurrentWrites,
+        );
+
+        await injectingSubscribeStore.deleteMany({ flag: true });
+
+        const remaining = await mongoStore.findAll({}, { _id: 1 });
+        assert.deepEqual(
+          remaining.map(({ _id }) => _id),
+          ["2", "3", "4"],
+        );
+        assert.deepEqual(emitted, [{ type: "deleted", prev: [stored1] }]);
+      });
+
+      it("partialUpdateMany emits exactly the documents it updated, with their true prev", async () => {
+        const stored1 = await loadStored();
+        const { injectingSubscribeStore, emitted } = subscribeInjecting(
+          injectConcurrentWrites,
+        );
+
+        await injectingSubscribeStore.partialUpdateMany(
+          { flag: true },
+          { $set: { label: "bulk" } },
+        );
+
+        const documents = await mongoStore.findAll({}, { _id: 1 });
+        assert.deepEqual(
+          documents.map(({ _id, label }) => [_id, label]),
+          [
+            ["1", "bulk"],
+            ["2", "b"],
+            ["3", "c"],
+            ["4", "late"],
+          ],
+        );
+        assert.deepEqual(emitted, [
+          { type: "updated", changes: [[stored1, documents[0]]] },
+        ]);
+      });
+    });
+
+    describe("write metadata", () => {
+      const oldCreated = new Date("2019-01-01");
+      const oldUpdated = new Date("2020-01-01");
+
+      const resetOld = async (): Promise<Item> => {
+        await mongoStore.deleteByKey("old");
+        return mongoStore.insertOne({
+          _id: "old",
+          label: "old",
+          flag: true,
+          created: oldCreated,
+          updated: oldUpdated,
+        });
+      };
+
+      const loadOld = async (): Promise<Item> =>
+        (await mongoStore.findByKey("old"))!;
+
+      const writes: [string, (old: Item) => Promise<unknown>][] = [
+        [
+          "MongoStore.partialUpdateByKey",
+          () => mongoStore.partialUpdateByKey("old", { $set: { label: "x" } }),
+        ],
+        [
+          "SubscribeStore.partialUpdateOne",
+          (old) =>
+            subscribeStore.partialUpdateOne(old, { $set: { label: "x" } }),
+        ],
+        [
+          "MongoStore.partialUpdateMany",
+          () =>
+            mongoStore.partialUpdateMany({ _id: "old" }, { $inc: { __v: 1 } }),
+        ],
+        [
+          "SubscribeStore.partialUpdateMany",
+          () =>
+            subscribeStore.partialUpdateMany(
+              { _id: "old" },
+              { $set: { label: "x" } },
+            ),
+        ],
+        [
+          "MongoStore.replaceOne with a loaded document",
+          (old) => mongoStore.replaceOne({ ...old, label: "x" }),
+        ],
+        [
+          "SubscribeStore.replaceOne with another created",
+          (old) =>
+            subscribeStore.replaceOne({
+              ...old,
+              created: new Date("2025-01-01"),
+            }),
+        ],
+        [
+          "upsertOne without updated",
+          () =>
+            subscribeStore.upsertOne({ _id: "old", label: "x", flag: false }),
+        ],
+      ];
+
+      for (const [name, write] of writes) {
+        it(`${name} moves updated forward and keeps created`, async () => {
+          const old = await resetOld();
+          const writeStart = Date.now();
+
+          await write(old);
+
+          const stored = await loadOld();
+          assert.ok(stored.updated.getTime() >= writeStart);
+          assert.deepEqual(stored.created, oldCreated);
+        });
+      }
+
+      it("keeps updated with touchUpdated: false", async () => {
+        await resetOld();
+
+        await mongoStore.partialUpdateMany(
+          { _id: "old" },
+          { $set: { label: "backfill" } },
+          { touchUpdated: false },
+        );
+        await subscribeStore.partialUpdateByKey(
+          "old",
+          { $set: { flag: false } },
+          undefined,
+          { touchUpdated: false },
+        );
+
+        const stored = await loadOld();
+        assert.deepEqual(stored.updated, oldUpdated);
+        assert.equal(stored.label, "backfill");
+      });
+
+      it("stores an explicit updated set by the caller", async () => {
+        await resetOld();
+        const callerTime = new Date("2024-06-01");
+
+        const afterSet = await mongoStore.partialUpdateByKey("old", {
+          $set: { updated: callerTime },
+        });
+        assert.deepEqual(afterSet.updated, callerTime);
+
+        const afterMax = await mongoStore.partialUpdateByKey("old", {
+          $max: { updated: new Date("2024-01-01") },
+        });
+        assert.deepEqual(afterMax.updated, callerTime);
+
+        const writeStart = Date.now();
+        const afterCurrentDate = await mongoStore.partialUpdateByKey("old", {
+          $currentDate: { updated: true },
+        });
+        // $currentDate uses the mongod clock; allow for a small skew with this process
+        assert.ok(afterCurrentDate.updated.getTime() >= writeStart - 1000);
+      });
+
+      it("rejects protected field updates before mongo sees them", async () => {
+        await resetOld();
+        const forbidden: Record<string, unknown>[] = [
+          { $set: { created: new Date() } },
+          { $unset: { updated: true } },
+          { $inc: { updated: 1 } },
+          { $min: { updated: new Date() } },
+          { $setOnInsert: { updated: new Date() } },
+          { $rename: { updated: "lastUpdate" } },
+          { $rename: { label: "created" } },
+        ];
+
+        for (const update of forbidden) {
+          await assert.rejects(
+            mongoStore.partialUpdateByKey("old", update),
+            ProtectedFieldUpdateError,
+            JSON.stringify(update),
+          );
+        }
+
+        const stored = await loadOld();
+        assert.deepEqual(stored.updated, oldUpdated);
+        assert.deepEqual(stored.created, oldCreated);
+      });
+
+      it("stores replacement values starting with $ as data", async () => {
+        const old = await resetOld();
+
+        const replaced = await mongoStore.replaceOne({
+          ...old,
+          label: "$label",
+        });
+        assert.equal(replaced.label, "$label");
+
+        await subscribeStore.replaceOne({
+          ...old,
+          flag: false,
+          label: "$flag",
+        });
+        const stored = await loadOld();
+        assert.equal(stored.label, "$flag");
+      });
+
+      it("returns the upserted next as stored, undefined values included", async () => {
+        await resetOld();
+
+        const result = await mongoStore.upsertOneWithInfo({
+          _id: "old",
+          label: "x",
+          flag: false,
+          note: undefined,
+        });
+
+        assert.deepEqual(result.object, await loadOld());
+      });
+    });
+  },
+);

package/src/indexes/diffIndexes.test.ts

@@ -256,9 +256,9 @@ describe("diffIndexes", () => {
           key: { _fts: "text", _ftsx: 1 },
           name: "label_text",
           weights: { label: 1 },
-          // eslint-disable-next-line camelcase -- mongo option name
+          // oxlint-disable-next-line eslint-js/camelcase -- mongo option name
           default_language: "english",
-          // eslint-disable-next-line camelcase -- mongo option name
+          // oxlint-disable-next-line eslint-js/camelcase -- mongo option name
           language_override: "language",
           textIndexVersion: 3,
         },

package/src/indexes/normalizeIndex.test.ts

@@ -101,9 +101,9 @@ describe("normalizeExistingIndex", () => {
       key: { completed: 1, _fts: "text", _ftsx: 1 },
       name: "completed_1_label_text",
       weights: { label: 1 },
-      // eslint-disable-next-line camelcase -- mongo option name
+      // oxlint-disable-next-line eslint-js/camelcase -- mongo option name
       default_language: "english",
-      // eslint-disable-next-line camelcase -- mongo option name
+      // oxlint-disable-next-line eslint-js/camelcase -- mongo option name
       language_override: "language",
       textIndexVersion: 3,
     });

package/src/indexes/normalizeIndex.ts

@@ -57,11 +57,11 @@ const buildDescription = (
   if (index.collation) description.collation = index.collation;
   if (index.weights) description.weights = index.weights;
   if (index.default_language) {
-    // eslint-disable-next-line camelcase -- mongo option name
+    // oxlint-disable-next-line eslint-js/camelcase -- mongo option name
     description.default_language = index.default_language;
   }
   if (index.language_override) {
-    // eslint-disable-next-line camelcase -- mongo option name
+    // oxlint-disable-next-line eslint-js/camelcase -- mongo option name
     description.language_override = index.language_override;
   }
   if (index.wildcardProjection) {

package/src/writes/createReplacePipeline.ts

@@ -0,0 +1,23 @@
+import type { Document } from "mongodb";
+import type { MongoBaseModel } from "../MongoBaseModel.ts";
+
+/**
+ * Replaces the stored document with `replacement` while keeping the stored `created`
+ * (falling back to the replacement's when the stored document has none).
+ * `$literal` keeps string values starting with `$` from being read as field paths.
+ * Update pipelines require MongoDB >= 4.2.
+ */
+export const createReplacePipeline = (
+  replacement: MongoBaseModel<any>,
+): Document[] => [
+  {
+    $replaceWith: {
+      $mergeObjects: [
+        { $literal: replacement },
+        {
+          created: { $ifNull: ["$created", { $literal: replacement.created }] },
+        },
+      ],
+    },
+  },
+];