liwi-mongo 13.1.0 → 13.2.0

package/src/MongoStore.ts

@@ -28,6 +28,16 @@ import type MongoConnection from "./MongoConnection.ts";
 import MongoCursor from "./MongoCursor.ts";
 import MongoQueryCollection from "./MongoQueryCollection.ts";
 import MongoQuerySingleItem from "./MongoQuerySingleItem.ts";
+import { applyIndexPlan } from "./indexes/applyIndexPlan.ts";
+import { diffIndexes } from "./indexes/diffIndexes.ts";
+import { isNamespaceNotFoundError } from "./indexes/isIndexConflictError.ts";
+import { normalizeDeclaredIndexes } from "./indexes/normalizeIndex.ts";
+import type {
+  MongoIndex,
+  MongoIndexPlan,
+  MongoIndexSyncResult,
+  SyncIndexesOptions,
+} from "./indexes/types.ts";
 
 export interface MongoUpsertResult<
   KeyValue extends AllowedKeyValue,
@@ -37,6 +47,10 @@ export interface MongoUpsertResult<
   inserted: boolean;
 }
 
+export interface MongoStoreOptions<Model extends MongoBaseModel<any>> {
+  indexes?: readonly MongoIndex<Model>[];
+}
+
 export default class MongoStore<
   Model extends MongoBaseModel<KeyValue>,
   KeyValue extends AllowedKeyValue = Model[MongoKeyPath],
@@ -51,15 +65,27 @@ export default class MongoStore<
 
   readonly connection: MongoConnection;
 
+  readonly collectionName: string;
+
+  private readonly declaredIndexes: readonly MongoIndex<Model>[];
+
   private _collection: Collection<Model> | Promise<Collection<Model>>;
 
-  constructor(connection: MongoConnection, collectionName: string) {
+  constructor(
+    connection: MongoConnection,
+    collectionName: string,
+    { indexes = [] }: MongoStoreOptions<Model> = {},
+  ) {
     this.connection = connection;
 
     if (!collectionName) {
       throw new Error(`Invalid collectionName: "${collectionName}"`);
     }
 
+    this.collectionName = collectionName;
+    this.declaredIndexes = indexes;
+    normalizeDeclaredIndexes({ collectionName, indexes });
+
     this._collection = connection.getConnection().then(
       (client: MongoClient) => {
         this._collection = client.db().collection(collectionName);
@@ -80,6 +106,42 @@ export default class MongoStore<
     return Promise.resolve(this._collection);
   }
 
+  async planIndexes({
+    dropUndeclaredIndexes = true,
+  }: SyncIndexesOptions = {}): Promise<MongoIndexPlan> {
+    const collection = await this.collection;
+
+    const existingIndexes = await collection
+      .listIndexes()
+      .toArray()
+      .catch((error: unknown) => {
+        if (isNamespaceNotFoundError(error)) return [];
+        throw error;
+      });
+
+    return diffIndexes<Model>({
+      collectionName: this.collectionName,
+      declaredIndexes: this.declaredIndexes,
+      existingIndexes,
+      dropUndeclaredIndexes,
+    });
+  }
+
+  async syncIndexes(
+    options: SyncIndexesOptions = {},
+  ): Promise<MongoIndexSyncResult> {
+    const plan = await this.planIndexes(options);
+    const collection = await this.collection;
+    const client = await this.connection.getConnection();
+
+    return applyIndexPlan({
+      plan,
+      collection: collection as Collection<any>,
+      db: client.db(),
+      dryRun: options.dryRun ?? false,
+    });
+  }
+
   createQuerySingleItem<
     Result extends Record<MongoKeyPath, KeyValue> = Model,
     Params extends QueryParams<Params> = never,

package/src/MongoStoreIndexes.test.ts

@@ -0,0 +1,259 @@
+import assert from "node:assert/strict";
+import { beforeEach, describe, it, mock } from "node:test";
+import type { IndexDescription, IndexDescriptionInfo } from "mongodb";
+import type { MongoBaseModel } from "./MongoBaseModel.ts";
+import type { MongoStoreOptions } from "./MongoStore.ts";
+
+interface TestModel extends MongoBaseModel {
+  completed: boolean;
+  label: string;
+  status: string;
+}
+
+const idIndex: IndexDescriptionInfo = { v: 2, key: { _id: 1 }, name: "_id_" };
+
+let calls: string[];
+let existingIndexes: IndexDescriptionInfo[];
+let createIndexesImplementation: (
+  descriptions: IndexDescription[],
+) => Promise<unknown>;
+
+const collection = {
+  find: () => ({ toArray: () => Promise.resolve([]) }),
+  countDocuments: () => Promise.resolve(0),
+  listIndexes: () => ({
+    toArray: () => {
+      calls.push("listIndexes");
+      return Promise.resolve(existingIndexes);
+    },
+  }),
+  createIndexes: (descriptions: IndexDescription[]) => {
+    calls.push(
+      `createIndexes:${descriptions.map(({ name }) => name).join(",")}`,
+    );
+    return createIndexesImplementation(descriptions);
+  },
+  dropIndex: (name: string) => {
+    calls.push(`dropIndex:${name}`);
+    return Promise.resolve({});
+  },
+};
+
+const db = {
+  collection: () => collection,
+  command: (command: Record<string, unknown>) => {
+    calls.push(`command:${JSON.stringify(command)}`);
+    return Promise.resolve({});
+  },
+};
+
+// eslint-disable-next-line n/no-unsupported-features/node-builtins
+mock.module("mongodb", {
+  // @ts-expect-error: not yet typed
+  exports: {
+    default: {
+      MongoClient: {
+        connect: () =>
+          Promise.resolve({
+            on: () => undefined,
+            db: () => db,
+          }),
+      },
+      ObjectId: class {
+        toString(): string {
+          return "generated-id";
+        }
+      },
+    },
+  },
+});
+
+const { default: MongoConnection } = await import("./MongoConnection.ts");
+const { default: MongoStore } = await import("./MongoStore.ts");
+const { default: MongoRegistry } = await import("./MongoRegistry.ts");
+
+const createStore = (
+  options?: MongoStoreOptions<TestModel>,
+): InstanceType<typeof MongoStore<TestModel>> =>
+  new MongoStore<TestModel>(
+    new MongoConnection({ database: "test" }),
+    "tasks",
+    options,
+  );
+
+describe("MongoStore indexes", () => {
+  beforeEach(() => {
+    calls = [];
+    existingIndexes = [idIndex];
+    createIndexesImplementation = () => Promise.resolve(["ok"]);
+  });
+
+  it("validates declared indexes in the constructor", () => {
+    assert.throws(
+      () =>
+        createStore({
+          indexes: [{ key: { label: 1 } }, { key: { label: 1 } }],
+        }),
+      /Duplicate index name/,
+    );
+  });
+
+  it("plans without mutating anything", async () => {
+    const store = createStore({ indexes: [{ key: { completed: 1 } }] });
+
+    const plan = await store.planIndexes();
+
+    assert.deepEqual(calls, ["listIndexes"]);
+    assert.deepEqual(
+      plan.toCreate.map(({ name }) => name),
+      ["completed_1"],
+    );
+  });
+
+  it("drops, then collMods, then creates", async () => {
+    existingIndexes = [
+      idIndex,
+      { v: 2, key: { status: 1 }, name: "legacy_status_1" },
+      { v: 2, key: { label: 1 }, name: "label_1", expireAfterSeconds: 3600 },
+    ];
+
+    const store = createStore({
+      indexes: [
+        { key: { completed: 1 } },
+        { key: { label: 1 }, expireAfterSeconds: 60 },
+      ],
+    });
+
+    await store.syncIndexes();
+
+    assert.deepEqual(calls, [
+      "listIndexes",
+      "dropIndex:legacy_status_1",
+      'command:{"collMod":"tasks","index":{"name":"label_1","expireAfterSeconds":60}}',
+      "createIndexes:completed_1",
+    ]);
+  });
+
+  it("never drops the _id index", async () => {
+    existingIndexes = [idIndex];
+    await createStore({ indexes: [] }).syncIndexes();
+
+    assert.equal(
+      calls.some((call) => call.startsWith("dropIndex")),
+      false,
+    );
+  });
+
+  it("mutates nothing on a dry run but still returns the plan", async () => {
+    existingIndexes = [
+      idIndex,
+      { v: 2, key: { status: 1 }, name: "legacy_status_1" },
+    ];
+
+    const result = await createStore({
+      indexes: [{ key: { completed: 1 } }],
+    }).syncIndexes({ dryRun: true });
+
+    assert.deepEqual(calls, ["listIndexes"]);
+    assert.equal(result.dryRun, true);
+    assert.deepEqual(
+      result.plan.toDrop.map(({ name }) => name),
+      ["legacy_status_1"],
+    );
+    assert.deepEqual(result.created, []);
+  });
+
+  it("keeps undeclared indexes when dropUndeclaredIndexes is false", async () => {
+    existingIndexes = [
+      idIndex,
+      { v: 2, key: { status: 1 }, name: "legacy_status_1" },
+    ];
+
+    await createStore({ indexes: [] }).syncIndexes({
+      dropUndeclaredIndexes: false,
+    });
+
+    assert.deepEqual(calls, ["listIndexes"]);
+  });
+
+  it("ignores an index already dropped by another process", async () => {
+    existingIndexes = [
+      idIndex,
+      { v: 2, key: { status: 1 }, name: "legacy_status_1" },
+    ];
+    const dropIndex = mock.method(collection, "dropIndex", () =>
+      Promise.reject(Object.assign(new Error("not found"), { code: 27 })),
+    );
+
+    const result = await createStore({ indexes: [] }).syncIndexes();
+
+    assert.equal(dropIndex.mock.callCount(), 1);
+    assert.deepEqual(result.dropped, []);
+    dropIndex.mock.restore();
+  });
+
+  it("recovers once from a conflicting index spec", async () => {
+    existingIndexes = [idIndex];
+    let createAttempts = 0;
+    createIndexesImplementation = () => {
+      createAttempts += 1;
+      if (createAttempts <= 2) {
+        existingIndexes = [
+          idIndex,
+          { v: 2, key: { completed: -1 }, name: "completed_1" },
+        ];
+        return Promise.reject(
+          Object.assign(new Error("conflict"), { code: 85 }),
+        );
+      }
+      return Promise.resolve(["ok"]);
+    };
+
+    const result = await createStore({
+      indexes: [{ key: { completed: 1 } }],
+    }).syncIndexes();
+
+    assert.equal(createAttempts, 3);
+    assert.deepEqual(calls.slice(1), [
+      "createIndexes:completed_1",
+      "createIndexes:completed_1",
+      "listIndexes",
+      "dropIndex:completed_1",
+      "createIndexes:completed_1",
+    ]);
+    assert.deepEqual(result.created, ["completed_1"]);
+  });
+});
+
+describe("MongoRegistry with mongo stores", () => {
+  beforeEach(() => {
+    calls = [];
+    existingIndexes = [idIndex];
+    createIndexesImplementation = () => Promise.resolve(["ok"]);
+  });
+
+  it("syncs every registered store", async () => {
+    const connection = new MongoConnection({ database: "test" });
+    const registry = new MongoRegistry([
+      new MongoStore<TestModel>(connection, "users", {
+        indexes: [{ key: { label: 1 } }],
+      }),
+      new MongoStore<TestModel>(connection, "tasks", {
+        indexes: [{ key: { completed: 1 } }],
+      }),
+    ]);
+
+    const results = await registry.syncIndexes();
+
+    assert.deepEqual(
+      results.map(({ collectionName }) => collectionName),
+      ["tasks", "users"],
+    );
+    assert.deepEqual(calls, [
+      "listIndexes",
+      "createIndexes:completed_1",
+      "listIndexes",
+      "createIndexes:label_1",
+    ]);
+  });
+});

package/src/index.ts

@@ -6,7 +6,7 @@ export type {
 
 export type { Update } from "liwi-store";
 
-export { default as MongoStore } from "./MongoStore.ts";
+export { default as MongoStore, type MongoStoreOptions } from "./MongoStore.ts";
 export {
   default as MongoConnection,
   type MongoConfig,
@@ -15,3 +15,28 @@ export {
   default as createMongoSubscribeStore,
   type SubscribeStore,
 } from "./createMongoSubscribeStore.ts";
+export {
+  default as MongoRegistry,
+  type MongoRegistryOptions,
+} from "./MongoRegistry.ts";
+export type { MongoRegistryStore } from "./MongoRegistryStore.ts";
+
+export type {
+  MongoIndex,
+  MongoIndexDirection,
+  MongoIndexKey,
+  MongoIndexOptions,
+  MongoIndexPlan,
+  MongoIndexSyncResult,
+  SyncIndexesOptions,
+} from "./indexes/types.ts";
+export { buildIndexName } from "./indexes/buildIndexName.ts";
+export { diffIndexes, isMongoIndexPlanEmpty } from "./indexes/diffIndexes.ts";
+export {
+  formatIndexPlan,
+  formatIndexPlans,
+} from "./indexes/formatIndexPlan.ts";
+export {
+  runIndexesCli,
+  type RunIndexesCliParams,
+} from "./indexes/runIndexesCli.ts";

package/src/indexes/applyIndexPlan.ts

@@ -0,0 +1,149 @@
+import type {
+  Collection,
+  Db,
+  IndexDescription,
+  IndexDescriptionInfo,
+} from "mongodb";
+import { Logger } from "nightingale-logger";
+import {
+  isIndexConflictError,
+  isIndexNotFoundError,
+} from "./isIndexConflictError.ts";
+import type { MongoIndexPlan, MongoIndexSyncResult } from "./types.ts";
+
+const logger = new Logger("liwi:mongo:indexes");
+
+const idIndexName = "_id_";
+
+const serializeKey = (key: IndexDescription["key"]): string =>
+  JSON.stringify(key instanceof Map ? [...key.entries()] : Object.entries(key));
+
+const findConflictingIndex = (
+  existingIndexes: readonly IndexDescriptionInfo[],
+  description: IndexDescription,
+): IndexDescriptionInfo | undefined => {
+  const byName = existingIndexes.find(({ name }) => name === description.name);
+  if (byName) return byName;
+
+  const serializedKey = serializeKey(description.key);
+  return existingIndexes.find(
+    ({ key, name }) =>
+      name !== idIndexName && serializeKey(key) === serializedKey,
+  );
+};
+
+interface CreateIndexParams {
+  collection: Collection<any>;
+  collectionName: string;
+  description: IndexDescription;
+}
+
+const createIndexWithConflictRecovery = async ({
+  collection,
+  collectionName,
+  description,
+}: CreateIndexParams): Promise<void> => {
+  try {
+    await collection.createIndexes([description]);
+    return;
+  } catch (error: unknown) {
+    if (!isIndexConflictError(error)) throw error;
+
+    logger.warn("index conflict, recovering", {
+      collectionName,
+      name: description.name,
+      error,
+    });
+
+    const existingIndexes = await collection.listIndexes().toArray();
+    const conflicting = findConflictingIndex(existingIndexes, description);
+    if (!conflicting?.name) throw error;
+
+    await collection.dropIndex(conflicting.name);
+  }
+
+  await collection.createIndexes([description]);
+};
+
+interface ApplyIndexPlanParams {
+  plan: MongoIndexPlan;
+  collection: Collection<any>;
+  db: Db;
+  dryRun: boolean;
+}
+
+export const applyIndexPlan = async ({
+  plan,
+  collection,
+  db,
+  dryRun,
+}: ApplyIndexPlanParams): Promise<MongoIndexSyncResult> => {
+  const { collectionName } = plan;
+  const result: MongoIndexSyncResult = {
+    collectionName,
+    plan,
+    dryRun,
+    created: [],
+    dropped: [],
+    modified: [],
+  };
+
+  if (dryRun) return result;
+
+  const namesToDrop = [...plan.toDrop, ...plan.toRecreate]
+    .map(({ name }) => name)
+    .filter((name) => name !== idIndexName);
+
+  for (const name of namesToDrop) {
+    try {
+      await collection.dropIndex(name);
+      logger.info("dropIndex", { collectionName, name });
+      result.dropped.push(name);
+    } catch (error: unknown) {
+      if (!isIndexNotFoundError(error)) throw error;
+      logger.warn("index already dropped", { collectionName, name });
+    }
+  }
+
+  for (const { name, changes } of plan.toCollMod) {
+    await db.command({
+      collMod: collectionName,
+      index: { name, ...changes },
+    });
+    logger.info("collMod", { collectionName, name, changes });
+    result.modified.push(name);
+  }
+
+  const descriptions = [...plan.toCreate, ...plan.toRecreate].map(
+    ({ index }) => index,
+  );
+
+  if (descriptions.length > 0) {
+    try {
+      await collection.createIndexes(descriptions);
+    } catch (error: unknown) {
+      logger.warn("createIndexes failed, retrying one by one", {
+        collectionName,
+        error,
+      });
+      for (const description of descriptions) {
+        await createIndexWithConflictRecovery({
+          collection,
+          collectionName,
+          description,
+        });
+      }
+    }
+    result.created.push(...descriptions.map(({ name }) => name!));
+    logger.info("createIndexes", { collectionName, names: result.created });
+  }
+
+  logger.info("syncIndexes done", {
+    collectionName,
+    created: result.created,
+    dropped: result.dropped,
+    modified: result.modified,
+  });
+
+  return result;
+};

package/src/indexes/buildIndexName.test.ts

@@ -0,0 +1,34 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { buildIndexName } from "./buildIndexName.ts";
+
+describe("buildIndexName", () => {
+  it("names a single ascending field like mongo does", () => {
+    assert.equal(buildIndexName({ completed: 1 }), "completed_1");
+  });
+
+  it("names a compound index", () => {
+    assert.equal(
+      buildIndexName({ completed: 1, created: 1 }),
+      "completed_1_created_1",
+    );
+  });
+
+  it("keeps descending directions", () => {
+    assert.equal(buildIndexName({ created: -1 }), "created_-1");
+  });
+
+  it("names dotted paths", () => {
+    assert.equal(buildIndexName({ "author.name": 1 }), "author.name_1");
+  });
+
+  it("names non numeric directions", () => {
+    assert.equal(buildIndexName({ label: "text" }), "label_text");
+    assert.equal(buildIndexName({ location: "2dsphere" }), "location_2dsphere");
+    assert.equal(buildIndexName({ tenant: "hashed" }), "tenant_hashed");
+  });
+
+  it("names wildcard indexes", () => {
+    assert.equal(buildIndexName({ "$**": 1 }), "$**_1");
+  });
+});

package/src/indexes/buildIndexName.ts

@@ -0,0 +1,8 @@
+import type { MongoIndexDirection } from "./types.ts";
+
+export const buildIndexName = (
+  key: Record<string, MongoIndexDirection>,
+): string =>
+  Object.entries(key)
+    .map(([field, direction]) => `${field}_${direction}`)
+    .join("_");

package/src/indexes/deepEqualUnordered.ts

@@ -0,0 +1,36 @@
+const isPlainRecord = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" &&
+  value !== null &&
+  !Array.isArray(value) &&
+  !(value instanceof Date) &&
+  !(value instanceof RegExp);
+
+export const deepEqualUnordered = (a: unknown, b: unknown): boolean => {
+  if (a === b) return true;
+
+  if (a instanceof Date && b instanceof Date) {
+    return a.getTime() === b.getTime();
+  }
+
+  if (a instanceof RegExp && b instanceof RegExp) {
+    return a.source === b.source && a.flags === b.flags;
+  }
+
+  if (Array.isArray(a) || Array.isArray(b)) {
+    if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) {
+      return false;
+    }
+    return a.every((item, index) => deepEqualUnordered(item, b[index]));
+  }
+
+  if (!isPlainRecord(a) || !isPlainRecord(b)) return false;
+
+  const keysA = Object.keys(a);
+  const keysB = Object.keys(b);
+  if (keysA.length !== keysB.length) return false;
+
+  return keysA.every(
+    (key) =>
+      Object.hasOwn(b, key) && deepEqualUnordered(a[key], (b as any)[key]),
+  );
+};