@lowdefy/server-dev 6.1.0 → 7.1.0

package/lib/docs/dataSets/groupDataSetByCollection.js

@@ -0,0 +1,52 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+// One group per database target and collection: `target` is the connection's target key (null when
+// computed per request), the database the session loads the group into.
+function ensureGroup({ groups, dataSet, connectionId }) {
+  const target = dataSet.targets?.[connectionId] ?? null;
+  const collection = dataSet.collections[connectionId];
+  const key = JSON.stringify([target, collection]);
+  if (!groups.has(key)) {
+    groups.set(key, { target, collection, fixtures: [], generated: [], indexes: [] });
+  }
+  return groups.get(key);
+}
+
+// The data set keyed by database target and collection, the unit the store loads: fixtures,
+// generated documents and indexes are keyed by connection id, and two connections that name one
+// collection in one target load into it together. Connections in different targets load into
+// different databases, so they never merge.
+function groupDataSetByCollection({ dataSet }) {
+  const groups = new Map();
+  Object.entries(dataSet.fixtures).forEach(([connectionId, documents]) => {
+    const group = ensureGroup({ groups, dataSet, connectionId });
+    documents.forEach((document, index) => {
+      group.fixtures.push({ connectionId, index, document });
+    });
+  });
+  Object.entries(dataSet.generated).forEach(([connectionId, documents]) => {
+    const group = ensureGroup({ groups, dataSet, connectionId });
+    group.generated.push({ connectionId, documents });
+  });
+  Object.entries(dataSet.indexes).forEach(([connectionId, indexes]) => {
+    const group = ensureGroup({ groups, dataSet, connectionId });
+    group.indexes.push(...indexes);
+  });
+  return [...groups.values()];
+}
+
+export default groupDataSetByCollection;

package/lib/docs/dataSets/insertFixture.js

@@ -0,0 +1,57 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { type } from '@lowdefy/helpers';
+
+import deserializeFixture from './deserializeFixture.js';
+import duplicateIndexName from './duplicateIndexName.js';
+
+const DUPLICATE_KEY = 11000;
+
+// A fixture that breaks a unique index fails the load, naming the fixture, the index, the duplicate
+// key and the document that already holds it. Loading indexes before documents is what lets this
+// catch a fixture that collides with an earlier one.
+async function describeDuplicate({ collection, dataSetName, fixture, error }) {
+  const holder = type.isObject(error.keyValue)
+    ? await collection.findOne(error.keyValue, { projection: { _id: 1 } })
+    : null;
+  const where = `Data set "${dataSetName}" fixture ${fixture.connectionId}[${fixture.index}]`;
+  const heldBy = type.isNone(holder) ? '' : ` held by document _id ${JSON.stringify(holder._id)}`;
+  return new Error(
+    `${where} breaks unique index "${duplicateIndexName(
+      error.message
+    )}": duplicate key ${JSON.stringify(error.keyValue)}${heldBy}.`
+  );
+}
+
+// A fixture with an _id replaces any earlier document with that _id; one without is inserted.
+async function insertFixture({ collection, dataSetName, fixture }) {
+  const document = deserializeFixture(fixture.document);
+  try {
+    if (type.isUndefined(document._id)) {
+      await collection.insertOne(document);
+      return;
+    }
+    await collection.replaceOne({ _id: document._id }, document, { upsert: true });
+  } catch (error) {
+    if (error.code === DUPLICATE_KEY) {
+      throw await describeDuplicate({ collection, dataSetName, fixture, error });
+    }
+    throw error;
+  }
+}
+
+export default insertFixture;

package/lib/docs/dataSets/insertGeneratedDocuments.js

@@ -0,0 +1,41 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import deserializeFixture from './deserializeFixture.js';
+import duplicateIndexName from './duplicateIndexName.js';
+
+const DUPLICATE_KEY = 11000;
+
+// One connection's generated documents, in one ordered insert: a document that breaks a unique
+// index stops the load, naming the document and the index.
+async function insertGeneratedDocuments({ collection, dataSetName, batch }) {
+  const documents = batch.documents.map((document) => deserializeFixture(document));
+  try {
+    await collection.insertMany(documents, { ordered: true });
+  } catch (error) {
+    if (error.code !== DUPLICATE_KEY) throw error;
+    // An ordered insert stops at its first failure, so the bulk error holds one write error.
+    const [{ index, errmsg }] = error.writeErrors;
+    const duplicate = /dup key: (.*)$/.exec(errmsg)?.[1] ?? 'unknown';
+    throw new Error(
+      `Data set "${dataSetName}" generated ${batch.connectionId}[${index}] (_id ${JSON.stringify(
+        documents[index]._id
+      )}) breaks unique index "${duplicateIndexName(errmsg)}": duplicate key ${duplicate}.`
+    );
+  }
+}
+
+export default insertGeneratedDocuments;

package/lib/docs/dataSets/isDataStoreGone.js

@@ -0,0 +1,29 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+// The driver's errors for a replica set that is no longer there: mongod exited, or the client was
+// closed under it. The store that raised one is dead, so the next journey must start a new one.
+const STORE_GONE_ERRORS = new Set([
+  'MongoServerSelectionError',
+  'MongoTopologyClosedError',
+  'MongoNotConnectedError',
+]);
+
+function isDataStoreGone(error) {
+  return STORE_GONE_ERRORS.has(error?.name);
+}
+
+export default isDataStoreGone;

package/lib/docs/dataSets/loadDataSetCollection.js

@@ -0,0 +1,42 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import insertFixture from './insertFixture.js';
+import insertGeneratedDocuments from './insertGeneratedDocuments.js';
+import mergeCollectionIndexes from './mergeCollectionIndexes.js';
+
+// Loads one collection into a fresh session database: indexes first, on the empty collection, so a
+// fixture that breaks a unique index fails here instead of loading; then the generated documents;
+// then the fixtures.
+async function loadDataSetCollection({ db, dataSetName, collectionName, group }) {
+  const collection = await db.createCollection(collectionName);
+  const indexes = mergeCollectionIndexes({
+    dataSetName,
+    collection: collectionName,
+    indexes: group.indexes,
+  });
+  if (indexes.length > 0) {
+    await collection.createIndexes(indexes);
+  }
+  for (const batch of group.generated) {
+    await insertGeneratedDocuments({ collection, dataSetName, batch });
+  }
+  for (const fixture of group.fixtures) {
+    await insertFixture({ collection, dataSetName, fixture });
+  }
+}
+
+export default loadDataSetCollection;

package/lib/docs/dataSets/mergeCollectionIndexes.js

@@ -0,0 +1,52 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { type } from '@lowdefy/helpers';
+
+function keyOf(index) {
+  return JSON.stringify(index.key);
+}
+
+// The indexes one collection is created with, from every connection the data set loads into it,
+// where a later index whose key equals an earlier one replaces it. expireAfterSeconds is
+// dropped, so a TTL index becomes a plain one: fixtures often carry old dates, and the TTL monitor
+// would delete them on its own clock, making a run depend on when it happened.
+function mergeCollectionIndexes({ dataSetName, collection, indexes }) {
+  const byKey = new Map();
+  indexes.forEach((index) => {
+    byKey.set(keyOf(index), index);
+  });
+  const byName = new Map();
+  const merged = [...byKey.values()].map((index) => {
+    const { expireAfterSeconds, v, ns, ...rest } = index;
+    return rest;
+  });
+  merged.forEach((index) => {
+    if (type.isUndefined(index.name)) return;
+    const other = byName.get(index.name);
+    if (!type.isUndefined(other)) {
+      throw new Error(
+        `Data set "${dataSetName}" collection "${collection}" has two indexes named "${
+          index.name
+        }" with different keys: ${keyOf(other)} and ${keyOf(index)}.`
+      );
+    }
+    byName.set(index.name, index);
+  });
+  return merged;
+}
+
+export default mergeCollectionIndexes;

package/lib/docs/dataSets/mergeCollectionIndexes.test.mjs

@@ -0,0 +1,59 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import mergeCollectionIndexes from './mergeCollectionIndexes.js';
+
+test('mergeCollectionIndexes lets a later index replace an earlier one with the same key', () => {
+  expect(
+    mergeCollectionIndexes({
+      dataSetName: 'sample',
+      collection: 'tickets',
+      indexes: [
+        { key: { a: 1 }, name: 'a_1' },
+        { key: { b: 1 }, name: 'b_1' },
+        { key: { a: 1 }, name: 'a_unique', unique: true },
+      ],
+    })
+  ).toEqual([
+    { key: { a: 1 }, name: 'a_unique', unique: true },
+    { key: { b: 1 }, name: 'b_1' },
+  ]);
+});
+
+test('mergeCollectionIndexes drops expireAfterSeconds and keeps every other option', () => {
+  expect(
+    mergeCollectionIndexes({
+      dataSetName: 'sample',
+      collection: 'events',
+      indexes: [{ key: { created: 1 }, name: 'ttl', expireAfterSeconds: 60, sparse: true }],
+    })
+  ).toEqual([{ key: { created: 1 }, name: 'ttl', sparse: true }]);
+});
+
+test('mergeCollectionIndexes throws naming both keys when two indexes share a name', () => {
+  expect(() =>
+    mergeCollectionIndexes({
+      dataSetName: 'sample',
+      collection: 'tickets',
+      indexes: [
+        { key: { a: 1 }, name: 'idx' },
+        { key: { b: 1 }, name: 'idx' },
+      ],
+    })
+  ).toThrow(
+    'Data set "sample" collection "tickets" has two indexes named "idx" with different keys: {"a":1} and {"b":1}.'
+  );
+});

package/lib/docs/dataSets/openDataSession.js

@@ -0,0 +1,133 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import crypto from 'node:crypto';
+import { type } from '@lowdefy/helpers';
+
+import dataSessionRegistry from './dataSessionRegistry.js';
+import drainSessionWork from './drainSessionWork.js';
+import getDataStore from './getDataStore.js';
+import groupDataSetByCollection from './groupDataSetByCollection.js';
+import isDataStoreGone from './isDataStoreGone.js';
+import loadDataSetCollection from './loadDataSetCollection.js';
+import sessionDatabaseName from './sessionDatabaseName.js';
+import sweepOrphanDatabases from './sweepOrphanDatabases.js';
+
+const DRAIN_TIMEOUT_MS = 30000;
+
+// Every database the session holds, the first included.
+function listSessionDatabases(session) {
+  return [...new Set([session.databaseName, ...session.databases.values()])];
+}
+
+async function dropSessionDatabases({ client, session }) {
+  await Promise.all(listSessionDatabases(session).map((name) => client.db(name).dropDatabase()));
+}
+
+function createClose({ session, client }) {
+  let closing;
+  async function close() {
+    session.state = 'closing';
+    const drained = await drainSessionWork({ work: session.work, timeoutMs: DRAIN_TIMEOUT_MS });
+    if (!drained) {
+      // eslint-disable-next-line no-console
+      console.warn(
+        `Data set "${session.name}": background work was still running ${
+          DRAIN_TIMEOUT_MS / 1000
+        } s after its journey ended. Dropping its databases anyway; that work now fails.`
+      );
+    }
+    // Closed before the drop: a connection read under this session throws from here on, so
+    // nothing that outlived the drain reaches any database, and no target is added after the drop.
+    session.state = 'closed';
+    dataSessionRegistry.delete(session.id);
+    await dropSessionDatabases({ client, session });
+  }
+  return function closeOnce() {
+    closing = closing ?? close();
+    return closing;
+  };
+}
+
+// Opens a data session for one journey run (or a mutant run): fresh databases on the dev server's
+// memory store, one per database target the build names (dataSet.databaseTargets), loaded with the
+// data set and registered under a random id. They are named <prefix>_<n> under the session's own
+// prefix (ld_<hex>), so the orphan sweep and close() find them all; the first, <prefix>_0, is also
+// the database a target computed per request reads. `cookie` is the id, the payload
+// openPage({ dataCookie }) writes into each actor's lowdefy_journey_data cookie. close() waits for
+// the session's background work (context.waitUntil), then drops the databases.
+async function openDataSession({ dataSet }) {
+  const store = await getDataStore();
+  const { client, uri } = store;
+  const id = crypto.randomBytes(16).toString('hex');
+  const prefix = `ld_${crypto.randomBytes(6).toString('hex')}`;
+  const databaseTargets = dataSet.databaseTargets ?? [];
+  const session = {
+    id,
+    databaseUri: uri,
+    prefix,
+    databaseName: `${prefix}_0`,
+    databases: new Map(databaseTargets.map((target, index) => [target, `${prefix}_${index}`])),
+    databaseCount: Math.max(databaseTargets.length, 1),
+    name: dataSet.name,
+    // Registered while it loads, so a concurrent open's orphan sweep leaves its databases alone.
+    // readDataSession serves only open and closing sessions.
+    state: 'loading',
+    work: new Set(),
+    // Requests sunk instead of sent to an outside service (see sinkExternalConnections).
+    sunk: [],
+  };
+  dataSessionRegistry.set(id, session);
+  try {
+    await sweepOrphanDatabases({ client });
+    const groups = groupDataSetByCollection({ dataSet });
+    // allSettled, so no collection is still loading when a failed load drops the databases.
+    const results = await Promise.allSettled(
+      groups.map((group) =>
+        loadDataSetCollection({
+          db: client.db(sessionDatabaseName({ session, target: group.target })),
+          dataSetName: dataSet.name,
+          collectionName: group.collection,
+          group,
+        })
+      )
+    );
+    const failed = results.find((result) => result.status === 'rejected');
+    if (!type.isUndefined(failed)) {
+      throw failed.reason;
+    }
+  } catch (error) {
+    dataSessionRegistry.delete(id);
+    if (isDataStoreGone(error)) {
+      // mongod is gone, and every session on it with it. Stopping the store clears it, so the next
+      // journey starts a fresh one instead of failing until the dev server restarts.
+      await store.stop().catch(() => {});
+      throw new Error(
+        `The journey data store stopped responding (${error.message}). The next journey starts a new one.`,
+        { cause: error }
+      );
+    }
+    // Best effort: the load error is the one to report, and the next open's orphan sweep drops a
+    // database this could not.
+    await dropSessionDatabases({ client, session }).catch(() => {});
+    throw error;
+  }
+  session.state = 'open';
+  // sunk is the session's own array, so requests sunk while the journey runs show in it.
+  return { id, cookie: id, session, sunk: session.sunk, close: createClose({ session, client }) };
+}
+
+export default openDataSession;