@lowdefy/server-dev 6.1.0 → 7.1.0

package/lib/docs/dataSets/resolveJourneyDataSet.test.mjs

@@ -0,0 +1,130 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import resolveJourneyDataSet from './resolveJourneyDataSet.js';
+
+let configDirectory;
+let buildDirectory;
+
+beforeEach(() => {
+  configDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'lowdefy-resolve-journey-data-set-'));
+  buildDirectory = path.join(configDirectory, '.lowdefy', 'dev', 'build');
+  fs.mkdirSync(path.join(configDirectory, 'tests', 'data'), { recursive: true });
+  fs.mkdirSync(path.join(buildDirectory, 'connections'), { recursive: true });
+  fs.writeFileSync(
+    path.join(buildDirectory, 'connections', 'tickets.json'),
+    JSON.stringify({
+      connectionId: 'tickets',
+      type: 'MongoDBCollection',
+      properties: { databaseUri: { _secret: 'MONGODB_URI' }, collection: 'tickets' },
+    })
+  );
+  fs.writeFileSync(
+    path.join(configDirectory, 'tests', 'data', 'empty-org.yaml'),
+    `fixtures:
+  tickets: [{ _id: t1, organizationId: org_a }]
+users:
+  owner: { id: u_1, roles: [admin], organizationId: org_a }
+  member: { id: u_2, roles: [member], organizationId: org_a }
+`
+  );
+});
+
+afterEach(() => {
+  fs.rmSync(configDirectory, { recursive: true, force: true });
+});
+
+function resolve(overrides) {
+  return resolveJourneyDataSet({
+    configDirectory,
+    buildDirectory,
+    authConfigured: true,
+    mockUserActive: false,
+    ...overrides,
+  });
+}
+
+test('resolveJourneyDataSet resolves a named user from the data set', async () => {
+  const { dataSet, user, error } = await resolve({ data: 'empty-org', user: 'member' });
+  expect(error).toBeUndefined();
+  expect(user).toEqual({ id: 'u_2', roles: ['member'], organizationId: 'org_a' });
+  expect(dataSet.name).toEqual('empty-org');
+  expect(dataSet.collections).toEqual({ tickets: 'tickets' });
+});
+
+test('resolveJourneyDataSet keeps an inline user on a data set journey', async () => {
+  const { user, dataSet } = await resolve({ data: 'empty-org', user: { id: 'x', roles: [] } });
+  expect(user).toEqual({ id: 'x', roles: [] });
+  expect(dataSet.users.owner.id).toEqual('u_1');
+});
+
+test('resolveJourneyDataSet refuses a user name the data set does not declare', async () => {
+  const { error } = await resolve({ data: 'empty-org', user: 'outsider' });
+  expect(error).toEqual(
+    'Data set "empty-org" declares no user "outsider". Declared: owner, member.'
+  );
+});
+
+test('resolveJourneyDataSet refuses a user name on a journey with no data', async () => {
+  const { error } = await resolve({ user: 'member' });
+  expect(error).toMatch(
+    'The journey\'s user "member" names a data set user, but the journey has no "data".'
+  );
+});
+
+test('resolveJourneyDataSet passes a journey with no data and no named user through untouched', async () => {
+  expect(await resolve({ user: 'none' })).toEqual({ user: 'none' });
+  expect(await resolve({ user: { roles: ['admin'] } })).toEqual({ user: { roles: ['admin'] } });
+});
+
+test('resolveJourneyDataSet refuses user none on a data set while auth is configured', async () => {
+  const { error } = await resolve({ data: 'empty-org', user: 'none' });
+  expect(error).toMatch('has user "none", which is refused while auth is configured');
+});
+
+test('resolveJourneyDataSet allows user none on a data set when auth is not configured', async () => {
+  const { error, user } = await resolve({ data: 'empty-org', user: 'none', authConfigured: false });
+  expect(error).toBeUndefined();
+  expect(user).toEqual('none');
+});
+
+test.each([['member'], [{ id: 'x', roles: ['admin'] }], [undefined]])(
+  'resolveJourneyDataSet refuses a data set journey with user %j while a dev mock user is active',
+  async (user) => {
+    const { error } = await resolve({ data: 'empty-org', user, mockUserActive: true });
+    expect(error).toMatch('cannot run while a dev mock user is active');
+  }
+);
+
+test('resolveJourneyDataSet returns readDataSet errors as its own', async () => {
+  const { error } = await resolve({ data: 'missing', user: 'member' });
+  expect(error).toEqual('Data set "missing" not found in tests/data. Declared: empty-org.');
+});
+
+test('resolveJourneyDataSet refuses user none while a dev mock user is active', async () => {
+  const { error } = await resolve({ user: 'none', mockUserActive: true });
+  expect(error).toMatch(
+    'The journey has user "none", which cannot run while a dev mock user is active'
+  );
+});
+
+test('resolveJourneyDataSet keeps user none when no dev mock user is active', async () => {
+  expect(await resolve({ user: 'none' })).toEqual({ user: 'none' });
+});

package/lib/docs/dataSets/sessionDatabaseName.js

@@ -0,0 +1,32 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { type } from '@lowdefy/helpers';
+
+// The session database a database target (describeTargetKey) reads. Each target the build names has
+// its own, numbered when the session opens; a target the build did not name (a config mutant's, say)
+// gets the next number when first met, so it never merges with another. A target computed per
+// request (null) reads the session's first database.
+function sessionDatabaseName({ session, target }) {
+  if (type.isNone(target)) return session.databaseName;
+  if (!session.databases.has(target)) {
+    session.databases.set(target, `${session.prefix}_${session.databaseCount}`);
+    session.databaseCount += 1;
+  }
+  return session.databases.get(target);
+}
+
+export default sessionDatabaseName;

package/lib/docs/dataSets/sweepOrphanDatabases.js

@@ -0,0 +1,36 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import dataSessionRegistry from './dataSessionRegistry.js';
+
+// Drops every session database on the store that no registered session owns: one left behind by a
+// write that was still in flight when its session dropped its databases, or by a dev server that
+// stopped mid-run. A session owns every database under its prefix (<prefix>_<n>), including one it
+// numbers after this sweep lists them.
+async function sweepOrphanDatabases({ client }) {
+  const prefixes = [...dataSessionRegistry.values()].map((session) => `${session.prefix}_`);
+  const { databases } = await client.db('admin').admin().listDatabases({ nameOnly: true });
+  await Promise.all(
+    databases
+      .map((database) => database.name)
+      .filter(
+        (name) => name.startsWith('ld_') && !prefixes.some((prefix) => name.startsWith(prefix))
+      )
+      .map((name) => client.db(name).dropDatabase())
+  );
+}
+
+export default sweepOrphanDatabases;

package/lib/docs/dataSets/test/app/lowdefy.yaml

@@ -0,0 +1,225 @@
+# The app the data set journey end-to-end test (dataSetJourneys.chromium.mongodb.test.mjs) builds and
+# serves. Its connections read the MONGODB_URI secret, which the test points at a stand-in for the
+# developer's real database; a journey on the "shop" data set must never touch it.
+# Auth is configured, with its adapter on the same stand-in, so the suite can prove that no journey
+# request reaches the auth engine or writes an auth collection there.
+lowdefy: local
+name: Data set journeys
+
+auth:
+  database:
+    id: auth_db
+    type: MongoDBAuthAdapter
+    properties:
+      uri:
+        _secret: MONGODB_URI
+  secret:
+    _secret: BETTER_AUTH_SECRET
+  emailAndPassword:
+    enabled: true
+  organizations:
+    policy: tenant
+
+# Under policy: tenant a MongoDBCollection connection is walled by default: every read is filtered
+# and every write stamped with the caller's organization_id.
+connections:
+  - id: tickets
+    type: MongoDBCollection
+    properties:
+      databaseUri:
+        _secret: MONGODB_URI
+      collection: tickets
+      write: true
+  # A second database that also has a tickets collection: under a data set it reads a session
+  # database of its own, so its fixtures never merge with the app's tickets.
+  - id: crm_tickets
+    type: MongoDBCollection
+    properties:
+      databaseUri:
+        _secret: CRM_MONGODB_URI
+      collection: tickets
+      write: true
+
+# A module connection is written under its module's folder (connections/archive/archive_db.json),
+# which the data set redirect must reach too.
+modules:
+  - id: archive
+    source: 'file:modules/archive'
+
+api:
+  - id: create_ticket
+    type: Api
+    routine:
+      - id: insert
+        type: MongoDBInsertOne
+        connectionId: tickets
+        properties:
+          doc:
+            _id:
+              _payload: id
+            title:
+              _payload: title
+      - :return:
+          inserted:
+            _payload: id
+  - id: list_tickets
+    type: Api
+    routine:
+      - id: find
+        type: MongoDBFind
+        connectionId: tickets
+        properties:
+          query: {}
+          options:
+            sort:
+              _id: 1
+      - :return:
+          _step: find
+  # The developer's own cookieless tab: no caller, so the read opts out of the wall.
+  - id: list_all_tickets
+    type: Api
+    routine:
+      - id: find
+        type: MongoDBFind
+        connectionId: tickets
+        tenant: none
+        properties:
+          query: {}
+          options:
+            sort:
+              _id: 1
+      - :return:
+          _step: find
+  - id: list_crm_tickets
+    type: Api
+    routine:
+      - id: find
+        type: MongoDBFind
+        connectionId: crm_tickets
+        properties:
+          query: {}
+          options:
+            sort:
+              _id: 1
+      - :return:
+          _step: find
+  - id: create_ticket_later
+    type: Api
+    routine:
+      - id: dispatch
+        type: CallApi
+        properties:
+          endpointId: insert_detached
+          detached: true
+          payload:
+            id:
+              _payload: id
+      - :return:
+          dispatched: true
+  - id: insert_detached
+    type: Api
+    routine:
+      - id: insert
+        type: MongoDBInsertOne
+        connectionId: tickets
+        properties:
+          doc:
+            _id:
+              _payload: id
+            detached: true
+
+  # Run under a config mutant (`lowdefy journeys harden`) that drops their "log" step: a detached
+  # target and a direct endpoint, each writing a ticket and then a log entry.
+  - id: create_ticket_later_logged
+    type: Api
+    routine:
+      - id: dispatch
+        type: CallApi
+        properties:
+          endpointId: insert_detached_logged
+          detached: true
+          payload:
+            id:
+              _payload: id
+      - :return:
+          dispatched: true
+  - id: insert_detached_logged
+    type: Api
+    routine:
+      - id: insert
+        type: MongoDBInsertOne
+        connectionId: tickets
+        properties:
+          doc:
+            _id:
+              _payload: id
+            detached: true
+      - id: log
+        type: MongoDBInsertOne
+        connectionId: tickets
+        properties:
+          doc:
+            _id: log-detached
+  - id: create_ticket_logged
+    type: Api
+    routine:
+      - id: insert
+        type: MongoDBInsertOne
+        connectionId: tickets
+        properties:
+          doc:
+            _id:
+              _payload: id
+      - id: log
+        type: MongoDBInsertOne
+        connectionId: tickets
+        properties:
+          doc:
+            _id: log-direct
+      - :return:
+          inserted: true
+
+  # A detached run whose routine reaches an auth step: under a data session it has no auth engine,
+  # so the step fails and the insert after it never runs.
+  - id: create_ticket_later_auth
+    type: Api
+    routine:
+      - id: dispatch
+        type: CallApi
+        properties:
+          endpointId: insert_detached_auth
+          detached: true
+          payload: {}
+      - :return:
+          dispatched: true
+  - id: insert_detached_auth
+    type: Api
+    routine:
+      - id: before
+        type: MongoDBInsertOne
+        connectionId: tickets
+        properties:
+          doc:
+            _id: later-auth-before
+      - id: members
+        type: ListMembers
+        properties: {}
+      - id: after
+        type: MongoDBInsertOne
+        connectionId: tickets
+        properties:
+          doc:
+            _id: later-auth-after
+
+websockets:
+  - id: ticket_changes
+    type: MongoDBChangeStream
+    connectionId: tickets
+    properties:
+      pipeline:
+        - $match:
+            operationType: insert
+
+pages:
+  - id: tickets
+    type: Box

package/lib/docs/dataSets/test/app/modules/archive/module.lowdefy.yaml

@@ -0,0 +1,30 @@
+name: Archive
+
+secrets:
+  - name: MONGODB_URI
+    description: The app's database, the same stand-in the app's own connection reads.
+
+connections:
+  - id: archive_db
+    type: MongoDBCollection
+    properties:
+      databaseUri:
+        _secret: MONGODB_URI
+      collection: archived
+      write: true
+
+api:
+  - id: archive_ticket
+    type: Api
+    routine:
+      - id: insert
+        type: MongoDBInsertOne
+        connectionId:
+          _module.connectionId: archive_db
+        properties:
+          doc:
+            _id:
+              _payload: id
+      - :return:
+          archived:
+            _payload: id

package/lib/docs/dataSets/test/app/tests/data/shop.yaml

@@ -0,0 +1,12 @@
+fixtures:
+  tickets:
+    - { _id: t-a1, organization_id: org_a, title: First }
+    - { _id: t-a2, organization_id: org_a, title: Second }
+    - { _id: t-b1, organization_id: org_b, title: Other org }
+  # The same collection name in another database: it loads into a session database of its own.
+  crm_tickets:
+    - { _id: t-a1, organization_id: org_a, title: CRM copy }
+    - { _id: crm-a2, organization_id: org_a, title: CRM only }
+users:
+  owner: { id: u_1, roles: [admin], organizationId: org_a }
+  outsider: { id: u_9, roles: [admin], organizationId: org_b }

package/lib/docs/dataSets/test/buildTestApp.mjs

@@ -0,0 +1,43 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { shallowBuild } from '@lowdefy/build/dev';
+
+// Builds an app into <serverDirectory>/build for a test, in a Node process of its own: the build
+// imports plugin modules dynamically, which jest's module runtime cannot load.
+//   node buildTestApp.mjs <configDirectory> <serverDirectory>
+const [configDirectory, serverDirectory] = process.argv.slice(2);
+const serverDevDirectory = path.resolve(import.meta.dirname, '../../../..');
+const directories = {
+  config: configDirectory,
+  build: path.join(serverDirectory, 'build'),
+  server: path.join(serverDirectory, 'server'),
+};
+fs.mkdirSync(path.join(directories.server, 'public_default'), { recursive: true });
+fs.copyFileSync(
+  path.join(serverDevDirectory, 'package.json'),
+  path.join(directories.server, 'package.json')
+);
+const quiet = () => {};
+await shallowBuild({
+  customMessagesMap: {},
+  customTypesMap: {},
+  directories,
+  logger: { debug: quiet, info: quiet, warn: quiet, error: console.error, ui: {} },
+  stage: 'dev',
+});

package/lib/docs/devEventBus.js

@@ -0,0 +1,136 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import path from 'node:path';
+
+import chokidar from 'chokidar';
+import { type } from '@lowdefy/helpers';
+
+import getStaleStatus from './getStaleStatus.js';
+import readBuildArtifact from './readBuildArtifact.js';
+
+// In-process publish/subscribe bus behind the dev server's push channels
+// (MCP notifications/message and GET /lowdefy-docs/events). Deliberately
+// unbuffered: a subscriber that missed an event polls build_status, which is
+// derived from the same build/buildStatus.json this bus watches, so the two
+// always agree. Event shape is { type, timestamp, ...entry } with type one of
+// build | client_error | server_error | restart.
+const EVENT_TYPES = ['build', 'client_error', 'server_error', 'restart'];
+
+// The Hono process boots once per (re)start, so module load time is the
+// restart time — subscribers receive it as a restart event on connect.
+const bootedAt = new Date().toISOString();
+
+const subscribers = [];
+
+let watcher = null;
+
+function publish(event) {
+  if (!EVENT_TYPES.includes(event?.type)) {
+    throw new Error(
+      `devEventBus event type must be one of ${EVENT_TYPES.join(', ')}. Received ${JSON.stringify(
+        event?.type
+      )}.`
+    );
+  }
+  const fullEvent = { timestamp: new Date().toISOString(), ...event };
+  // Iterate over a copy: a failing send unsubscribes itself mid-loop.
+  [...subscribers].forEach((send) => {
+    let result;
+    try {
+      result = send(fullEvent);
+    } catch {
+      unsubscribe(send);
+      return;
+    }
+    if (type.isFunction(result?.then)) {
+      result.catch(() => unsubscribe(send));
+    }
+  });
+}
+
+function readBuildEvent() {
+  // The manager writes buildStatus.json from another process; a change event
+  // can fire while the write is still in flight. A half-written file is not an
+  // error worth surfacing — the completed write fires its own change event.
+  try {
+    const build = readBuildArtifact({ name: 'buildStatus.json' });
+    if (type.isNone(build)) return null;
+    return {
+      type: 'build',
+      status: build.status,
+      errorCount: (build.errors ?? []).length,
+      warningCount: (build.warnings ?? []).length,
+      errors: build.errors ?? [],
+      warnings: build.warnings ?? [],
+      stale: false,
+      staleSince: null,
+      ...getStaleStatus(),
+    };
+  } catch {
+    return null;
+  }
+}
+
+function publishBuildStatus() {
+  const event = readBuildEvent();
+  if (event === null) return;
+  publish(event);
+}
+
+// The watcher is held open only while someone is listening. Starting it at
+// module load would pin every process that imports an error store — including
+// test runners — on a persistent fs handle for a file it may never need.
+function startWatcher() {
+  watcher = chokidar.watch(path.join(process.cwd(), 'build', 'buildStatus.json'), {
+    persistent: true,
+    ignoreInitial: true,
+  });
+  watcher.on('add', publishBuildStatus);
+  watcher.on('change', publishBuildStatus);
+  // No unlink handler: cleanBuildDirectory removes the file mid-rebuild and the
+  // rebuild's own write re-adds it.
+}
+
+function stopWatcher() {
+  const closing = watcher.close();
+  watcher = null;
+  return closing;
+}
+
+function unsubscribe(send) {
+  const index = subscribers.indexOf(send);
+  if (index === -1) return;
+  subscribers.splice(index, 1);
+  if (subscribers.length === 0 && watcher !== null) {
+    stopWatcher();
+  }
+}
+
+function subscribe(send) {
+  if (!type.isFunction(send)) {
+    throw new Error(
+      `devEventBus.subscribe requires a "send" function. Received ${JSON.stringify(send)}.`
+    );
+  }
+  subscribers.push(send);
+  if (watcher === null) {
+    startWatcher();
+  }
+  return () => unsubscribe(send);
+}
+
+export { bootedAt, publish, subscribe };