@lowdefy/server-dev 6.0.0 → 7.0.0

package/lib/docs/dataSets/test/app/lowdefy.yaml

@@ -0,0 +1,203 @@
+# The app the data set journey end-to-end test (dataSetJourneys.chromium.mongodb.test.mjs) builds and
+# serves. Its connections read the MONGODB_URI secret, which the test points at a stand-in for the
+# developer's real database; a journey on the "shop" data set must never touch it.
+# Auth is configured, with its adapter on the same stand-in, so the suite can prove that no journey
+# request reaches the auth engine or writes an auth collection there.
+lowdefy: local
+name: Data set journeys
+
+auth:
+  database:
+    id: auth_db
+    type: MongoDBAuthAdapter
+    properties:
+      uri:
+        _secret: MONGODB_URI
+  secret:
+    _secret: BETTER_AUTH_SECRET
+  emailAndPassword:
+    enabled: true
+  organizations:
+    policy: tenant
+
+# Under policy: tenant a MongoDBCollection connection is walled by default: every read is filtered
+# and every write stamped with the caller's organization_id.
+connections:
+  - id: tickets
+    type: MongoDBCollection
+    properties:
+      databaseUri:
+        _secret: MONGODB_URI
+      collection: tickets
+      write: true
+
+# A module connection is written under its module's folder (connections/archive/archive_db.json),
+# which the data set redirect must reach too.
+modules:
+  - id: archive
+    source: 'file:modules/archive'
+
+api:
+  - id: create_ticket
+    type: Api
+    routine:
+      - id: insert
+        type: MongoDBInsertOne
+        connectionId: tickets
+        properties:
+          doc:
+            _id:
+              _payload: id
+            title:
+              _payload: title
+      - :return:
+          inserted:
+            _payload: id
+  - id: list_tickets
+    type: Api
+    routine:
+      - id: find
+        type: MongoDBFind
+        connectionId: tickets
+        properties:
+          query: {}
+          options:
+            sort:
+              _id: 1
+      - :return:
+          _step: find
+  # The developer's own cookieless tab: no caller, so the read opts out of the wall.
+  - id: list_all_tickets
+    type: Api
+    routine:
+      - id: find
+        type: MongoDBFind
+        connectionId: tickets
+        tenant: none
+        properties:
+          query: {}
+          options:
+            sort:
+              _id: 1
+      - :return:
+          _step: find
+  - id: create_ticket_later
+    type: Api
+    routine:
+      - id: dispatch
+        type: CallApi
+        properties:
+          endpointId: insert_detached
+          detached: true
+          payload:
+            id:
+              _payload: id
+      - :return:
+          dispatched: true
+  - id: insert_detached
+    type: Api
+    routine:
+      - id: insert
+        type: MongoDBInsertOne
+        connectionId: tickets
+        properties:
+          doc:
+            _id:
+              _payload: id
+            detached: true
+
+  # Run under a config mutant (`lowdefy journeys harden`) that drops their "log" step: a detached
+  # target and a direct endpoint, each writing a ticket and then a log entry.
+  - id: create_ticket_later_logged
+    type: Api
+    routine:
+      - id: dispatch
+        type: CallApi
+        properties:
+          endpointId: insert_detached_logged
+          detached: true
+          payload:
+            id:
+              _payload: id
+      - :return:
+          dispatched: true
+  - id: insert_detached_logged
+    type: Api
+    routine:
+      - id: insert
+        type: MongoDBInsertOne
+        connectionId: tickets
+        properties:
+          doc:
+            _id:
+              _payload: id
+            detached: true
+      - id: log
+        type: MongoDBInsertOne
+        connectionId: tickets
+        properties:
+          doc:
+            _id: log-detached
+  - id: create_ticket_logged
+    type: Api
+    routine:
+      - id: insert
+        type: MongoDBInsertOne
+        connectionId: tickets
+        properties:
+          doc:
+            _id:
+              _payload: id
+      - id: log
+        type: MongoDBInsertOne
+        connectionId: tickets
+        properties:
+          doc:
+            _id: log-direct
+      - :return:
+          inserted: true
+
+  # A detached run whose routine reaches an auth step: under a data session it has no auth engine,
+  # so the step fails and the insert after it never runs.
+  - id: create_ticket_later_auth
+    type: Api
+    routine:
+      - id: dispatch
+        type: CallApi
+        properties:
+          endpointId: insert_detached_auth
+          detached: true
+          payload: {}
+      - :return:
+          dispatched: true
+  - id: insert_detached_auth
+    type: Api
+    routine:
+      - id: before
+        type: MongoDBInsertOne
+        connectionId: tickets
+        properties:
+          doc:
+            _id: later-auth-before
+      - id: members
+        type: ListMembers
+        properties: {}
+      - id: after
+        type: MongoDBInsertOne
+        connectionId: tickets
+        properties:
+          doc:
+            _id: later-auth-after
+
+websockets:
+  - id: ticket_changes
+    type: MongoDBChangeStream
+    connectionId: tickets
+    properties:
+      pipeline:
+        - $match:
+            operationType: insert
+
+pages:
+  - id: tickets
+    type: Box

package/lib/docs/dataSets/test/app/modules/archive/module.lowdefy.yaml

@@ -0,0 +1,30 @@
+name: Archive
+
+secrets:
+  - name: MONGODB_URI
+    description: The app's database, the same stand-in the app's own connection reads.
+
+connections:
+  - id: archive_db
+    type: MongoDBCollection
+    properties:
+      databaseUri:
+        _secret: MONGODB_URI
+      collection: archived
+      write: true
+
+api:
+  - id: archive_ticket
+    type: Api
+    routine:
+      - id: insert
+        type: MongoDBInsertOne
+        connectionId:
+          _module.connectionId: archive_db
+        properties:
+          doc:
+            _id:
+              _payload: id
+      - :return:
+          archived:
+            _payload: id

package/lib/docs/dataSets/test/app/tests/data/shop.yaml

@@ -0,0 +1,8 @@
+fixtures:
+  tickets:
+    - { _id: t-a1, organization_id: org_a, title: First }
+    - { _id: t-a2, organization_id: org_a, title: Second }
+    - { _id: t-b1, organization_id: org_b, title: Other org }
+users:
+  owner: { id: u_1, roles: [admin], organizationId: org_a }
+  outsider: { id: u_9, roles: [admin], organizationId: org_b }

package/lib/docs/dataSets/test/buildTestApp.mjs

@@ -0,0 +1,43 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { shallowBuild } from '@lowdefy/build/dev';
+
+// Builds an app into <serverDirectory>/build for a test, in a Node process of its own: the build
+// imports plugin modules dynamically, which jest's module runtime cannot load.
+//   node buildTestApp.mjs <configDirectory> <serverDirectory>
+const [configDirectory, serverDirectory] = process.argv.slice(2);
+const serverDevDirectory = path.resolve(import.meta.dirname, '../../../..');
+const directories = {
+  config: configDirectory,
+  build: path.join(serverDirectory, 'build'),
+  server: path.join(serverDirectory, 'server'),
+};
+fs.mkdirSync(path.join(directories.server, 'public_default'), { recursive: true });
+fs.copyFileSync(
+  path.join(serverDevDirectory, 'package.json'),
+  path.join(directories.server, 'package.json')
+);
+const quiet = () => {};
+await shallowBuild({
+  customMessagesMap: {},
+  customTypesMap: {},
+  directories,
+  logger: { debug: quiet, info: quiet, warn: quiet, error: console.error, ui: {} },
+  stage: 'dev',
+});

package/lib/docs/devEventBus.js

@@ -0,0 +1,136 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import path from 'node:path';
+
+import chokidar from 'chokidar';
+import { type } from '@lowdefy/helpers';
+
+import getStaleStatus from './getStaleStatus.js';
+import readBuildArtifact from './readBuildArtifact.js';
+
+// In-process publish/subscribe bus behind the dev server's push channels
+// (MCP notifications/message and GET /lowdefy-docs/events). Deliberately
+// unbuffered: a subscriber that missed an event polls build_status, which is
+// derived from the same build/buildStatus.json this bus watches, so the two
+// always agree. Event shape is { type, timestamp, ...entry } with type one of
+// build | client_error | server_error | restart.
+const EVENT_TYPES = ['build', 'client_error', 'server_error', 'restart'];
+
+// The Hono process boots once per (re)start, so module load time is the
+// restart time — subscribers receive it as a restart event on connect.
+const bootedAt = new Date().toISOString();
+
+const subscribers = [];
+
+let watcher = null;
+
+function publish(event) {
+  if (!EVENT_TYPES.includes(event?.type)) {
+    throw new Error(
+      `devEventBus event type must be one of ${EVENT_TYPES.join(', ')}. Received ${JSON.stringify(
+        event?.type
+      )}.`
+    );
+  }
+  const fullEvent = { timestamp: new Date().toISOString(), ...event };
+  // Iterate over a copy: a failing send unsubscribes itself mid-loop.
+  [...subscribers].forEach((send) => {
+    let result;
+    try {
+      result = send(fullEvent);
+    } catch {
+      unsubscribe(send);
+      return;
+    }
+    if (type.isFunction(result?.then)) {
+      result.catch(() => unsubscribe(send));
+    }
+  });
+}
+
+function readBuildEvent() {
+  // The manager writes buildStatus.json from another process; a change event
+  // can fire while the write is still in flight. A half-written file is not an
+  // error worth surfacing — the completed write fires its own change event.
+  try {
+    const build = readBuildArtifact({ name: 'buildStatus.json' });
+    if (type.isNone(build)) return null;
+    return {
+      type: 'build',
+      status: build.status,
+      errorCount: (build.errors ?? []).length,
+      warningCount: (build.warnings ?? []).length,
+      errors: build.errors ?? [],
+      warnings: build.warnings ?? [],
+      stale: false,
+      staleSince: null,
+      ...getStaleStatus(),
+    };
+  } catch {
+    return null;
+  }
+}
+
+function publishBuildStatus() {
+  const event = readBuildEvent();
+  if (event === null) return;
+  publish(event);
+}
+
+// The watcher is held open only while someone is listening. Starting it at
+// module load would pin every process that imports an error store — including
+// test runners — on a persistent fs handle for a file it may never need.
+function startWatcher() {
+  watcher = chokidar.watch(path.join(process.cwd(), 'build', 'buildStatus.json'), {
+    persistent: true,
+    ignoreInitial: true,
+  });
+  watcher.on('add', publishBuildStatus);
+  watcher.on('change', publishBuildStatus);
+  // No unlink handler: cleanBuildDirectory removes the file mid-rebuild and the
+  // rebuild's own write re-adds it.
+}
+
+function stopWatcher() {
+  const closing = watcher.close();
+  watcher = null;
+  return closing;
+}
+
+function unsubscribe(send) {
+  const index = subscribers.indexOf(send);
+  if (index === -1) return;
+  subscribers.splice(index, 1);
+  if (subscribers.length === 0 && watcher !== null) {
+    stopWatcher();
+  }
+}
+
+function subscribe(send) {
+  if (!type.isFunction(send)) {
+    throw new Error(
+      `devEventBus.subscribe requires a "send" function. Received ${JSON.stringify(send)}.`
+    );
+  }
+  subscribers.push(send);
+  if (watcher === null) {
+    startWatcher();
+  }
+  return () => unsubscribe(send);
+}
+
+export { bootedAt, publish, subscribe };

package/lib/docs/devEventBus.test.mjs

@@ -0,0 +1,269 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+import { jest } from '@jest/globals';
+
+import flushFsEvents from '../../test-utils/flushFsEvents.mjs';
+import spyOnChokidar from '../../test-utils/spyOnChokidar.mjs';
+
+// macOS can hold file events back for tens of seconds on a busy machine.
+jest.setTimeout(180000);
+
+// The bus watches build/buildStatus.json under process.cwd() — point it at a
+// throwaway server directory before the module is imported.
+const originalCwd = process.cwd();
+const fixtureDir = fs.realpathSync(
+  fs.mkdtempSync(path.join(os.tmpdir(), 'lowdefy-dev-event-bus-test-'))
+);
+fs.mkdirSync(path.join(fixtureDir, 'build'));
+process.chdir(fixtureDir);
+
+const statusPath = path.join(fixtureDir, 'build', 'buildStatus.json');
+
+const watchers = spyOnChokidar();
+const { bootedAt, publish, subscribe } = await import('./devEventBus.js');
+
+const unsubscribers = [];
+
+function listen(send) {
+  const unsubscribe = subscribe(send);
+  unsubscribers.push(unsubscribe);
+  return unsubscribe;
+}
+
+// The bus starts its watcher with the first subscriber, so a test waits for
+// that watcher to be ready before it writes the file.
+function watcherReady() {
+  return watchers.at(-1).ready;
+}
+
+// Resolves with the next event of the given type, so file-watcher tests wait
+// for the real fs event instead of sleeping a guessed interval.
+function nextEvent(type, { timeout = 60000 } = {}) {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      unsubscribe();
+      reject(new Error(`No ${type} event within ${timeout}ms.`));
+    }, timeout);
+    const unsubscribe = listen((event) => {
+      if (event.type !== type) return;
+      clearTimeout(timer);
+      unsubscribe();
+      resolve(event);
+    });
+  });
+}
+
+async function writeUntilSeen({ pending, content }) {
+  fs.writeFileSync(statusPath, content);
+  const retry = setInterval(() => fs.writeFileSync(statusPath, content), 1000);
+  try {
+    return await pending;
+  } finally {
+    clearInterval(retry);
+  }
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// fsevents can report a write made just before the watcher attached, so a
+// file that must already exist when a test subscribes is flushed through
+// before the test subscribes.
+async function writeExistingStatus(status) {
+  fs.writeFileSync(statusPath, JSON.stringify(status));
+  await flushFsEvents();
+}
+
+afterEach(async () => {
+  unsubscribers.splice(0).forEach((unsubscribe) => unsubscribe());
+  await Promise.all(watchers.map(({ watcher }) => watcher.close()));
+  if (fs.existsSync(statusPath)) fs.rmSync(statusPath);
+  // The removal must not reach the next test's watcher.
+  await flushFsEvents();
+});
+
+afterAll(() => {
+  process.chdir(originalCwd);
+  fs.rmSync(fixtureDir, { recursive: true, force: true });
+});
+
+test('bootedAt is an ISO timestamp fixed at module load', () => {
+  expect(new Date(bootedAt).toISOString()).toEqual(bootedAt);
+});
+
+test('subscribe delivers published events with an ISO timestamp and unsubscribe stops delivery', () => {
+  const send = jest.fn();
+  const unsubscribe = listen(send);
+
+  publish({ type: 'client_error', message: 'boom' });
+  expect(send).toHaveBeenCalledTimes(1);
+  const [event] = send.mock.calls[0];
+  expect(event).toMatchObject({ type: 'client_error', message: 'boom' });
+  expect(new Date(event.timestamp).toISOString()).toEqual(event.timestamp);
+
+  unsubscribe();
+  publish({ type: 'client_error', message: 'after unsubscribe' });
+  expect(send).toHaveBeenCalledTimes(1);
+});
+
+test('publish keeps the entry timestamp when the entry carries one', () => {
+  const send = jest.fn();
+  listen(send);
+  publish({ type: 'server_error', timestamp: '2026-01-02T03:04:05.000Z', message: 'x' });
+  expect(send.mock.calls[0][0].timestamp).toEqual('2026-01-02T03:04:05.000Z');
+});
+
+test('publish rejects an unknown event type', () => {
+  expect(() => publish({ type: 'nope' })).toThrow(
+    'devEventBus event type must be one of build, client_error, server_error, restart. Received "nope".'
+  );
+});
+
+test('subscribe rejects a non-function send', () => {
+  expect(() => subscribe('not a function')).toThrow(
+    'devEventBus.subscribe requires a "send" function. Received "not a function".'
+  );
+});
+
+test('a subscriber that throws is removed and the others still receive', () => {
+  const throwing = jest.fn(() => {
+    throw new Error('stream gone');
+  });
+  const healthy = jest.fn();
+  listen(throwing);
+  listen(healthy);
+
+  publish({ type: 'client_error', message: 'first' });
+  publish({ type: 'client_error', message: 'second' });
+
+  expect(throwing).toHaveBeenCalledTimes(1);
+  expect(healthy).toHaveBeenCalledTimes(2);
+});
+
+test('a subscriber whose send rejects is removed after the rejection settles', async () => {
+  const rejecting = jest.fn(() => Promise.reject(new Error('stream gone')));
+  const healthy = jest.fn(() => Promise.resolve());
+  listen(rejecting);
+  listen(healthy);
+
+  publish({ type: 'client_error', message: 'first' });
+  await sleep(0);
+  publish({ type: 'client_error', message: 'second' });
+
+  expect(rejecting).toHaveBeenCalledTimes(1);
+  expect(healthy).toHaveBeenCalledTimes(2);
+});
+
+test('a buildStatus.json write publishes a build event carrying the stale flag when the build failed', async () => {
+  const pending = nextEvent('build');
+  await watcherReady();
+  // buildStatus.json does not exist yet, so the watcher watches its directory
+  // for it, and that watch can start a moment after ready. The same write is
+  // repeated until one is seen.
+  const event = await writeUntilSeen({
+    pending,
+    content: JSON.stringify({
+      status: 'error',
+      timestamp: '2026-02-03T04:05:06.000Z',
+      errors: [{ message: 'Block type "Buton" not found.' }],
+      warnings: [],
+    }),
+  });
+  expect(event).toMatchObject({
+    type: 'build',
+    status: 'error',
+    errorCount: 1,
+    warningCount: 0,
+    errors: [{ message: 'Block type "Buton" not found.' }],
+    warnings: [],
+    stale: true,
+    staleSince: '2026-02-03T04:05:06.000Z',
+  });
+  expect(event.staleReason).toContain('The last build failed.');
+});
+
+test('a successful buildStatus.json write publishes a build event that is not stale', async () => {
+  await writeExistingStatus({ status: 'error', errors: [{}], warnings: [] });
+  const pending = nextEvent('build');
+  await watcherReady();
+  fs.writeFileSync(
+    statusPath,
+    JSON.stringify({
+      status: 'ok',
+      timestamp: '2026-02-03T04:06:00.000Z',
+      errors: [],
+      warnings: [{ message: 'Unused block.' }],
+    })
+  );
+
+  const event = await pending;
+  expect(event).toMatchObject({
+    type: 'build',
+    status: 'ok',
+    errorCount: 0,
+    warningCount: 1,
+    stale: false,
+    staleSince: null,
+  });
+  expect(event).not.toHaveProperty('staleReason');
+});
+
+test('removing buildStatus.json publishes nothing', async () => {
+  await writeExistingStatus({ status: 'ok', errors: [], warnings: [] });
+  const send = jest.fn();
+  listen(send);
+  const { watcher } = watchers.at(-1);
+  await watcherReady();
+  // The bus handles a watcher event synchronously, so once the watcher has
+  // reported the removal, anything the bus publishes for it has been sent.
+  const removed = new Promise((resolve, reject) => {
+    const timer = setTimeout(
+      () => reject(new Error('The watcher did not report the removal within 60000ms.')),
+      60000
+    );
+    watcher.on('unlink', (filePath) => {
+      if (filePath !== statusPath) return;
+      clearTimeout(timer);
+      resolve();
+    });
+  });
+
+  fs.rmSync(statusPath);
+  await removed;
+
+  expect(send).not.toHaveBeenCalled();
+});
+
+test('the watcher closes when the last subscriber leaves so a later write publishes nothing', async () => {
+  const send = jest.fn();
+  const unsubscribe = listen(send);
+  const { watcher } = watchers.at(-1);
+  await watcherReady();
+  unsubscribe();
+  expect(watcher.closed).toBe(true);
+
+  fs.writeFileSync(statusPath, JSON.stringify({ status: 'ok', errors: [], warnings: [] }));
+  // Every write so far has been handed out by the OS once this returns.
+  await flushFsEvents();
+
+  expect(send).not.toHaveBeenCalled();
+});