@lowdefy/server-dev 6.1.0 → 7.1.0

package/lib/server/contextMapBudget.test.mjs

@@ -0,0 +1,98 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { jest } from '@jest/globals';
+
+// A budget one page build passes, so the next request recreates the context.
+jest.unstable_mockModule('./contextMapBudget.js', () => ({ default: 1 }));
+
+const { default: buildPageIfNeeded, getBuildContext } = await import('./jitPageBuilder.js');
+
+jest.setTimeout(30000);
+
+function writeJson(filePath, data) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, JSON.stringify(data));
+}
+
+test('a build context past its map budget is recreated, and every page rebuilds on the new one', async () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ldf-context-budget-'));
+  const buildDirectory = path.join(root, 'server', 'build');
+  const configDirectory = path.join(root, 'config');
+  writeJson(path.join(buildDirectory, 'idCounter.json'), { prefix: 'cfg1_', counter: 10 });
+  writeJson(path.join(buildDirectory, 'installedPluginPackages.json'), ['@lowdefy/blocks-basic']);
+  writeJson(path.join(buildDirectory, 'theme.json'), {});
+  const registry = {};
+  for (const pageId of ['a', 'b']) {
+    fs.mkdirSync(path.join(configDirectory, 'pages'), { recursive: true });
+    fs.writeFileSync(
+      path.join(configDirectory, 'pages', `${pageId}.yaml`),
+      `id: ${pageId}\ntype: Box\n`
+    );
+    registry[pageId] = {
+      pageId,
+      auth: { public: true },
+      refId: `ref-${pageId}`,
+      refPath: `pages/${pageId}.yaml`,
+    };
+  }
+  writeJson(path.join(buildDirectory, 'pageRegistry.json'), registry);
+  const request = (pageId) => buildPageIfNeeded({ pageId, buildDirectory, configDirectory });
+
+  expect(await request('a')).toEqual({ built: true, warnings: undefined });
+  const firstContext = getBuildContext(buildDirectory, configDirectory);
+  expect(await request('b')).toEqual({ built: true, warnings: undefined });
+  const secondContext = getBuildContext(buildDirectory, configDirectory);
+  expect(secondContext).not.toBe(firstContext);
+
+  // b was built past the budget too, so the next request recreates again and
+  // a, built two contexts ago, rebuilds.
+  expect(await request('a')).toEqual({ built: true, warnings: undefined });
+  expect(getBuildContext(buildDirectory, configDirectory)).not.toBe(secondContext);
+  fs.rmSync(root, { recursive: true, force: true });
+});
+
+test('a failed page build counts toward the budget', async () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ldf-context-budget-'));
+  const buildDirectory = path.join(root, 'server', 'build');
+  const configDirectory = path.join(root, 'config');
+  writeJson(path.join(buildDirectory, 'idCounter.json'), { prefix: 'cfg1_', counter: 10 });
+  writeJson(path.join(buildDirectory, 'installedPluginPackages.json'), ['@lowdefy/blocks-basic']);
+  writeJson(path.join(buildDirectory, 'theme.json'), {});
+  fs.mkdirSync(path.join(configDirectory, 'pages'), { recursive: true });
+  fs.writeFileSync(
+    path.join(configDirectory, 'pages', 'broken.yaml'),
+    'id: broken\ntype: Box\nblocks:\n  - id: typo\n    type: Buton\n'
+  );
+  writeJson(path.join(buildDirectory, 'pageRegistry.json'), {
+    broken: {
+      pageId: 'broken',
+      auth: { public: true },
+      refId: 'ref-broken',
+      refPath: 'pages/broken.yaml',
+    },
+  });
+  const request = (pageId) => buildPageIfNeeded({ pageId, buildDirectory, configDirectory });
+
+  await expect(request('broken')).rejects.toThrow('Buton');
+  const firstContext = getBuildContext(buildDirectory, configDirectory);
+  await expect(request('broken')).rejects.toThrow('Buton');
+  expect(getBuildContext(buildDirectory, configDirectory)).not.toBe(firstContext);
+  fs.rmSync(root, { recursive: true, force: true });
+});

package/lib/server/continueJitKeys.js

@@ -0,0 +1,28 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { makeId } from '@lowdefy/build/dev';
+
+// JIT keys extend the config build's prefix with the dev server process's id, so
+// they never repeat a key of that build, of an earlier config build, or of another
+// process serving the same config build (a restarted server continues the build
+// the one before it served). Within a process the counter only moves forward, so
+// a recreated build context never repeats an earlier page build's key.
+function continueJitKeys({ idCounter, childId }) {
+  makeId.continueFrom({ prefix: `${idCounter.prefix}${childId}_`, counter: 0 });
+}
+
+export default continueJitKeys;

package/lib/server/continueJitKeys.test.mjs

@@ -0,0 +1,43 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { makeId } from '@lowdefy/build/dev';
+
+import continueJitKeys from './continueJitKeys.js';
+
+test('two dev server processes serving one config build never hand out the same key', () => {
+  const idCounter = { prefix: 'a1b2_', counter: 40 };
+
+  continueJitKeys({ idCounter, childId: 'aaaaaa' });
+  const first = [makeId.next(), makeId.next()];
+  continueJitKeys({ idCounter, childId: 'bbbbbb' });
+  const second = [makeId.next(), makeId.next()];
+
+  expect(first).toEqual(['a1b2_aaaaaa_1', 'a1b2_aaaaaa_2']);
+  expect(second).toEqual(['a1b2_bbbbbb_1', 'a1b2_bbbbbb_2']);
+});
+
+test('a recreated build context in the same process continues its keys', () => {
+  const idCounter = { prefix: 'c3d4_', counter: 40 };
+
+  continueJitKeys({ idCounter, childId: 'aaaaaa' });
+  const first = makeId.next();
+  continueJitKeys({ idCounter, childId: 'aaaaaa' });
+  const second = makeId.next();
+
+  expect(first).toBe('c3d4_aaaaaa_1');
+  expect(second).toBe('c3d4_aaaaaa_2');
+});

package/lib/server/createDataSessionEndedError.js

@@ -0,0 +1,33 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { HTTPException } from 'hono/http-exception';
+
+// A request whose verified data cookie names a session that has closed outlived its journey. The
+// verified token proves it came from a journey, so it is answered 410 and never falls back to the
+// app's own database. Thrown as an HTTPException, which the error handler sends as it is.
+function createDataSessionEndedError({ id }) {
+  const message = `Data session ${id} has ended; this request outlived its journey.`;
+  return new HTTPException(410, {
+    message,
+    res: new Response(JSON.stringify({ name: 'DataSessionEnded', message }), {
+      status: 410,
+      headers: { 'content-type': 'application/json' },
+    }),
+  });
+}
+
+export default createDataSessionEndedError;

package/lib/server/createDataSetRefusal.js

@@ -0,0 +1,26 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { ConfigError } from '@lowdefy/errors';
+
+function createDataSetRefusal({ artifact, session, reason }) {
+  return new ConfigError(
+    `Connection "${artifact.connectionId}" (type ${artifact.type}) cannot run under data set "${session.name}": ${reason}.`,
+    { configKey: artifact['~k'] }
+  );
+}
+
+export default createDataSetRefusal;

package/lib/server/createLowdefyContext.js

@@ -14,65 +14,81 @@
   limitations under the License.
 */
 
-import fs from 'node:fs';
 import path from 'node:path';
-import { createApiContext } from '@lowdefy/api';
+import {
+  createApiContext,
+  createRequestSignal,
+  normalizeInjectedCaller,
+  ensureMcpOauthResource,
+  resolveAuthentication,
+  resolvePinnedOrganization,
+  resolveTenantPreflight,
+} from '@lowdefy/api';
+import { type } from '@lowdefy/helpers';
 import { getSecretsFromEnv } from '@lowdefy/node-utils';
 import { v4 as uuid } from 'uuid';
 
 import agents from '../../build/plugins/agents.js';
 import appMeta from '../build/appMeta.js';
+import applyDataSetRedirect from './applyDataSetRedirect.js';
+import authJson from '../build/auth.js';
 import config from '../build/config.js';
 import connections from '../../build/plugins/connections.js';
+import createDataSessionEndedError from './createDataSessionEndedError.js';
 import createHandleError from './log/createHandleError.js';
 import createLogger from './log/createLogger.js';
+import createMutantReadConfigFile from './mutants/createMutantReadConfigFile.js';
 import fileCache from './fileCache.js';
-import getSession from './auth/session.js';
-import getStrategyCaller from './auth/strategies.js';
+import getAuth from './auth/getAuth.js';
+import getHeadlessUser from './auth/getHeadlessUser.js';
+import getMockUser from './auth/getMockUser.js';
+import getStrategies from './auth/getStrategies.js';
+import { forwardJourneyCookies } from './journeyCookies.js';
 import i18nConfig from '../build/i18n.js';
+import loadDynamicJsMap from './loadDynamicJsMap.js';
 import logRequest from './log/logRequest.js';
+import { readMutantRun } from './mutants/mutantRuns.js';
+import readDataSession from '../docs/dataSets/readDataSession.js';
+import readMergedMaps from './readMergedMaps.js';
+import readRecordingCookie from './recording/readRecordingCookie.js';
 import scrubSecrets from './scrubSecrets.js';
+import sinkExternalConnections from './sinkExternalConnections.js';
 import notifications, {
   interpolateProperties,
   renderEmail,
 } from '../../build/plugins/notifications.js';
 import operators from '../../build/plugins/operators/server.js';
-import staticJsMap from '../../build/plugins/operators/serverJsMap.js';
+import resolveHeadlessUser from './auth/resolveHeadlessUser.js';
+import steps from '../../build/plugins/steps.js';
+import trackSessionWork from './trackSessionWork.js';
 import websockets from '../../build/plugins/websockets.js';
 
 const secrets = getSecretsFromEnv();
 
-// Dynamic JS map loading for JIT-built pages — the build rewrites
-// serverJsMap.js when a JIT page discovers new _js operators.
-let cachedJsMapMtime = null;
-let cachedJsMap = staticJsMap;
-
-function loadDynamicJsMap(buildDirectory) {
-  const jsMapPath = path.join(buildDirectory, 'plugins', 'operators', 'serverJsMap.js');
-  try {
-    const stat = fs.statSync(jsMapPath);
-    if (cachedJsMapMtime && stat.mtimeMs === cachedJsMapMtime) {
-      return cachedJsMap;
-    }
-    cachedJsMapMtime = stat.mtimeMs;
-    // For server-side, we can read and eval the JS file
-    const content = fs.readFileSync(jsMapPath, 'utf8');
-    const fn = new Function('exports', content.replace('export default', 'exports.default ='));
-    const exports = {};
-    fn(exports);
-    cachedJsMap = { ...staticJsMap, ...(exports.default ?? {}) };
-    return cachedJsMap;
-  } catch {
-    return cachedJsMap;
-  }
+// The MCP route authenticates by access token alone - the mcp option switches
+// resolveAuthentication onto its bearer branch, so it is set only for the MCP
+// path. Every other path leaves it false and authentication behaves as before.
+function isMcpPath(path) {
+  return path.endsWith('/api/mcp');
 }
 
 // Builds the per-request lowdefy API context (connections, secrets,
-// operators, logger, session, etc.). Factored out of the /api/* middleware
-// (src/middleware/apiContext.js) so run_request (lib/docs/runRequest.js) can
-// build an identical context outside the Hono middleware chain, to call
-// callRequest directly for agent-driven request execution.
-async function createLowdefyContext({ c }) {
+// operators, logger, resolved user, etc.). Factored out of the /api/*
+// middleware (src/middleware/apiContext.js) so run_request
+// (lib/docs/runRequest.js) can build an identical context outside the Hono
+// middleware chain, to call callRequest directly for agent-driven request
+// execution. A `user` option injects a per-call caller (agent tools that run
+// outside a browser, e.g. run_request), resolved the same way the headless
+// renderer's cookie user is. A `dataSession` option runs a call the dev
+// server makes itself (run_request, run_endpoint on a data set) on that data
+// session's databases, as a journey's cookie does for its browser requests.
+async function createLowdefyContext({ c, user, dataSession: callDataSession }) {
+  // A journey on a data set: this request, and only this one, reads the session's databases.
+  // Read before anything else, so a request that outlived its session never reaches any database.
+  const dataSession = callDataSession ?? readDataSession(c.req.header('cookie'));
+  if (!type.isNone(dataSession?.ended)) {
+    throw createDataSessionEndedError({ id: dataSession.ended });
+  }
   const buildDirectory = path.join(process.cwd(), 'build');
   const jsMap = loadDynamicJsMap(buildDirectory);
 
@@ -96,7 +112,7 @@ async function createLowdefyContext({ c }) {
     i18n: i18nConfig,
     interpolateProperties,
     jsMap,
-    logger: createLogger(),
+    logger: createLogger({ rid }),
     mode: 'dev',
     notifications,
     operators,
@@ -108,19 +124,130 @@ async function createLowdefyContext({ c }) {
     },
     scrubSecrets,
     secrets,
+    // Error locations resolve through it (handleError, logClientError): JIT
+    // page builds write their keys to jitMaps/, not keyMap.json.
+    readMaps: () => readMergedMaps({ buildDirectory }),
+    // Aborts when the client disconnects before the response is sent, or when the
+    // request timeout (src/middleware/requestTimeout.js) answers first.
+    signal: createRequestSignal({
+      clientSignal: c.req.raw.signal,
+      timeoutSignal: c.get('requestTimeoutSignal'),
+    }),
+    steps,
     websockets,
   };
+  // The one writer of loopbackHeaders: a detached CallApi carries these on its
+  // loopback fetch, so the target runs with the journey's data set and mutant.
+  // With no journey cookie to forward it carries no Cookie header at all.
+  const journeyCookies = forwardJourneyCookies({
+    cookieHeader: c.req.header('cookie'),
+    dataSessionId: callDataSession?.id,
+  });
+  context.loopbackHeaders = journeyCookies === '' ? {} : { cookie: journeyCookies };
+  // Which browser context sent the request: errors it causes are stamped with
+  // it, so a journey run claims only its own (see createHandleError).
+  context.recording = readRecordingCookie(c.req.header('cookie'));
   context.handleError = createHandleError({ context });
-  if (!c.req.path.includes('/api/auth')) {
-    context.session = await getSession(c);
-    if (!context.session?.user) {
-      const caller = await getStrategyCaller(c, context.logger);
-      if (caller) {
-        context.session = { user: caller };
-      }
+  const mockUser = getMockUser();
+  const headlessCaller = getHeadlessUser(c);
+  if (!type.isNone(user)) {
+    // An explicit per-call user wins over the ambient auth.dev.mockUser: the
+    // caller named an identity for this call, so honour it. Merged over the
+    // roleless headless default by resolveHeadlessUser, exactly as the
+    // headless page tools' `user` param is.
+    context.auth = null;
+    context.user = normalizeInjectedCaller(resolveHeadlessUser({ user }));
+  } else if (headlessCaller?.explicit === true) {
+    // A headless context whose tool call or journey named its user (getBrowser.js)
+    // wins over the mock user for the same reason: a journey with
+    // user: { roles: [admin] } must act as that user, not as the mock user.
+    context.auth = null;
+    context.user = normalizeInjectedCaller(headlessCaller.user);
+  } else if (mockUser) {
+    // The mock user is a pre-resolved caller - it substitutes for the
+    // whole resolveAuthentication step and its roles are authoritative.
+    // No auth engine runs while dev.mockUser is active (see src/app.js),
+    // so mock mode never touches the auth database. normalizeInjectedCaller
+    // floors it to the resolved-caller shape.
+    context.auth = null;
+    context.user = normalizeInjectedCaller(mockUser);
+  } else if (headlessCaller !== null) {
+    // The headless renderer (docs/MCP screenshot and state tools) injects a
+    // user cookie on its own browser context (getBrowser.js), so its /api/*
+    // fetches carry a pre-resolved caller and auth-protected pages render.
+    // The developer's real browser has no cookie and resolves through the
+    // auth engine below, so it is unaffected. The roleless default it injects
+    // when no user was named yields to the mock user above.
+    context.auth = null;
+    context.user = normalizeInjectedCaller(headlessCaller.user);
+  } else if (!type.isNone(dataSession)) {
+    // A data-session request never builds the auth engine: it is bound to the real auth database.
+    // This is the detached hop (data cookie, no headless cookie); runDetachedEndpoint sets
+    // context.user from the principal the dispatcher carried. Auth steps fail, as for any injected
+    // caller.
+    context.auth = null;
+    context.user = null;
+  } else {
+    // Hoisted once per request - resolveAuthentication also needs it, and
+    // getBetterAuth memoizes the instance, but this keeps the auth engine
+    // construction to a single call site per request.
+    context.auth = getAuth({ logger: context.logger });
+    // The engine is constructed lazily on the first request, which would
+    // otherwise race the startup pinned-org ensure - await the memoized
+    // resolve so createApiContext reads a retained binding.
+    await resolvePinnedOrganization({ auth: context.auth, logger: context.logger });
+    // The one oauthResource row the MCP token audience validates against -
+    // ensured once per process, memoized like the pinned resolve above; a
+    // failure retries on the next request. No-op when the app is not an
+    // authorization server.
+    await ensureMcpOauthResource({ auth: context.auth, logger: context.logger });
+    if (!c.req.path.includes('/api/auth')) {
+      // resolveAuthentication is the single writer of context.user.
+      await resolveAuthentication(context, {
+        auth: context.auth,
+        headers: c.req.raw.headers,
+        strategies: getStrategies({ logger: context.logger }),
+        mcp: isMcpPath(c.req.path),
+      });
     }
   }
   createApiContext(context);
+  // A journey's browser contexts carry a mutant cookie while `lowdefy journeys
+  // harden` runs a mutant through them: their requests, and only theirs, read
+  // the mutated artifact. The developer's own tabs carry none.
+  const mutantRun = readMutantRun(c.req.header('cookie'));
+  if (mutantRun !== null) {
+    context.readConfigFile = createMutantReadConfigFile({
+      readConfigFile: context.readConfigFile,
+      run: mutantRun,
+    });
+  }
+  if (!type.isNone(dataSession)) {
+    applyDataSetRedirect({ context, session: dataSession, connections });
+    // Requests of connections that reach an outside service are recorded, never sent.
+    context.connections = sinkExternalConnections({
+      context,
+      connections,
+      session: dataSession,
+    });
+    context.dataSet = dataSession.name;
+    context.waitUntil = trackSessionWork({ session: dataSession });
+  }
+  if (!context.auth && authJson.organizations) {
+    // Mock and headless callers run no auth engine, so createApiContext
+    // retains no organization binding. Derive the policy from the built auth
+    // config so the tenant wall's policy gate still engages under
+    // policy: tenant (a mock caller there must carry organizationId, or the
+    // wall fails closed - loudly, as it should).
+    context.organization = { policy: authJson.organizations.policy, pinned: null };
+  }
+  // Under policy: tenant, report walled collections that hold unstamped rows
+  // (lazily-run-once, logged once per offending collection). It never blocks or
+  // fails a request. No-op under pinned. Skipped under a data session: the report runs once per
+  // process, so it would describe the data set's database and never the developer's.
+  if (type.isNone(dataSession)) {
+    void resolveTenantPreflight(context);
+  }
   logRequest({ context });
   return context;
 }