@lowdefy/server-dev 4.5.1 → 4.6.0

package/lib/server/apiWrapper.js

@@ -1,5 +1,5 @@
 /*
-  Copyright 2020-2024 Lowdefy, Inc
+  Copyright 2020-2026 Lowdefy, Inc
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -13,35 +13,68 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 */
+import fs from 'fs';
 import path from 'path';
 import { createApiContext } from '@lowdefy/api';
 import { getSecretsFromEnv } from '@lowdefy/node-utils';
+import { serializer } from '@lowdefy/helpers';
 import { v4 as uuid } from 'uuid';
 
-import config from '../../build/config.json';
+import config from '../build/config.js';
 import connections from '../../build/plugins/connections.js';
 import createLogger from './log/createLogger.js';
 import fileCache from './fileCache.js';
 import getServerSession from './auth/getServerSession.js';
-import logError from './log/logError.js';
+import createHandleError from './log/createHandleError.js';
 import logRequest from './log/logRequest.js';
 import operators from '../../build/plugins/operators/server.js';
-import jsMap from '../../build/plugins/operators/serverJsMap.js';
+import staticJsMap from '../../build/plugins/operators/serverJsMap.js';
 import getAuthOptions from './auth/getAuthOptions.js';
 
 const secrets = getSecretsFromEnv();
 
+// Dynamic JS map loading for JIT-built pages
+let cachedJsMapMtime = null;
+let cachedJsMap = staticJsMap;
+
+function loadDynamicJsMap(buildDirectory) {
+  const jsMapPath = path.join(buildDirectory, 'plugins', 'operators', 'serverJsMap.js');
+  try {
+    const stat = fs.statSync(jsMapPath);
+    if (cachedJsMapMtime && stat.mtimeMs === cachedJsMapMtime) {
+      return cachedJsMap;
+    }
+    cachedJsMapMtime = stat.mtimeMs;
+    // For server-side, we can read and eval the JS file
+    const content = fs.readFileSync(jsMapPath, 'utf8');
+    const fn = new Function('exports', content.replace('export default', 'exports.default ='));
+    const exports = {};
+    fn(exports);
+    cachedJsMap = { ...staticJsMap, ...(exports.default ?? {}) };
+    return cachedJsMap;
+  } catch {
+    return cachedJsMap;
+  }
+}
+
 function apiWrapper(handler) {
   return async function wrappedHandler(req, res) {
+    const buildDirectory = path.join(process.cwd(), 'build');
+    const jsMap = loadDynamicJsMap(buildDirectory);
+
     const context = {
       // Important to give absolute path so Next can trace build files
       rid: uuid(),
-      buildDirectory: path.join(process.cwd(), 'build'),
+      buildDirectory,
+      configDirectory: process.env.LOWDEFY_DIRECTORY_CONFIG || process.cwd(),
       config,
       connections,
       fileCache,
       headers: req?.headers,
       jsMap,
+      handleError: async (err) => {
+        console.error(err);
+      },
       logger: console,
       operators,
       req,
@@ -49,7 +82,8 @@ function apiWrapper(handler) {
       secrets,
     };
     try {
-      context.logger = createLogger({ rid: context.rid });
+      context.logger = createLogger();
+      context.handleError = createHandleError({ context });
       context.authOptions = getAuthOptions(context);
       if (!req.url.startsWith('/api/auth')) {
         context.session = await getServerSession(context);
@@ -61,8 +95,14 @@ function apiWrapper(handler) {
       // TODO: Log response time?
       return response;
     } catch (error) {
-      logError({ error, context });
-      res.status(500).json({ name: error.name, message: error.message });
+      await context.handleError(error);
+      const serialized = serializer.serialize(error);
+      if (serialized?.['~e']) {
+        delete serialized['~e'].received;
+        delete serialized['~e'].stack;
+        delete serialized['~e'].configKey;
+      }
+      res.status(500).json(serialized);
     }
   };
 }

package/lib/server/auth/getAuthOptions.js

@@ -1,5 +1,5 @@
 /*
-  Copyright 2020-2024 Lowdefy, Inc
+  Copyright 2020-2026 Lowdefy, Inc
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@ import { getNextAuthConfig } from '@lowdefy/api';
 import { getSecretsFromEnv } from '@lowdefy/node-utils';
 
 import adapters from '../../../build/plugins/auth/adapters.js';
-import authJson from '../../../build/auth.json';
+import authJson from '../../build/auth.js';
 import callbacks from '../../../build/plugins/auth/callbacks.js';
 import events from '../../../build/plugins/auth/events.js';
 import providers from '../../../build/plugins/auth/providers.js';

package/lib/server/auth/getMockSession.js

@@ -0,0 +1,67 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { createSessionCallback } from '@lowdefy/api';
+import { serializer } from '@lowdefy/helpers';
+
+import authJson from '../../build/auth.js';
+import callbacks from '../../../build/plugins/auth/callbacks.js';
+
+async function getMockSession() {
+  const mockUserJson = process.env.LOWDEFY_DEV_USER;
+  let mockUser;
+
+  if (mockUserJson) {
+    try {
+      mockUser = JSON.parse(mockUserJson);
+    } catch (error) {
+      throw new Error('Invalid JSON in LOWDEFY_DEV_USER environment variable.', { cause: error });
+    }
+  } else {
+    mockUser = authJson.dev?.mockUser;
+  }
+
+  if (!mockUser) {
+    return undefined;
+  }
+
+  // Deserialize to restore arrays from ~arr markers and remove other build markers
+  mockUser = serializer.deserialize(mockUser);
+
+  if (authJson.configured !== true) {
+    throw new Error(
+      'Mock user configured but auth is not configured in lowdefy.yaml. ' +
+        'Add auth configuration to use mock user feature.'
+    );
+  }
+
+  // Create session callback to transform mock user
+  const sessionCallback = createSessionCallback({
+    authConfig: authJson,
+    plugins: { callbacks },
+  });
+
+  // Transform mock user through session callback (mock user acts as token)
+  const session = await sessionCallback({
+    session: { user: {} },
+    token: mockUser,
+    user: mockUser,
+  });
+
+  return session;
+}
+
+export default getMockSession;

package/lib/server/auth/getServerSession.js

@@ -1,5 +1,5 @@
 /*
-  Copyright 2020-2024 Lowdefy, Inc
+  Copyright 2020-2026 Lowdefy, Inc
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -16,9 +16,16 @@
 
 import { getServerSession as getNextAuthServerSession } from 'next-auth/next';
 
-import authJson from '../../../build/auth.json';
+import authJson from '../../build/auth.js';
+import getMockSession from './getMockSession.js';
+
+async function getServerSession({ authOptions, req, res }) {
+  // Check for mock user first (dev server only)
+  const mockSession = await getMockSession();
+  if (mockSession) {
+    return mockSession;
+  }
 
-function getServerSession({ authOptions, req, res }) {
   if (authJson.configured === true) {
     return getNextAuthServerSession(req, res, authOptions);
   }

package/lib/server/fileCache.js

@@ -1,5 +1,5 @@
 /*
-  Copyright 2020-2024 Lowdefy, Inc
+  Copyright 2020-2026 Lowdefy, Inc
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.

package/lib/server/jitPageBuilder.js

@@ -0,0 +1,162 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import fs from 'fs';
+import path from 'path';
+import { serializer } from '@lowdefy/helpers';
+import { buildPageJit, createContext } from '@lowdefy/build/dev';
+
+import createLogger from './log/createLogger.js';
+import PageCache from './pageCache.mjs';
+
+const jitLogger = createLogger({ name: 'jit-build' });
+
+function formatDuration(ms) {
+  if (ms < 1000) return `${ms}ms`;
+  return `${(ms / 1000).toFixed(2)}s`;
+}
+const pageCache = new PageCache();
+let cachedRegistryMtime = null;
+let cachedRegistry = null;
+let cachedBuildContext = null;
+let lastInvalidationMtime = null;
+
+function readJsonFile(filePath) {
+  try {
+    const content = fs.readFileSync(filePath, 'utf8');
+    return serializer.deserialize(JSON.parse(content));
+  } catch {
+    return null;
+  }
+}
+
+function checkPageInvalidations(buildDirectory) {
+  const invalidatePath = path.join(buildDirectory, 'invalidatePages');
+  try {
+    const stat = fs.statSync(invalidatePath);
+    if (lastInvalidationMtime && stat.mtimeMs === lastInvalidationMtime) {
+      return;
+    }
+    lastInvalidationMtime = stat.mtimeMs;
+    pageCache.invalidateAll();
+    cachedBuildContext = null;
+  } catch {
+    // File doesn't exist yet — nothing to invalidate
+  }
+}
+
+function loadPageRegistry(buildDirectory) {
+  const registryPath = path.join(buildDirectory, 'pageRegistry.json');
+  try {
+    const stat = fs.statSync(registryPath);
+    // Only reload if file has changed
+    if (cachedRegistryMtime && stat.mtimeMs === cachedRegistryMtime) {
+      return cachedRegistry;
+    }
+    cachedRegistryMtime = stat.mtimeMs;
+    cachedRegistry = readJsonFile(registryPath);
+    // Invalidate all pages when registry changes (skeleton rebuild happened)
+    pageCache.invalidateAll();
+    cachedBuildContext = null;
+    return cachedRegistry;
+  } catch {
+    return null;
+  }
+}
+
+function getBuildContext(buildDirectory, configDirectory) {
+  if (cachedBuildContext) return cachedBuildContext;
+
+  const refMap = readJsonFile(path.join(buildDirectory, 'refMap.json')) ?? {};
+  const keyMap = readJsonFile(path.join(buildDirectory, 'keyMap.json')) ?? {};
+  const jsMap = readJsonFile(path.join(buildDirectory, 'jsMap.json')) ?? { client: {}, server: {} };
+  const connectionIds = readJsonFile(path.join(buildDirectory, 'connectionIds.json')) ?? [];
+
+  const customTypesMap = readJsonFile(path.join(buildDirectory, 'customTypesMap.json')) ?? {};
+
+  cachedBuildContext = createContext({
+    customTypesMap,
+    directories: {
+      build: buildDirectory,
+      config: configDirectory,
+      server: path.resolve(buildDirectory, '..'),
+    },
+    logger: jitLogger,
+    stage: 'dev',
+  });
+
+  // Restore refMap, keyMap, jsMap, and connectionIds from skeleton build
+  Object.assign(cachedBuildContext.refMap, refMap);
+  Object.assign(cachedBuildContext.keyMap, keyMap);
+  cachedBuildContext.jsMap.client = jsMap.client ?? {};
+  cachedBuildContext.jsMap.server = jsMap.server ?? {};
+  for (const id of connectionIds) {
+    cachedBuildContext.connectionIds.add(id);
+  }
+
+  // Load installed packages snapshot from skeleton build for missing-package detection
+  const installedPluginPackages =
+    readJsonFile(path.join(buildDirectory, 'installedPluginPackages.json')) ?? [];
+  cachedBuildContext.installedPluginPackages = new Set(installedPluginPackages);
+
+  return cachedBuildContext;
+}
+
+async function buildPageIfNeeded({ pageId, buildDirectory, configDirectory }) {
+  checkPageInvalidations(buildDirectory);
+  const registry = loadPageRegistry(buildDirectory);
+  if (!registry || !registry[pageId]) {
+    return false;
+  }
+
+  if (pageCache.isCompiled(pageId)) {
+    return true;
+  }
+
+  const shouldBuild = await pageCache.acquireBuildLock(pageId);
+  if (!shouldBuild) {
+    // Another request completed the build
+    return true;
+  }
+
+  jitLogger.info({ spin: 'start' }, `Building page "${pageId}"...`);
+  const startTime = Date.now();
+  try {
+    const context = getBuildContext(buildDirectory, configDirectory);
+    const result = await buildPageJit({
+      pageId,
+      pageRegistry: registry,
+      context,
+    });
+    if (result && result.installing) {
+      jitLogger.info(
+        `Installing plugin packages for page "${pageId}": ${result.packages.join(', ')}. ` +
+          'The page will be available after the server restarts.'
+      );
+      return result;
+    }
+    pageCache.markCompiled(pageId);
+    jitLogger.info(
+      { spin: 'succeed', color: 'white' },
+      `Built page "${pageId}" in ${formatDuration(Date.now() - startTime)}.`
+    );
+    return true;
+  } finally {
+    pageCache.releaseBuildLock(pageId);
+  }
+}
+
+export default buildPageIfNeeded;

package/lib/server/log/createHandleError.js

@@ -0,0 +1,47 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { LowdefyInternalError, loadAndResolveErrorLocation } from '@lowdefy/errors';
+
+function createHandleError({ context }) {
+  return async function handleError(error) {
+    try {
+      // For internal lowdefy errors, don't resolve config location
+      const location =
+        error instanceof LowdefyInternalError
+          ? null
+          : await loadAndResolveErrorLocation({
+              error,
+              readConfigFile: context.readConfigFile,
+              configDirectory: context.configDirectory,
+            });
+
+      // Attach resolved location to error for display layer
+      if (location) {
+        error.source = location.source;
+        error.config = location.config;
+      }
+
+      context.logger.error(error);
+    } catch (e) {
+      console.error(error);
+      console.error('An error occurred while logging the error.');
+      console.error(e);
+    }
+  };
+}
+
+export default createHandleError;

package/lib/server/log/createLogger.js

@@ -1,5 +1,5 @@
 /*
-  Copyright 2020-2024 Lowdefy, Inc
+  Copyright 2020-2026 Lowdefy, Inc
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -14,10 +14,10 @@
   limitations under the License.
 */
 
-import pino from 'pino';
+import { createNodeLogger } from '@lowdefy/logger/node';
 
-const logger = pino({
-  name: 'lowdefy_server',
+const logger = createNodeLogger({
+  name: 'lowdefy_server_dev',
   level: process.env.LOWDEFY_LOG_LEVEL ?? 'info',
   base: { pid: undefined, hostname: undefined },
 });

package/lib/server/log/logRequest.js

@@ -1,5 +1,5 @@
 /*
-  Copyright 2020-2024 Lowdefy, Inc
+  Copyright 2020-2026 Lowdefy, Inc
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.

package/lib/server/pageCache.mjs

@@ -0,0 +1,61 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+class PageCache {
+  constructor() {
+    this.compiledPages = new Set();
+    this.buildLocks = new Map();
+  }
+
+  isCompiled(pageId) {
+    return this.compiledPages.has(pageId);
+  }
+
+  markCompiled(pageId) {
+    this.compiledPages.add(pageId);
+  }
+
+  async acquireBuildLock(pageId) {
+    // If page build already in progress, wait for it
+    if (this.buildLocks.has(pageId)) {
+      await this.buildLocks.get(pageId);
+      return false;
+    }
+
+    // Create new lock
+    let resolve;
+    const promise = new Promise((r) => {
+      resolve = r;
+    });
+    promise.resolve = resolve;
+    this.buildLocks.set(pageId, promise);
+    return true;
+  }
+
+  releaseBuildLock(pageId) {
+    const lock = this.buildLocks.get(pageId);
+    if (lock) {
+      lock.resolve();
+      this.buildLocks.delete(pageId);
+    }
+  }
+
+  invalidateAll() {
+    this.compiledPages.clear();
+  }
+}
+
+export default PageCache;

package/lib/server/pageCache.test.mjs

@@ -0,0 +1,80 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import PageCache from './pageCache.mjs';
+
+test('isCompiled returns false for uncompiled page', () => {
+  const cache = new PageCache();
+  expect(cache.isCompiled('home')).toBe(false);
+});
+
+test('isCompiled returns true after markCompiled', () => {
+  const cache = new PageCache();
+  cache.markCompiled('home');
+  expect(cache.isCompiled('home')).toBe(true);
+});
+
+test('invalidateAll clears all compiled pages', () => {
+  const cache = new PageCache();
+  cache.markCompiled('home');
+  cache.markCompiled('dashboard');
+  cache.invalidateAll();
+  expect(cache.isCompiled('home')).toBe(false);
+  expect(cache.isCompiled('dashboard')).toBe(false);
+});
+
+test('acquireBuildLock returns true for first request', async () => {
+  const cache = new PageCache();
+  const shouldBuild = await cache.acquireBuildLock('home');
+  expect(shouldBuild).toBe(true);
+  cache.releaseBuildLock('home');
+});
+
+test('acquireBuildLock returns false for concurrent request (waits for first)', async () => {
+  const cache = new PageCache();
+
+  // First request acquires lock
+  const shouldBuild1 = await cache.acquireBuildLock('home');
+  expect(shouldBuild1).toBe(true);
+
+  // Second request waits for lock and returns false
+  const promise2 = cache.acquireBuildLock('home');
+
+  // Release first lock
+  cache.releaseBuildLock('home');
+
+  const shouldBuild2 = await promise2;
+  expect(shouldBuild2).toBe(false);
+});
+
+test('acquireBuildLock for different pages does not block', async () => {
+  const cache = new PageCache();
+
+  const shouldBuild1 = await cache.acquireBuildLock('home');
+  const shouldBuild2 = await cache.acquireBuildLock('dashboard');
+
+  expect(shouldBuild1).toBe(true);
+  expect(shouldBuild2).toBe(true);
+
+  cache.releaseBuildLock('home');
+  cache.releaseBuildLock('dashboard');
+});
+
+test('releaseBuildLock does nothing for non-existent lock', () => {
+  const cache = new PageCache();
+  // Should not throw
+  cache.releaseBuildLock('nonexistent');
+});

package/manager/getContext.mjs

@@ -1,5 +1,5 @@
 /*
-  Copyright 2020-2024 Lowdefy, Inc
+  Copyright 2020-2026 Lowdefy, Inc
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -19,7 +19,9 @@ import path from 'path';
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 
-import createLogger from './utils/createLogger.mjs';
+import pino from 'pino';
+import { createNodeLogger } from '@lowdefy/logger/node';
+import checkMockUserWarning from './processes/checkMockUserWarning.mjs';
 import initialBuild from './processes/initialBuild.mjs';
 import installPlugins from './processes/installPlugins.mjs';
 import lowdefyBuild from './processes/lowdefyBuild.mjs';
@@ -46,7 +48,12 @@ async function getContext() {
       config: path.resolve(argv.configDirectory ?? env.LOWDEFY_DIRECTORY_CONFIG ?? process.cwd()),
       server: process.cwd(),
     },
-    logger: createLogger({ level: env.LOWDEFY_LOG_LEVEL }),
+    logger: createNodeLogger({
+      name: 'lowdefy build',
+      level: env.LOWDEFY_LOG_LEVEL ?? 'info',
+      base: { pid: undefined, hostname: undefined },
+      destination: pino.destination({ dest: 1, sync: true }),
+    }),
     options: {
       port: argv.port ?? env.PORT ?? 3000,
       refResolver: argv.refResolver ?? env.LOWDEFY_BUILD_REF_RESOLVER,
@@ -58,12 +65,27 @@ async function getContext() {
           : [],
     },
     version: env.npm_package_version,
+
+    // JIT build state
+    pageRegistry: null,
+    buildContext: null,
   };
 
   context.packageManagerCmd = process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm';
+  context.checkMockUserWarning = checkMockUserWarning(context);
   context.initialBuild = initialBuild(context);
   context.installPlugins = installPlugins(context);
-  context.lowdefyBuild = lowdefyBuild(context);
+
+  // Wrap lowdefyBuild to capture and store the shallow build result
+  const buildFn = lowdefyBuild(context);
+  context.lowdefyBuild = async () => {
+    const result = await buildFn();
+    if (result) {
+      context.pageRegistry = result.pageRegistry;
+      context.buildContext = result.context;
+    }
+  };
+
   context.nextBuild = nextBuild(context);
   context.readDotEnv = readDotEnv(context);
   context.reloadClients = reloadClients(context);