@lowdefy/server-dev 0.0.0-alpha.7 → 0.0.0-experimental-20231123124425

package/manager/processes/startServer.mjs

@@ -0,0 +1,46 @@
+/*
+  Copyright 2020-2023 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { spawnProcess } from '@lowdefy/node-utils';
+import createStdOutLineHandler from '../utils/createStdOutLineHandler.mjs';
+
+function startServer(context) {
+  context.shutdownServer();
+
+  const nextServer = spawnProcess({
+    stdOutLineHandler: createStdOutLineHandler({ context }),
+    stdErrLineHandler: (line) => context.logger.error(line),
+    command: 'node',
+    args: [context.bin.next, 'start'],
+    processOptions: {
+      env: {
+        ...process.env,
+        PORT: context.options.port,
+      },
+    },
+    returnProcess: true,
+  });
+  context.logger.debug(`Started next server with pid ${nextServer.pid}.`);
+  nextServer.on('exit', (code, signal) => {
+    context.logger.debug(`nextServer exit ${nextServer.pid}, signal: ${signal}, code: ${code}`);
+  });
+  nextServer.on('error', (error) => {
+    context.logger.error(error);
+  });
+  context.nextServer = nextServer;
+}
+
+export default startServer;

package/manager/processes/startWatchers.mjs

@@ -0,0 +1,31 @@
+/*
+  Copyright 2020-2023 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import envWatcher from '../watchers/envWatcher.mjs';
+import lowdefyBuildWatcher from '../watchers/lowdefyBuildWatcher.mjs';
+import nextBuildWatcher from '../watchers/nextBuildWatcher.mjs';
+
+function startWatchers(context) {
+  return async () => {
+    await Promise.all([
+      envWatcher(context),
+      lowdefyBuildWatcher(context),
+      nextBuildWatcher(context),
+    ]);
+  };
+}
+
+export default startWatchers;

package/manager/run.mjs

@@ -0,0 +1,109 @@
+#!/usr/bin/env node
+/*
+  Copyright 2020-2023 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { wait } from '@lowdefy/helpers';
+import opener from 'opener';
+import getContext from './getContext.mjs';
+import startServer from './processes/startServer.mjs';
+
+/*
+The run script does the following:
+  - Run the initial Lowdefy build, install plugins, and next build and read .env
+  - Start file watchers to reload config and restart server if necessary
+  - Start the server
+  - Open a browser window.
+
+  Three watchers are started:
+
+  ## Lowdefy build watcher
+  Watches:
+    - <config-dir>,
+    - <watch-dirs>
+    - !<ignore-dirs>
+  The Lowdefy build watcher watches the Lowdefy config files for changes
+  and runs Lowdefy build when they change, and triggers a soft reload.
+
+  If lowdefy version in lowdefy.yaml
+  is changed, the server warns and exits.
+
+  ## .env watcher
+
+  If the .env file is changed, the new file is parsed, and the server restarted with the new env
+  and the server hard reloads.
+
+  ## Next build watcher
+
+  The Next build watcher watches for any files where the app should be rebuilt and restarted.
+  It watches:
+  - <build-dir>/plugins/**
+  - <build-dir>/config.json
+  - <server-dir>/package.json
+
+  If app config changes:
+    - <build-dir>/config.json changes, rebuild and restart server.
+
+  If user styles change:
+    - <public-dir>/styles.less changes, rebuild and restart server.
+
+  If new plugin type in an existing plugin package is used:
+    - <build-dir>/plugins/** changes,  rebuild next and restart server.
+
+  If new plugin type in a new plugin package is used:
+    - <server-dir>/package.json changes,  run npm install, rebuild next and restart server.
+
+  # Reload mechanism
+
+  The web client creates a Server Sent Events connection with the server on the /api/reload route.
+  The server watches the <build-dir>/reload file, which is written every time the server should reload,
+  and sends an event to the client to reload the config. The client then uses a SWR cache mutation to
+  fetch the new config.
+
+  If the server is restarted, the event stream is closed because the original server was shut down. The client starts
+  pinging the /api/ping route, until it detects a new server has started, and then reloads the window.
+ */
+
+/* TODO:
+Not killing server on errors properly
+when:
+- initial build fails
+*/
+
+const context = await getContext();
+
+try {
+  try {
+    await context.initialBuild();
+  } catch (error) {
+    context.logger.error(error);
+  }
+
+  // We are not waiting for the startWatchers promise to resolve (all watchers have fired the ready event)
+  // because chokidar sometimes doesn't fire this event, and it seems like there isn't an issue with not waiting.
+  context.startWatchers();
+
+  startServer(context);
+  await wait(800);
+  if (process.env.LOWDEFY_SERVER_DEV_OPEN_BROWSER === 'true') {
+    // TODO: Wait 1 sec for a ping and don't open if a ping is seen
+    opener(`http://localhost:${context.options.port}`);
+  }
+  await new Promise(() => {});
+} catch (error) {
+  context.logger.error(error);
+  context.shutdownServer();
+  process.exit();
+}

package/manager/utils/BatchChanges.mjs

@@ -0,0 +1,64 @@
+/*
+  Copyright 2020-2023 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { type } from '@lowdefy/helpers';
+
+class BatchChanges {
+  constructor({ context, fn, delay }) {
+    this.context = context;
+    this._call = this._call.bind(this);
+    this.args = [];
+    this.delay = delay || 500;
+    this.fn = fn;
+    this.repeat = false;
+    this.running = false;
+  }
+
+  newChange(...args) {
+    this.args.push(args.filter((arg) => type.isString(arg))); // filter for string paths since chokidar also returns an stats object on windows.
+    this._startTimer();
+  }
+
+  _startTimer() {
+    if (this.timer) {
+      clearTimeout(this.timer);
+    }
+    if (this.running) {
+      this.repeat = true;
+    } else {
+      this.timer = setTimeout(this._call, this.delay);
+    }
+  }
+
+  async _call() {
+    this.running = true;
+    try {
+      const args = this.args;
+      this.args = [];
+      await this.fn(args);
+      this.running = false;
+      if (this.repeat) {
+        this.repeat = false;
+        this._call();
+      }
+    } catch (error) {
+      this.running = false;
+      this.context.logger.error(error?.message ?? error);
+    }
+  }
+}
+
+export default BatchChanges;

package/manager/utils/createCustomPluginTypesMap.mjs

@@ -0,0 +1,80 @@
+#!/usr/bin/env node
+/*
+  Copyright 2020-2023 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import path from 'path';
+import { get } from '@lowdefy/helpers';
+import { readFile } from '@lowdefy/node-utils';
+import { createPluginTypesMap } from '@lowdefy/build';
+import YAML from 'yaml';
+
+async function getPluginDefinitions({ directories }) {
+  let lowdefyYaml = await readFile(path.join(directories.config, 'lowdefy.yaml'));
+  if (!lowdefyYaml) {
+    lowdefyYaml = await readFile(path.join(directories.config, 'lowdefy.yml'));
+  }
+  const lowdefy = YAML.parse(lowdefyYaml);
+  return get(lowdefy, 'plugins', { default: [] });
+}
+
+async function createCustomPluginTypesMap({ directories, logger }) {
+  const customTypesMap = {
+    actions: {},
+    auth: {
+      adapters: {},
+      callbacks: {},
+      events: {},
+      providers: {},
+    },
+    blocks: {},
+    connections: {},
+    icons: {},
+    operators: {
+      client: {},
+      server: {},
+    },
+    requests: {},
+    styles: {
+      packages: {},
+      blocks: {},
+    },
+  };
+
+  const pluginDefinitions = await getPluginDefinitions({ directories });
+
+  for (const plugin of pluginDefinitions) {
+    let types;
+    try {
+      types = (await import(`${plugin.name}/types`)).default;
+    } catch (e) {
+      logger.error(`Failed to import plugin "${plugin.name}".`);
+      logger.debug(e);
+      logger.info('If the plugin was added while the server was running, restart the server.');
+      throw new Error(`Failed to import plugin "${plugin.name}".`);
+    }
+    createPluginTypesMap({
+      packageTypes: types,
+      typesMap: customTypesMap,
+      packageName: plugin.name,
+      version: plugin.version,
+      typePrefix: plugin.typePrefix,
+    });
+  }
+
+  return customTypesMap;
+}
+
+export default createCustomPluginTypesMap;

package/manager/utils/createLogger.mjs

@@ -0,0 +1,33 @@
+/*
+  Copyright 2020-2023 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+import pino from 'pino';
+
+function createLogger({ level = 'info' }) {
+  const logger = pino({
+    name: 'lowdefy build',
+    level,
+    base: { pid: undefined, hostname: undefined },
+    mixin: (context, level) => {
+      return {
+        ...context,
+        print: context.print ?? logger.levels.labels[level],
+      };
+    },
+  });
+  return logger;
+}
+
+export default createLogger;

package/manager/utils/createStdOutLineHandler.mjs

@@ -0,0 +1,33 @@
+/*
+  Copyright 2020-2023 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+function createStdOutLineHandler({ context }) {
+  const { logger } = context;
+  function stdOutLineHandler(line) {
+    try {
+      const { level, name, time, rid, ...msgObj } = JSON.parse(line);
+      if (level) {
+        const print = level === 30 ? 'log' : logger.levels.labels[level];
+        logger[logger.levels.labels[level]]({ print }, JSON.stringify(msgObj));
+      }
+    } catch (error) {
+      logger.info({ print: 'log' }, line);
+    }
+  }
+  return stdOutLineHandler;
+}
+
+export default createStdOutLineHandler;

package/manager/utils/getLowdefyVersion.mjs

@@ -0,0 +1,51 @@
+/*
+  Copyright 2020-2023 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import path from 'path';
+import { type } from '@lowdefy/helpers';
+import { readFile } from '@lowdefy/node-utils';
+import YAML from 'yaml';
+
+async function getLowdefyVersion(context) {
+  let lowdefyYaml = await readFile(path.join(context.directories.config, 'lowdefy.yaml'));
+  if (!lowdefyYaml) {
+    lowdefyYaml = await readFile(path.join(context.directories.config, 'lowdefy.yml'));
+  }
+  if (!lowdefyYaml) {
+    throw new Error(`Could not find "lowdefy.yaml" file.`);
+  }
+  let lowdefy;
+  try {
+    lowdefy = YAML.parse(lowdefyYaml);
+  } catch (error) {
+    throw new Error(`Could not parse "lowdefy.yaml" file. Received error ${error.message}.`);
+  }
+  if (!lowdefy.lowdefy) {
+    throw new Error(
+      `No version specified in "lowdefy.yaml" file. Specify a version in the "lowdefy" field.`
+    );
+  }
+  if (!type.isString(lowdefy.lowdefy)) {
+    throw new Error(
+      `Version number specified in "lowdefy.yaml" file should be a string. Received ${JSON.stringify(
+        lowdefy.lowdefy
+      )}.`
+    );
+  }
+  return lowdefy.lowdefy;
+}
+
+export default getLowdefyVersion;

package/manager/utils/getNextBin.mjs

@@ -0,0 +1,36 @@
+/*
+  Copyright 2020-2023 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import path from 'path';
+import { createRequire } from 'module';
+
+function getNextBin() {
+  const require = createRequire(import.meta.url);
+  const nextPackageJson = require('next/package.json');
+
+  const nextPath = require.resolve('next');
+  let nextMainFragment = nextPackageJson.main.substring(1);
+  let nextBinFragment = nextPackageJson.bin.next;
+
+  if (process.platform === 'win32') {
+    nextMainFragment = nextMainFragment.replaceAll('/', '\\');
+    nextBinFragment = nextBinFragment.replaceAll('/', '\\');
+  }
+
+  return path.join(nextPath.replace(nextMainFragment, ''), nextBinFragment);
+}
+
+export default getNextBin;

package/manager/utils/setupWatcher.mjs

@@ -0,0 +1,50 @@
+/*
+  Copyright 2020-2023 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import chokidar from 'chokidar';
+import BatchChanges from './BatchChanges.mjs';
+
+function setupWatcher({
+  callback,
+  context,
+  watchDotfiles = false,
+  ignorePaths = [],
+  watchPaths,
+  delay = 500,
+}) {
+  return new Promise((resolve) => {
+    // const { watch = [], watchIgnore = [] } = context.options;
+    // const resolvedWatchPaths = watch.map((pathName) => path.resolve(pathName));
+
+    const batchChanges = new BatchChanges({ context, fn: callback, delay });
+    const defaultIgnorePaths = watchDotfiles
+      ? []
+      : [
+          /(^|[/\\])\../, // ignore dotfiles
+        ];
+    const configWatcher = chokidar.watch(watchPaths, {
+      ignored: [...defaultIgnorePaths, ...ignorePaths],
+      persistent: true,
+      ignoreInitial: true,
+    });
+    configWatcher.on('add', (...args) => batchChanges.newChange(...args));
+    configWatcher.on('change', (...args) => batchChanges.newChange(...args));
+    configWatcher.on('unlink', (...args) => batchChanges.newChange(...args));
+    configWatcher.on('ready', () => resolve());
+  });
+}
+
+export default setupWatcher;

package/manager/watchers/envWatcher.mjs

@@ -0,0 +1,35 @@
+/*
+  Copyright 2020-2023 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import path from 'path';
+import setupWatcher from '../utils/setupWatcher.mjs';
+
+function envWatcher(context) {
+  const callback = async () => {
+    context.logger.warn('.env file changed.');
+    context.readDotEnv();
+    await context.lowdefyBuild();
+    context.restartServer();
+  };
+  return setupWatcher({
+    callback,
+    context,
+    watchDotfiles: true,
+    watchPaths: [path.join(context.directories.config, '.env')],
+  });
+}
+
+export default envWatcher;

package/manager/watchers/lowdefyBuildWatcher.mjs

@@ -0,0 +1,55 @@
+/*
+  Copyright 2020-2023 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import path from 'path';
+import getLowdefyVersion from '../utils/getLowdefyVersion.mjs';
+import setupWatcher from '../utils/setupWatcher.mjs';
+
+function lowdefyBuildWatcher(context) {
+  const fixRelativePathConfigDir = (item) =>
+    path.isAbsolute(item) ? item : path.resolve(context.directories.config, item);
+
+  const callback = async (filePaths) => {
+    const lowdefyYamlModified = filePaths
+      .flat()
+      .some((filePath) => filePath.includes('lowdefy.yaml') || filePath.includes('lowdefy.yml'));
+    if (lowdefyYamlModified) {
+      const lowdefyVersion = await getLowdefyVersion(context);
+      if (lowdefyVersion !== context.version && lowdefyVersion !== 'local') {
+        context.shutdownServer();
+        context.logger.warn('Lowdefy version changed. You should restart your development server.');
+        process.exit();
+      }
+    }
+
+    await context.lowdefyBuild();
+    context.reloadClients();
+  };
+  return setupWatcher({
+    callback,
+    context,
+    ignorePaths: [
+      '**/node_modules/**',
+      ...context.options.watchIgnore.map(fixRelativePathConfigDir),
+    ],
+    watchPaths: [
+      context.directories.config,
+      ...context.options.watch.map(fixRelativePathConfigDir),
+    ],
+  });
+}
+
+export default lowdefyBuildWatcher;