@underpostnet/underpost 2.7.82

package/src/server/cron.js

@@ -0,0 +1,35 @@
+import cron from 'node-cron';
+import { loggerFactory } from './logger.js';
+
+const logger = loggerFactory(import.meta);
+
+const CronManagement = {
+  data: {},
+  init: function () {
+    // verify tokens
+    // https://github.com/settings/tokens
+    for (const cronKey of Object.keys(this.data)) {
+      if (this.data[cronKey].valid) {
+        this.data[cronKey].task.start();
+        logger.info(`Cron task "${this.data[cronKey].name}" started`);
+      } else {
+        logger.error(
+          `Invalid cron expression "${this.data[cronKey].expression}" for task "${this.data[cronKey].name}"`,
+        );
+      }
+    }
+  },
+  add: function (name = 'task', expression = '* * * * *', callback = async () => null) {
+    const args = { name, expression, valid: cron.validate(expression) };
+    this.data[name] = {
+      ...args,
+      task: cron.schedule(expression, callback, {
+        scheduled: false,
+        timezone: process.env.TIME_ZONE || 'America/New_York',
+        name,
+      }),
+    };
+  },
+};
+
+export { CronManagement };

package/src/server/dns.js

@@ -0,0 +1,89 @@
+import axios from 'axios';
+import dotenv from 'dotenv';
+import fs from 'fs';
+import cron from 'node-cron';
+
+import { ip } from './network.js';
+import { loggerFactory } from './logger.js';
+import { isIPv4 } from 'is-ip';
+
+dotenv.config();
+
+const logger = loggerFactory(import.meta);
+
+const Dns = {
+  ip: null,
+  ipDaemon: null,
+  callback: () => null,
+  InitIpDaemon: async function () {
+    // WAN | NAT-VPS | LAN
+    // enabled DMZ Host to proxy IP 80-443 (79-444) sometimes router block first port
+    // LAN server or device's local servers port ->  3000-3100 (2999-3101)
+    // DNS Records: [ANAME](Address Dynamic) -> [A](ipv4) host | [AAAA](ipv6) host -> [ip]
+    // DHCP (Dynamic Host Configuration Protocol) LAN reserver IP -> MAC ID
+    // Forward the router's TCP/UDP ports to the LAN device's IP address
+
+    const privateCronConfPath = `./engine-private/conf/${process.argv[2]}/conf.cron.json`;
+
+    const confCronPath = fs.existsSync(privateCronConfPath) ? privateCronConfPath : './conf/conf.cron.json';
+
+    let confCronData = JSON.parse(fs.readFileSync(confCronPath, 'utf8'));
+    if (confCronData.ipDaemon.disabled) return;
+    Dns.ip = confCronData.ipDaemon.ip;
+    logger.info(`Current ip`, Dns.ip);
+    if (Dns.ipDaemon) clearInterval(Dns.ipDaemon);
+    const callback = async () => {
+      let testIp;
+      try {
+        testIp = await ip.public.ipv4();
+      } catch (error) {
+        logger.error(error, { testIp, stack: error.stack });
+      }
+      if (testIp && typeof testIp === 'string' && isIPv4(testIp) && Dns.ip !== testIp) {
+        logger.info(`New ip`, testIp);
+        Dns.ip = testIp;
+        confCronData.ipDaemon.ip = Dns.ip;
+        fs.writeFileSync(confCronPath, JSON.stringify(confCronData, null, 4), 'utf8');
+        for (const recordType of Object.keys(confCronData.records)) {
+          switch (recordType) {
+            case 'A':
+              for (const dnsProvider of confCronData.records[recordType]) {
+                if (typeof Dns.services.updateIp[dnsProvider.dns] === 'function')
+                  await Dns.services.updateIp[dnsProvider.dns](dnsProvider);
+              }
+              break;
+
+            default:
+              break;
+          }
+        }
+      }
+    };
+    this.callback = callback;
+    return callback;
+  },
+  services: {
+    updateIp: {
+      dondominio: (options) => {
+        const { user, api_key, host, dns } = options;
+        const url = `https://dondns.dondominio.com/json/?user=${user}&password=${api_key}&host=${host}&ip=${Dns.ip}`;
+        logger.info(`${dns} update ip url`, url);
+        if (process.env.NODE_ENV !== 'production') return false;
+        return new Promise((resolve) => {
+          axios
+            .get(url)
+            .then((response) => {
+              logger.info(`${dns} update ip success`, response.data);
+              return resolve(true);
+            })
+            .catch((error) => {
+              logger.error(error, `${dns} update ip error`);
+              return resolve(false);
+            });
+        });
+      },
+    },
+  },
+};
+
+export { Dns };

package/src/server/downloader.js

@@ -0,0 +1,42 @@
+import axios from 'axios';
+import fs from 'fs';
+import { loggerFactory } from './logger.js';
+import dotenv from 'dotenv';
+import https from 'https';
+
+dotenv.config();
+
+const httpsAgent = new https.Agent({
+  rejectUnauthorized: false,
+});
+
+axios.defaults.httpsAgent = httpsAgent;
+
+const logger = loggerFactory(import.meta);
+
+const Downloader = (url, fullPath, options = { method: 'get', responseType: 'stream' }) =>
+  new Promise((resolve, reject) =>
+    axios({
+      url,
+      ...options,
+    })
+      .then((response) => {
+        // Create a write stream to save the file to the specified path
+        const writer = fs.createWriteStream(fullPath);
+        response.data.pipe(writer);
+        writer.on('finish', () => {
+          logger.info('Download complete. File saved at', fullPath);
+          return resolve(fullPath);
+        });
+        writer.on('error', (error) => {
+          logger.error(error, 'Error downloading the file');
+          return reject(error);
+        });
+      })
+      .catch((error) => {
+        logger.error(error, 'Error in the request');
+        return reject(error);
+      }),
+  );
+
+export { Downloader };

package/src/server/logger.js

@@ -0,0 +1,190 @@
+/**
+ * Module for managing logger control and configuration
+ * @module src/server/logger.js
+ * @namespace Logger
+ */
+
+'use strict';
+
+import dotenv from 'dotenv';
+import winston from 'winston';
+import morgan from 'morgan';
+import colorize from 'json-colorizer';
+import colors from 'colors';
+import v8 from 'v8';
+import isAdmin from 'is-admin';
+import { clearTerminalStringColor, formatBytes } from '../client/components/core/CommonJs.js';
+
+colors.enable();
+dotenv.config();
+
+// Define your severity levels.
+// With them, You can create log files,
+// see or hide levels based on the running ENV.
+const levels = {
+  error: 0,
+  warn: 1,
+  info: 2,
+  http: 3,
+  debug: 4,
+};
+
+// This method set the current severity based on
+// the current NODE_ENV: show all the log levels
+// if the server was run in development mode; otherwise,
+// if it was run in production, show only warn and error messages.
+const level = () => 'info'; // (process.env.NODE_ENV || 'development' ? 'debug' : 'warn');
+
+// Define different colors for each level.
+// Colors make the log message more visible,
+// adding the ability to focus or ignore messages.
+
+// Tell winston that you want to link the colors
+// defined above to the severity levels.
+winston.addColors({
+  error: 'red',
+  warn: 'yellow',
+  info: 'green',
+  http: 'magenta',
+  debug: 'white',
+});
+
+// Chose the aspect of your log customizing the log format.
+const format = (meta) =>
+  winston.format.combine(
+    // winston.format.errors({ stack: true }),
+    // Add the message timestamp with the preferred format
+    winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss:ms' }),
+    // Tell Winston that the logs must be colored
+    winston.format.colorize({ all: true }),
+    // Define the format of the message showing the timestamp, the level and the message
+    winston.format.printf((info) => {
+      const symbols = Object.getOwnPropertySymbols(info);
+      return `${`[${meta}]`.green} ${info.timestamp} ${info.level} ${
+        symbols[1]
+          ? `${clearTerminalStringColor(info.message)}: ${colorize(JSON.stringify(info[symbols[1]][0], null, 4), {
+              colors: {
+                STRING_KEY: 'green',
+                STRING_LITERAL: 'magenta.bold',
+                NUMBER_LITERAL: '#FF0000',
+              },
+            })}`
+          : info.message
+      }`;
+    }),
+  );
+
+/**
+ * Logs information about the current process environment to the console.
+ *
+ * This function is used to log details about
+ * the execution context, such as command-line arguments,
+ * environment variables, the process's administrative privileges,
+ * and the maximum available heap space size.
+ *
+ * @param {winston.Logger} logger - A pre-configured Winston logger object.
+ * @memberof Logger
+ */
+const setUpInfo = async (logger = new winston.Logger()) => {
+  logger.info('argv', process.argv);
+  logger.info('platform', process.platform);
+  logger.info('env', process.env.NODE_ENV);
+  logger.info('admin', await isAdmin());
+  logger.info('--max-old-space-size', {
+    total_available_size: formatBytes(v8.getHeapStatistics().total_available_size),
+  });
+};
+
+/**
+ * The function `loggerFactory` creates a logger instance with specified transports for printing out
+ * messages.
+ * @param meta - The `meta` parameter in the `loggerFactory` function is used to extract the last part
+ * of a URL and use it to create log files in a specific directory.
+ * @returns {winston.Logger} The `loggerFactory` function returns a logger instance created using Winston logger
+ * library. The logger instance is configured with various transports for printing out messages to
+ * different destinations such as the terminal, error.log file, and all.log file. The logger instance
+ * also has a method `setUpInfo` attached to it for setting up additional information.
+ * @memberof Logger
+ */
+const loggerFactory = (meta = { url: '' }) => {
+  meta = meta.url.split('/').pop();
+  // Define which transports the logger must use to print out messages.
+  // In this example, we are using three different transports
+  const transports = [
+    // Allow the use the terminal to print the messages
+    new winston.transports.Console(),
+    // Allow to print all the error level messages inside the error.log file
+    new winston.transports.File({
+      filename: `logs/${meta}/error.log`,
+      level: 'error',
+    }),
+    // Allow to print all the error message inside the all.log file
+    // (also the error log that are also printed inside the error.log(
+    new winston.transports.File({ filename: `logs/${meta}/all.log` }),
+  ];
+
+  // Create the logger instance that has to be exported
+  // and used to log messages.
+  const logger = winston.createLogger({
+    defaultMeta: meta,
+    level: level(),
+    levels,
+    format: format(meta),
+    transports,
+    // exceptionHandlers: [new winston.transports.File({ filename: 'exceptions.log' })],
+    // rejectionHandlers: [new winston.transports.File({ filename: 'rejections.log' })],
+    // exitOnError: false,
+  });
+  logger.setUpInfo = async () => {
+    await setUpInfo(logger);
+  };
+  return logger;
+};
+
+/**
+ * The `loggerMiddleware` function creates a middleware for logging HTTP requests using Morgan with
+ * custom message format and options.
+ * @param meta - The `meta` parameter in the `loggerMiddleware` function is an object that contains
+ * information about the request URL. It has a default value of an empty object `{ url: '' }`. This
+ * object is used to provide additional metadata for logging purposes.
+ * @returns {Handler<any, any>} The `loggerMiddleware` function returns a middleware function that uses the Morgan library
+ * to log HTTP request information. The middleware function formats the log message using predefined
+ * tokens provided by Morgan and custom tokens like `:host` to include specific request details. The
+ * log message format includes information such as remote address, HTTP method, host, URL, status code,
+ * content length, and response time in milliseconds. The middleware
+ * @memberof Logger
+ */
+const loggerMiddleware = (meta = { url: '' }) => {
+  const stream = {
+    // Use the http severity
+    write: (message) => loggerFactory(meta).http(message),
+  };
+
+  const skip = (req, res) => process.env.NODE_ENV === 'production';
+
+  morgan.token('host', function (req, res) {
+    return req.headers['host'];
+  });
+
+  return morgan(
+    // Define message format string (this is the default one).
+    // The message format is made from tokens, and each token is
+    // defined inside the Morgan library.
+    // You can create your custom token to show what do you want from a request.
+    `:remote-addr :method :host:url :status :res[content-length] - :response-time ms`,
+    // Options: in this case, I overwrote the stream and the skip logic.
+    // See the methods above.
+    { stream, skip },
+  );
+};
+
+const underpostASCI = () => `
+▗▖ ▗▖▗▖  ▗▖▗▄▄▄  ▗▄▄▄▖▗▄▄▖ ▄▄▄▄   ▄▄▄   ▄▄▄  ■  
+▐▌ ▐▌▐▛▚▖▐▌▐▌  █ ▐▌   ▐▌ ▐▌█   █ █   █ ▀▄▄▗▄▟▙▄▖
+▐▌ ▐▌▐▌ ▝▜▌▐▌  █ ▐▛▀▀▘▐▛▀▚▖█▄▄▄▀ ▀▄▄▄▀ ▄▄▄▀ ▐▌  
+▝▚▄▞▘▐▌  ▐▌▐▙▄▄▀ ▐▙▄▄▖▐▌ ▐▌█                ▐▌  
+                           ▀                ▐▌  
+                                                
+                                                `;
+
+export { loggerFactory, loggerMiddleware, setUpInfo, underpostASCI };

package/src/server/network.js

@@ -0,0 +1,122 @@
+import detect from 'detect-port';
+import fs from 'fs-extra';
+
+import { publicIp, publicIpv4, publicIpv6 } from 'public-ip';
+import { killPortProcess } from 'kill-port-process';
+import { loggerFactory } from './logger.js';
+import { orderArrayFromAttrInt } from '../client/components/core/CommonJs.js';
+
+// Network Address Translation Management
+
+// import dotenv from 'dotenv';
+// dotenv.config();
+
+const logger = loggerFactory(import.meta);
+
+const network = {
+  port: {
+    status: async (ports) => {
+      const status = [];
+      for (const port of ports) {
+        status.push({
+          port,
+          open: await new Promise((resolve) =>
+            detect(port)
+              .then((_port) => {
+                if (port == _port)
+                  // `port: ${port} was not occupied`
+                  return resolve(false);
+
+                // `port: ${port} was occupied, try port: ${_port}`
+                return resolve(true);
+              })
+              .catch((error) => resolve(`${error.message}`)),
+          ),
+        });
+      }
+      return status;
+    },
+    kill: async (ports) => await killPortProcess(ports),
+    portClean: async function (port) {
+      const [portStatus] = await this.status([port]);
+      // logger.info('port status', portStatus);
+      if (portStatus.open) await this.kill([port]);
+    },
+  },
+};
+
+const ip = {
+  public: {
+    get: async () => await publicIp(), // => 'fe80::200:f8ff:fe21:67cf'
+    ipv4: async () => await publicIpv4(), // => '46.5.21.123'
+    ipv6: async () => await publicIpv6(), // => 'fe80::200:f8ff:fe21:67cf'
+  },
+};
+
+let ipInstance = '';
+const networkRouter = {};
+
+const logRuntimeRouter = () => {
+  const displayLog = {};
+
+  for (const host of Object.keys(networkRouter))
+    for (const path of Object.keys(networkRouter[host]))
+      displayLog[networkRouter[host][path].publicHost] = networkRouter[host][path].local;
+
+  logger.info('Runtime network', displayLog);
+};
+
+const saveRuntimeRouter = () =>
+  fs.writeFileSync(
+    `./tmp/runtime-router.${process.argv[3] ? process.argv[3] : 'default'}.json`,
+    JSON.stringify(networkRouter, null, 4),
+    'utf-8',
+  );
+
+const listenServerFactory = (logic = async () => {}) => {
+  return {
+    listen: async (...args) => (logic ? await logic(...args) : undefined, args[1]()),
+  };
+};
+
+const listenPortController = async (server, port, metadata) =>
+  new Promise((resolve) => {
+    try {
+      if (!server) server = listenServerFactory();
+
+      const { host, path, client, runtime, meta } = metadata;
+      const error = [];
+      if (port === undefined) error.push(`port`);
+      if (host === undefined) error.push(`host`);
+      if (path === undefined) error.push(`path`);
+      if (client === undefined) error.push(`client`);
+      if (runtime === undefined) error.push(`runtime`);
+      if (meta === undefined) error.push(`meta`);
+      if (error.length > 0) throw new Error('Listen port controller requires values: ' + error.join(', '));
+
+      server.listen(port, () => {
+        if (!networkRouter[host]) networkRouter[host] = {};
+        networkRouter[host][path] = {
+          meta,
+          client,
+          runtime,
+          port,
+          public: `http://${ipInstance}:${port}${path}`,
+          publicHost:
+            port === 80
+              ? `http://${host}${path}`
+              : port === 443
+              ? `https://${host}${path}`
+              : `http://${host}:${port}${path}`,
+          local: `http://localhost:${port}${path}`,
+        };
+
+        return resolve(true);
+      });
+    } catch (error) {
+      logger.error(error, { metadata, port, stack: error.stack });
+      resolve(false);
+    }
+  });
+
+export { ip, network, listenPortController, networkRouter, saveRuntimeRouter, logRuntimeRouter, listenServerFactory };

package/src/server/peer.js

@@ -0,0 +1,33 @@
+import { PeerServer } from 'peer';
+import dotenv from 'dotenv';
+import { loggerFactory } from './logger.js';
+import fs from 'fs-extra';
+import { listenServerFactory } from './network.js';
+
+dotenv.config();
+
+const logger = loggerFactory(import.meta);
+
+// https://github.com/peers/peerjs
+// https://github.com/peers/peerjs-server
+
+const createPeerServer = async ({ port, devPort, origins, host, path }) => {
+  if (process.env.NODE_ENV === 'development' && devPort) origins.push(`http://localhost:${devPort}`);
+  /** @type {import('peer').IConfig} */
+  const options = {
+    port,
+    path: `${path === '/' ? '' : path}/peer`,
+    corsOptions: {
+      origin: origins,
+    },
+    proxied: true,
+    // key: fs.readFileSync(''),
+    // cert: fs.readFileSync(''),
+    // ca: fs.readFileSync(''),
+  };
+  const peerServer = listenServerFactory(() => PeerServer(options));
+
+  return { options, peerServer, meta: import.meta };
+};
+
+export { createPeerServer };

package/src/server/process.js

@@ -0,0 +1,66 @@
+// https://nodejs.org/api/process
+
+import shell from 'shelljs';
+import dotenv from 'dotenv';
+import fs from 'fs-extra';
+
+import { loggerFactory } from './logger.js';
+
+dotenv.config();
+
+const logger = loggerFactory(import.meta);
+
+// process.exit();
+
+const getRootDirectory = () => process.cwd().replace(/\\/g, '/');
+
+const ProcessController = {
+  SIG: [
+    'SIGPIPE',
+    'SIGHUP',
+    'SIGTERM',
+    'SIGINT',
+    'SIGBREAK',
+    'SIGWINCH',
+    // 'SIGKILL',
+    // 'SIGSTOP',
+    'SIGBUS',
+    'SIGFPE',
+    'SIGSEGV',
+    'SIGILL',
+  ],
+  onSigListen: function () {
+    return this.SIG.map((sig) =>
+      process.on(sig, (...args) => {
+        this.logger.info(`process on ${sig}`, args);
+        switch (sig) {
+          case 'SIGINT':
+            return process.exit();
+
+          default:
+            break;
+        }
+      }),
+    );
+  },
+  init: function (logger) {
+    this.logger = logger;
+    process.on('exit', (...args) => {
+      this.logger.info(`process on exit`, args);
+    });
+    this.onSigListen();
+    if (fs.existsSync(`./tmp/await-deploy`)) fs.remove(`./tmp/await-deploy`);
+  },
+};
+
+const shellExec = (cmd, options = { silent: false, async: false, stdout: false, disableLog: false }) => {
+  if (!options.disableLog) logger.info(`cmd`, cmd);
+  return options.stdout ? shell.exec(cmd, options).stdout : shell.exec(cmd, options);
+};
+
+const shellCd = (cd, options = { disableLog: false }) => {
+  if (!options.disableLog) logger.info(`cd`, cd);
+  return shell.cd(cd);
+};
+
+export { ProcessController, getRootDirectory, shellExec, shellCd };

package/src/server/prompt-optimizer.js

@@ -0,0 +1,28 @@
+// https://github.com/xenova/transformers.js/blob/f43d3dd348fd7b293008802590bb3a1afa218dc7/src/models.js#L10
+
+import { AutoModelForSeq2SeqLM, AutoTokenizer } from '@xenova/transformers';
+import { loggerFactory } from './logger.js';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+const logger = loggerFactory(import.meta);
+
+const tokenizer = await AutoTokenizer.from_pretrained('Xenova/t5-small');
+
+const model = await AutoModelForSeq2SeqLM.from_pretrained('Xenova/t5-small');
+
+const prompt = 'translate English to German: I love transformers!';
+
+logger.info('input', { prompt });
+
+const tokenizerData = await tokenizer(prompt);
+
+const { input_ids } = tokenizerData;
+
+const outputs = await model.generate(input_ids);
+
+for (const output of outputs) {
+  const decoded = tokenizer.decode(output, { skip_special_tokens: true });
+  logger.info('decoded', { decoded });
+}