@rpcbase/server 0.70.0 → 0.73.0

package/bin.js

@@ -6,7 +6,7 @@ require("dotenv").config({path: path.join(__dirname, "./.env")})
 const yargs = require("yargs/yargs")
 const {hideBin} = require("yargs/helpers")
 
-const start_server = require("./cli/start_server")
+const start_server_infrastructure = require("./cli/start_server_infrastructure")
 const build_server = require("./cli/build_server")
 const run_agent = require("./cli/run_agent")
 
@@ -14,9 +14,9 @@ let is_command = false
 
 
 const args = yargs(hideBin(process.argv))
-  .command("start", "run server/infrastructure", () => {}, (args) => {
+  .command("start", "runs server/infrastructure", () => {}, (args) => {
     is_command = true
-    start_server(args)
+    start_server_infrastructure(args)
   })
   .command("agent", "run the agent", () => {}, (args) => {
     is_command = true

package/cli/run_native.js

@@ -3,13 +3,17 @@ const {spawn} = require("child_process")
 const path = require("path")
 const fs = require("fs")
 
+const colors = require("picocolors")
+const validator = require("validator")
 const dotenv = require("dotenv")
 const mkdirp = require("mkdirp")
 
-// TODO: verify mongod and redis-sever executables are present
+// TODO: verify mongod and redis-sever and elasticsearch executables are present
 // and have the right version
 
-// "redis-server", "/usr/local/etc/redis/redis.conf", "--port", $WORKER_QUEUE_PORT
+const NO_PRINT_LIST = [
+  // "database",
+]
 
 // load env
 const get_env = () => {
@@ -19,54 +23,102 @@ const get_env = () => {
   return env
 }
 
-const init_processes = () => {
+const init_processes = (args) => {
   const env = get_env()
 
   const infrastructure_conf_dir = path.join(__dirname, "../infrastructure")
 
-  // create data dirs
+  // db storage
   const db_path = path.join(process.cwd(), "../infrastructure/data/database/db/")
-  const db_logs_path = path.join(process.cwd(), "../infrastructure/data/database/log/")
 
+  // db log file
+  const db_logs_path = path.join(process.cwd(), "../infrastructure/data/database/log/logs.txt")
+
+  // session-store + worker-queue
   const session_store_dir = path.join(process.cwd(), "../infrastructure/data/session-store/data")
+  const session_store_logs_path = path.join(process.cwd(), "../infrastructure/data/session-store/log/logs.txt")
+  const worker_queue_dir = path.join(process.cwd(), "../infrastructure/data/worker-queue/data")
+  const worker_queue_logs_path = path.join(process.cwd(), "../infrastructure/data/worker-queue/log/logs.txt")
 
   // TODO: add a redis cache
   const processes = [
     // web server
     // TODO: do not use yarn dev server command
+    // use command: rb-server dev
     {
       name: "server",
       dirs: [],
+      // yarn dev
       cmd: ["yarn", ["dev"]]
     },
     // mongodb database
     {
       name: "database",
       dirs: [
-        db_path, db_logs_path
+        db_path,
+        path.dirname(db_logs_path),
       ],
       cmd: ["mongod", [
         "--quiet",
         "--dbpath", db_path,
         "--port", env.DATABASE_PORT,
         // "--bind_ip_all",
-        // "--logpath", "/var/log/logs.txt",
+        // if in verbose mode, print to stdout, else write to file
+        ...(args.verbose ? [] : ["--logpath", db_logs_path]),
         "--replSet", "rs0"
       ]]
     },
+    // session-store
     {
       name: "session-store",
       dirs: [
         session_store_dir,
+        path.dirname(session_store_logs_path),
       ],
       cmd: ["redis-server", [
         path.join(infrastructure_conf_dir, "./redis/redis.conf"),
         "--dir", session_store_dir,
         "--port", env.SESSION_STORE_PORT,
+        ...(args.verbose ? [] : ["--logfile", session_store_logs_path]),
+      ]]
+    },
+    // worker-queue
+    {
+      name: "worker-queue",
+      dirs: [
+        worker_queue_dir,
+        path.dirname(worker_queue_logs_path),
+      ],
+      cmd: ["redis-server", [
+        path.join(infrastructure_conf_dir, "./redis/redis.conf"),
+        "--dir", worker_queue_dir,
+        "--port", env.WORKER_QUEUE_PORT,
+        ...(args.verbose ? [] : ["--logfile", worker_queue_logs_path]),
       ]]
     },
   ]
 
+  // add search-index elasticsearch if port is defined
+  if (env.SEARCH_INDEX_PORT && validator.isPort(env.SEARCH_INDEX_PORT)) {
+    const data_dir = path.join(process.cwd(), "../infrastructure/data/search-index/data")
+    const logs_dir = path.join(process.cwd(), "../infrastructure/data/search-index/logs")
+    processes.push({
+      name: "search-index",
+      dirs: [
+        data_dir,
+        logs_dir,
+      ],
+      cmd: ["elasticsearch", [
+        "-s",
+        "-E", `path.logs=${logs_dir}`,
+        "-E", `http.port=${env.SEARCH_INDEX_PORT}`,
+        "-E", `path.data=${data_dir}`,
+        "-E", "discovery.type=single-node",
+        "-E", "ingest.geoip.downloader.enabled=false",
+      ]]
+    })
+  }
+
   // create missing directories
   processes.forEach((item) => {
     item.dirs.forEach((dir) => {
@@ -81,22 +133,32 @@ const init_processes = () => {
 
 // TODO: run each native process from list
 // TODO: handle process close
-const run_native = (infrastructure_dir, proj_prefix) => {
-  console.log("running in NATIVE mode")
+const run_native = (infrastructure_dir, proj_prefix, args) => {
+  console.log(`running in ${colors.bold(colors.magenta("NATIVE"))} mode`)
 
-  const processes = init_processes()
+  const processes = init_processes(args)
 
   const run_process = (item) => {
-    // console.log("start process", item)
+    // console.log("run_native:run_process:item", item)
     const ps = spawn(item.cmd[0], item.cmd[1], {env: {...process.env, CONTAINER_MODE: "native"}})
 
-    ps.stdout.on("data", (data) => {
-      process.stdout.write(`${item.name}: ${data.toString()}`)
-    })
+    // TODO: add flag to force print
+    if (!NO_PRINT_LIST.includes(item.name)) {
 
-    ps.stderr.on("data", (data) => {
-      process.stderr.write(`${item.name}: ${data.toString()}`)
-    })
+      let prefix = ""
+      // avoid printing [server] server:
+      if (!args.verbose && item.name !== "server") {
+        prefix = `${colors.green(item.name)}: `
+      }
+
+      ps.stdout.on("data", (data) => {
+        process.stdout.write(`${prefix}${data.toString()}`)
+      })
+
+      ps.stderr.on("data", (data) => {
+        process.stderr.write(`${prefix}${data.toString()}`)
+      })
+    }
   }
 
   const spawned = processes.map(run_process)

package/cli/{start_server.js → start_server_infrastructure.js}

@@ -19,9 +19,9 @@ const start_server = async(args) => {
   const proj_prefix = path.basename(proj_parent_dir)
 
   if (runtime === "docker") {
-    run_docker(infrastructure_dir, proj_prefix)
+    run_docker(infrastructure_dir, proj_prefix, args)
   } else if (runtime === "native") {
-    run_native(infrastructure_dir, proj_prefix)
+    run_native(infrastructure_dir, proj_prefix, args)
   } else {
     throw new Error("unknown runtime")
   }

package/database.js

@@ -24,6 +24,7 @@ module.exports = async() => {
   })
 
   mongoose.connection.on("disconnected", (arg) => {
+    // TODO: add tracing
     console.log("Mongoose disconnected", arg)
   })
 

package/index.js

@@ -4,8 +4,6 @@ const database = require("./database")
 const express = require("./express")
 const rpc_router = require("./rpc/rpc_router")
 
-
-
 module.exports = {
   client_router,
   database,

package/infrastructure/elasticsearch/elasticsearch.yml

@@ -0,0 +1,88 @@
+# ======================== Elasticsearch Configuration =========================
+#
+# NOTE: Elasticsearch comes with reasonable defaults for most settings.
+#       Before you set out to tweak and tune the configuration, make sure you
+#       understand what are you trying to accomplish and the consequences.
+#
+# The primary way of configuring a node is via this file. This template lists
+# the most important settings you may want to configure for a production cluster.
+#
+# Please consult the documentation for further information on configuration options:
+# https://www.elastic.co/guide/en/elasticsearch/reference/index.html
+#
+# ---------------------------------- Cluster -----------------------------------
+#
+# Use a descriptive name for your cluster:
+#
+#cluster.name: my-application
+#
+# ------------------------------------ Node ------------------------------------
+#
+# Use a descriptive name for the node:
+#
+#node.name: node-1
+#
+# Add custom attributes to the node:
+#
+#node.attr.rack: r1
+#
+# ----------------------------------- Paths ------------------------------------
+#
+# Path to directory where to store the data (separate multiple locations by comma):
+#
+@path.data@
+#
+# Path to log files:
+#
+@path.logs@
+#
+# ----------------------------------- Memory -----------------------------------
+#
+# Lock the memory on startup:
+#
+#bootstrap.memory_lock: true
+#
+# Make sure that the heap size is set to about half the memory available
+# on the system and that the owner of the process is allowed to use this
+# limit.
+#
+# Elasticsearch performs poorly when the system is swapping the memory.
+#
+# ---------------------------------- Network -----------------------------------
+#
+# By default Elasticsearch is only accessible on localhost. Set a different
+# address here to expose this node on the network:
+#
+#network.host: 192.168.0.1
+#
+# By default Elasticsearch listens for HTTP traffic on the first free port it
+# finds starting at 9200. Set a specific HTTP port here:
+#
+#http.port: 9200
+#
+# For more information, consult the network module documentation.
+#
+# --------------------------------- Discovery ----------------------------------
+#
+# Pass an initial list of hosts to perform discovery when this node is started:
+# The default list of hosts is ["127.0.0.1", "[::1]"]
+#
+#discovery.seed_hosts: ["host1", "host2"]
+#
+# Bootstrap the cluster using an initial set of master-eligible nodes:
+#
+#cluster.initial_master_nodes: ["node-1", "node-2"]
+#
+# For more information, consult the discovery and cluster formation module documentation.
+#
+# --------------------------------- Readiness ----------------------------------
+#
+# Enable an unauthenticated TCP readiness endpoint on localhost
+#
+#readiness.port: 9399
+#
+# ---------------------------------- Various -----------------------------------
+#
+# Allow wildcard deletion of indices:
+#
+#action.destructive_requires_name: false

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/server",
-  "version": "0.70.0",
+  "version": "0.73.0",
   "license": "SSPL-1.0",
   "main": "./index.js",
   "bin": {
@@ -10,7 +10,7 @@
     "test": "echo \"Error: no test specified\" && exit 0"
   },
   "dependencies": {
-    "@rpcbase/agent": "0.7.0",
+    "@rpcbase/agent": "0.9.0",
     "body-parser": "1.20.0",
     "connect-redis": "6.1.3",
     "cors": "2.8.5",
@@ -19,7 +19,8 @@
     "express": "4.18.1",
     "express-session": "1.17.3",
     "glob": "8.0.3",
-    "mongoose": "6.3.5",
+    "mongoose": "6.4.0",
+    "picocolors": "1.0.0",
     "postmark": "3.0.11",
     "redis": "4.1.0",
     "validator": "13.7.0",