@budibase/backend-core 2.8.11 → 2.8.12-alpha.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.8.11",
+  "version": "2.8.12-alpha.0",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -22,7 +22,7 @@
   "dependencies": {
     "@budibase/nano": "10.1.2",
     "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/types": "2.8.11",
+    "@budibase/types": "2.8.12-alpha.0",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",
@@ -51,6 +51,7 @@
     "pouchdb": "7.3.0",
     "pouchdb-find": "7.2.2",
     "redlock": "4.2.0",
+    "rotating-file-stream": "3.1.0",
     "sanitize-s3-objectkey": "0.0.1",
     "semver": "7.3.7",
     "tar-fs": "2.1.1",
@@ -101,5 +102,5 @@
       }
     }
   },
-  "gitHead": "8d0194b6f7bb76a41c244450f0c30f7548c6a0d5"
+  "gitHead": "2aeebd290d78e13ab253daa2a28a5101e443fbc8"
 }

package/src/environment.ts

@@ -47,7 +47,10 @@ function httpLogging() {
   return process.env.HTTP_LOGGING
 }
 
-function findVersion() {
+function getPackageJsonFields(): {
+  VERSION: string
+  SERVICE_NAME: string
+} {
   function findFileInAncestors(
     fileName: string,
     currentDir: string
@@ -69,10 +72,14 @@ function findVersion() {
   try {
     const packageJsonFile = findFileInAncestors("package.json", process.cwd())
     const content = readFileSync(packageJsonFile!, "utf-8")
-    return JSON.parse(content).version
+    const parsedContent = JSON.parse(content)
+    return {
+      VERSION: parsedContent.version,
+      SERVICE_NAME: parsedContent.name,
+    }
   } catch {
     // throwing an error here is confusing/causes backend-core to be hard to import
-    return undefined
+    return { VERSION: "", SERVICE_NAME: "" }
   }
 }
 
@@ -154,13 +161,14 @@ const environment = {
   ENABLE_SSO_MAINTENANCE_MODE: selfHosted
     ? process.env.ENABLE_SSO_MAINTENANCE_MODE
     : false,
-  VERSION: findVersion(),
+  ...getPackageJsonFields(),
   DISABLE_PINO_LOGGER: process.env.DISABLE_PINO_LOGGER,
   _set(key: any, value: any) {
     process.env[key] = value
     // @ts-ignore
     environment[key] = value
   },
+  ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M",
 }
 
 // clean up any environment variable edge cases

package/src/logging/index.ts

@@ -1,6 +1,7 @@
 export * as correlation from "./correlation/correlation"
 export { logger } from "./pino/logger"
 export * from "./alerts"
+export * as system from "./system"
 
 // turn off or on context logging i.e. tenantId, appId etc
 export let LOG_CONTEXT = true

package/src/logging/pino/logger.ts

@@ -1,10 +1,15 @@
-import env from "../../environment"
 import pino, { LoggerOptions } from "pino"
+import pinoPretty from "pino-pretty"
+
+import { IdentityType } from "@budibase/types"
+
+import env from "../../environment"
 import * as context from "../../context"
 import * as correlation from "../correlation"
-import { IdentityType } from "@budibase/types"
 import { LOG_CONTEXT } from "../index"
 
+import { localFileDestination } from "../system"
+
 // LOGGER
 
 let pinoInstance: pino.Logger | undefined
@@ -16,22 +21,27 @@ if (!env.DISABLE_PINO_LOGGER) {
         return { level: label.toUpperCase() }
       },
       bindings: () => {
-        return {}
+        return {
+          service: env.SERVICE_NAME,
+        }
       },
     },
     timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
   }
 
+  const destinations: pino.DestinationStream[] = []
+
   if (env.isDev()) {
-    pinoOptions.transport = {
-      target: "pino-pretty",
-      options: {
-        singleLine: true,
-      },
-    }
+    destinations.push(pinoPretty({ singleLine: true }))
+  }
+
+  if (env.SELF_HOSTED) {
+    destinations.push(localFileDestination())
   }
 
-  pinoInstance = pino(pinoOptions)
+  pinoInstance = destinations.length
+    ? pino(pinoOptions, pino.multistream(destinations))
+    : pino(pinoOptions)
 
   // CONSOLE OVERRIDES
 

package/src/logging/system.ts

@@ -0,0 +1,81 @@
+import fs from "fs"
+import path from "path"
+import * as rfs from "rotating-file-stream"
+
+import env from "../environment"
+import { budibaseTempDir } from "../objectStore"
+
+const logsFileName = `budibase.log`
+const budibaseLogsHistoryFileName = "budibase-logs-history.txt"
+
+const logsPath = path.join(budibaseTempDir(), "systemlogs")
+
+function getFullPath(fileName: string) {
+  return path.join(logsPath, fileName)
+}
+
+export function getSingleFileMaxSizeInfo(totalMaxSize: string) {
+  const regex = /(\d+)([A-Za-z])/
+  const match = totalMaxSize?.match(regex)
+  if (!match) {
+    console.warn(`totalMaxSize does not have a valid value`, {
+      totalMaxSize,
+    })
+    return undefined
+  }
+
+  const size = +match[1]
+  const unit = match[2]
+  if (size === 1) {
+    switch (unit) {
+      case "B":
+        return { size: `${size}B`, totalHistoryFiles: 1 }
+      case "K":
+        return { size: `${(size * 1000) / 2}B`, totalHistoryFiles: 1 }
+      case "M":
+        return { size: `${(size * 1000) / 2}K`, totalHistoryFiles: 1 }
+      case "G":
+        return { size: `${(size * 1000) / 2}M`, totalHistoryFiles: 1 }
+      default:
+        return undefined
+    }
+  }
+
+  if (size % 2 === 0) {
+    return { size: `${size / 2}${unit}`, totalHistoryFiles: 1 }
+  }
+
+  return { size: `1${unit}`, totalHistoryFiles: size - 1 }
+}
+
+export function localFileDestination() {
+  const fileInfo = getSingleFileMaxSizeInfo(env.ROLLING_LOG_MAX_SIZE)
+  const outFile = rfs.createStream(logsFileName, {
+    // As we have a rolling size, we want to half the max size
+    size: fileInfo?.size,
+    path: logsPath,
+    maxFiles: fileInfo?.totalHistoryFiles || 1,
+    immutable: true,
+    history: budibaseLogsHistoryFileName,
+    initialRotation: false,
+  })
+
+  return outFile
+}
+
+export function getLogReadStream() {
+  const streams = []
+  const historyFile = getFullPath(budibaseLogsHistoryFileName)
+  if (fs.existsSync(historyFile)) {
+    const fileContent = fs.readFileSync(historyFile, "utf-8")
+    const historyFiles = fileContent.split("\n")
+    for (const historyFile of historyFiles.filter(x => x)) {
+      streams.push(fs.readFileSync(historyFile))
+    }
+  }
+
+  streams.push(fs.readFileSync(getFullPath(logsFileName)))
+
+  const combinedContent = Buffer.concat(streams)
+  return combinedContent
+}

package/src/logging/tests/system.spec.ts

@@ -0,0 +1,61 @@
+import { getSingleFileMaxSizeInfo } from "../system"
+
+describe("system", () => {
+  describe("getSingleFileMaxSizeInfo", () => {
+    it.each([
+      ["100B", "50B"],
+      ["200K", "100K"],
+      ["20M", "10M"],
+      ["4G", "2G"],
+    ])(
+      "Halving even number (%s) returns halved size and 1 history file (%s)",
+      (totalValue, expectedMaxSize) => {
+        const result = getSingleFileMaxSizeInfo(totalValue)
+        expect(result).toEqual({
+          size: expectedMaxSize,
+          totalHistoryFiles: 1,
+        })
+      }
+    )
+
+    it.each([
+      ["5B", "1B", 4],
+      ["17K", "1K", 16],
+      ["21M", "1M", 20],
+      ["3G", "1G", 2],
+    ])(
+      "Halving an odd number (%s) returns as many files as size (-1) (%s)",
+      (totalValue, expectedMaxSize, totalHistoryFiles) => {
+        const result = getSingleFileMaxSizeInfo(totalValue)
+        expect(result).toEqual({
+          size: expectedMaxSize,
+          totalHistoryFiles,
+        })
+      }
+    )
+
+    it.each([
+      ["1B", "1B"],
+      ["1K", "500B"],
+      ["1M", "500K"],
+      ["1G", "500M"],
+    ])(
+      "Halving '%s' returns halved unit (%s)",
+      (totalValue, expectedMaxSize) => {
+        const result = getSingleFileMaxSizeInfo(totalValue)
+        expect(result).toEqual({
+          size: expectedMaxSize,
+          totalHistoryFiles: 1,
+        })
+      }
+    )
+
+    it.each([[undefined], [""], ["50"], ["wrongvalue"]])(
+      "Halving wrongly formatted value ('%s') returns undefined",
+      totalValue => {
+        const result = getSingleFileMaxSizeInfo(totalValue!)
+        expect(result).toBeUndefined()
+      }
+    )
+  })
+})