@budibase/backend-core 2.8.16-alpha.4 → 2.8.18

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.8.16-alpha.4",
+  "version": "2.8.18",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -22,7 +22,7 @@
   "dependencies": {
     "@budibase/nano": "10.1.2",
     "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/types": "2.8.16-alpha.4",
+    "@budibase/types": "2.8.18",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",
@@ -51,7 +51,6 @@
     "pouchdb": "7.3.0",
     "pouchdb-find": "7.2.2",
     "redlock": "4.2.0",
-    "rotating-file-stream": "3.1.0",
     "sanitize-s3-objectkey": "0.0.1",
     "semver": "7.3.7",
     "tar-fs": "2.1.1",
@@ -102,5 +101,5 @@
       }
     }
   },
-  "gitHead": "678ac0869da16991b5e54a87c4cdc404e25fd03c"
+  "gitHead": "38d2a24ac7392c2d8df507bf82fbb53e6da16527"
 }

package/src/auth/auth.ts

@@ -159,7 +159,7 @@ export async function updateUserOAuth(userId: string, oAuthConfig: any) {
 
   try {
     const db = getGlobalDB()
-    const dbUser = await db.get<any>(userId)
+    const dbUser = await db.get(userId)
 
     //Do not overwrite the refresh token if a valid one is not provided.
     if (typeof details.refreshToken !== "string") {

package/src/cache/user.ts

@@ -12,7 +12,7 @@ const EXPIRY_SECONDS = 3600
  */
 async function populateFromDB(userId: string, tenantId: string) {
   const db = tenancy.getTenantDB(tenantId)
-  const user = await db.get<any>(userId)
+  const user = await db.get(userId)
   user.budibaseAccess = true
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(user.email)

package/src/constants/misc.ts

@@ -20,8 +20,6 @@ export enum Header {
   TYPE = "x-budibase-type",
   PREVIEW_ROLE = "x-budibase-role",
   TENANT_ID = "x-budibase-tenant-id",
-  VERIFICATION_CODE = "x-budibase-verification-code",
-  RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code",
   TOKEN = "x-budibase-token",
   CSRF_TOKEN = "x-csrf-token",
   CORRELATION_ID = "x-budibase-correlation-id",

package/src/db/searchIndexes/searchIndexes.ts

@@ -5,7 +5,7 @@ export async function createUserIndex() {
   const db = getGlobalDB()
   let designDoc
   try {
-    designDoc = await db.get<any>("_design/database")
+    designDoc = await db.get("_design/database")
   } catch (err: any) {
     if (err.status === 404) {
       designDoc = { _id: "_design/database" }

package/src/environment.ts

@@ -47,10 +47,7 @@ function httpLogging() {
   return process.env.HTTP_LOGGING
 }
 
-function getPackageJsonFields(): {
-  VERSION: string
-  SERVICE_NAME: string
-} {
+function findVersion() {
   function findFileInAncestors(
     fileName: string,
     currentDir: string
@@ -72,14 +69,10 @@ function getPackageJsonFields(): {
   try {
     const packageJsonFile = findFileInAncestors("package.json", process.cwd())
     const content = readFileSync(packageJsonFile!, "utf-8")
-    const parsedContent = JSON.parse(content)
-    return {
-      VERSION: parsedContent.version,
-      SERVICE_NAME: parsedContent.name,
-    }
+    return JSON.parse(content).version
   } catch {
     // throwing an error here is confusing/causes backend-core to be hard to import
-    return { VERSION: "", SERVICE_NAME: "" }
+    return undefined
   }
 }
 
@@ -161,14 +154,13 @@ const environment = {
   ENABLE_SSO_MAINTENANCE_MODE: selfHosted
     ? process.env.ENABLE_SSO_MAINTENANCE_MODE
     : false,
-  ...getPackageJsonFields(),
+  VERSION: findVersion(),
   DISABLE_PINO_LOGGER: process.env.DISABLE_PINO_LOGGER,
   _set(key: any, value: any) {
     process.env[key] = value
     // @ts-ignore
     environment[key] = value
   },
-  ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M",
 }
 
 // clean up any environment variable edge cases

package/src/logging/index.ts

@@ -1,4 +1,6 @@
 export * as correlation from "./correlation/correlation"
 export { logger } from "./pino/logger"
 export * from "./alerts"
-export * as system from "./system"
+
+// turn off or on context logging i.e. tenantId, appId etc
+export let LOG_CONTEXT = true

package/src/logging/pino/logger.ts

@@ -1,12 +1,9 @@
-import pino, { LoggerOptions } from "pino"
-import pinoPretty from "pino-pretty"
-
-import { IdentityType } from "@budibase/types"
 import env from "../../environment"
+import pino, { LoggerOptions } from "pino"
 import * as context from "../../context"
 import * as correlation from "../correlation"
-
-import { localFileDestination } from "../system"
+import { IdentityType } from "@budibase/types"
+import { LOG_CONTEXT } from "../index"
 
 // LOGGER
 
@@ -19,27 +16,22 @@ if (!env.DISABLE_PINO_LOGGER) {
         return { level: label.toUpperCase() }
       },
       bindings: () => {
-        return {
-          service: env.SERVICE_NAME,
-        }
+        return {}
       },
     },
     timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
   }
 
-  const destinations: pino.DestinationStream[] = []
-
   if (env.isDev()) {
-    destinations.push(pinoPretty({ singleLine: true }))
-  }
-
-  if (env.SELF_HOSTED) {
-    destinations.push(localFileDestination())
+    pinoOptions.transport = {
+      target: "pino-pretty",
+      options: {
+        singleLine: true,
+      },
+    }
   }
 
-  pinoInstance = destinations.length
-    ? pino(pinoOptions, pino.multistream(destinations))
-    : pino(pinoOptions)
+  pinoInstance = pino(pinoOptions)
 
   // CONSOLE OVERRIDES
 
@@ -91,13 +83,15 @@ if (!env.DISABLE_PINO_LOGGER) {
 
     let contextObject = {}
 
-    contextObject = {
-      tenantId: getTenantId(),
-      appId: getAppId(),
-      automationId: getAutomationId(),
-      identityId: identity?._id,
-      identityType: identity?.type,
-      correlationId: correlation.getId(),
+    if (LOG_CONTEXT) {
+      contextObject = {
+        tenantId: getTenantId(),
+        appId: getAppId(),
+        automationId: getAutomationId(),
+        identityId: identity?._id,
+        identityType: identity?.type,
+        correlationId: correlation.getId(),
+      }
     }
 
     const mergingObject: any = {

package/src/users.ts

@@ -67,9 +67,9 @@ export const bulkUpdateGlobalUsers = async (users: User[]) => {
 
 export async function getById(id: string, opts?: GetOpts): Promise<User> {
   const db = context.getGlobalDB()
-  let user = await db.get<User>(id)
+  let user = await db.get(id)
   if (opts?.cleanup) {
-    user = removeUserPassword(user) as User
+    user = removeUserPassword(user)
   }
   return user
 }

package/dist/src/logging/system.d.ts

@@ -1,9 +0,0 @@
-/// <reference types="node" />
-/// <reference types="pouchdb-core" />
-import * as rfs from "rotating-file-stream";
-export declare function getSingleFileMaxSizeInfo(totalMaxSize: string): {
-    size: string;
-    totalHistoryFiles: number;
-} | undefined;
-export declare function localFileDestination(): rfs.RotatingFileStream;
-export declare function getLogReadStream(): Buffer;

package/dist/src/logging/system.js

@@ -1,101 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getLogReadStream = exports.localFileDestination = exports.getSingleFileMaxSizeInfo = void 0;
-const fs_1 = __importDefault(require("fs"));
-const path_1 = __importDefault(require("path"));
-const rfs = __importStar(require("rotating-file-stream"));
-const environment_1 = __importDefault(require("../environment"));
-const objectStore_1 = require("../objectStore");
-const logsFileName = `budibase.log`;
-const budibaseLogsHistoryFileName = "budibase-logs-history.txt";
-const logsPath = path_1.default.join((0, objectStore_1.budibaseTempDir)(), "systemlogs");
-function getFullPath(fileName) {
-    return path_1.default.join(logsPath, fileName);
-}
-function getSingleFileMaxSizeInfo(totalMaxSize) {
-    const regex = /(\d+)([A-Za-z])/;
-    const match = totalMaxSize === null || totalMaxSize === void 0 ? void 0 : totalMaxSize.match(regex);
-    if (!match) {
-        console.warn(`totalMaxSize does not have a valid value`, {
-            totalMaxSize,
-        });
-        return undefined;
-    }
-    const size = +match[1];
-    const unit = match[2];
-    if (size === 1) {
-        switch (unit) {
-            case "B":
-                return { size: `${size}B`, totalHistoryFiles: 1 };
-            case "K":
-                return { size: `${(size * 1000) / 2}B`, totalHistoryFiles: 1 };
-            case "M":
-                return { size: `${(size * 1000) / 2}K`, totalHistoryFiles: 1 };
-            case "G":
-                return { size: `${(size * 1000) / 2}M`, totalHistoryFiles: 1 };
-            default:
-                return undefined;
-        }
-    }
-    if (size % 2 === 0) {
-        return { size: `${size / 2}${unit}`, totalHistoryFiles: 1 };
-    }
-    return { size: `1${unit}`, totalHistoryFiles: size - 1 };
-}
-exports.getSingleFileMaxSizeInfo = getSingleFileMaxSizeInfo;
-function localFileDestination() {
-    const fileInfo = getSingleFileMaxSizeInfo(environment_1.default.ROLLING_LOG_MAX_SIZE);
-    const outFile = rfs.createStream(logsFileName, {
-        // As we have a rolling size, we want to half the max size
-        size: fileInfo === null || fileInfo === void 0 ? void 0 : fileInfo.size,
-        path: logsPath,
-        maxFiles: (fileInfo === null || fileInfo === void 0 ? void 0 : fileInfo.totalHistoryFiles) || 1,
-        immutable: true,
-        history: budibaseLogsHistoryFileName,
-        initialRotation: false,
-    });
-    return outFile;
-}
-exports.localFileDestination = localFileDestination;
-function getLogReadStream() {
-    const streams = [];
-    const historyFile = getFullPath(budibaseLogsHistoryFileName);
-    if (fs_1.default.existsSync(historyFile)) {
-        const fileContent = fs_1.default.readFileSync(historyFile, "utf-8");
-        const historyFiles = fileContent.split("\n");
-        for (const historyFile of historyFiles.filter(x => x)) {
-            streams.push(fs_1.default.readFileSync(historyFile));
-        }
-    }
-    streams.push(fs_1.default.readFileSync(getFullPath(logsFileName)));
-    const combinedContent = Buffer.concat(streams);
-    return combinedContent;
-}
-exports.getLogReadStream = getLogReadStream;
-//# sourceMappingURL=system.js.map

package/dist/src/logging/system.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"system.js","sourceRoot":"","sources":["../../../src/logging/system.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,4CAAmB;AACnB,gDAAuB;AACvB,0DAA2C;AAE3C,iEAAgC;AAChC,gDAAgD;AAEhD,MAAM,YAAY,GAAG,cAAc,CAAA;AACnC,MAAM,2BAA2B,GAAG,2BAA2B,CAAA;AAE/D,MAAM,QAAQ,GAAG,cAAI,CAAC,IAAI,CAAC,IAAA,6BAAe,GAAE,EAAE,YAAY,CAAC,CAAA;AAE3D,SAAS,WAAW,CAAC,QAAgB;IACnC,OAAO,cAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;AACtC,CAAC;AAED,SAAgB,wBAAwB,CAAC,YAAoB;IAC3D,MAAM,KAAK,GAAG,iBAAiB,CAAA;IAC/B,MAAM,KAAK,GAAG,YAAY,aAAZ,YAAY,uBAAZ,YAAY,CAAE,KAAK,CAAC,KAAK,CAAC,CAAA;IACxC,IAAI,CAAC,KAAK,EAAE;QACV,OAAO,CAAC,IAAI,CAAC,0CAA0C,EAAE;YACvD,YAAY;SACb,CAAC,CAAA;QACF,OAAO,SAAS,CAAA;KACjB;IAED,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IACtB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IACrB,IAAI,IAAI,KAAK,CAAC,EAAE;QACd,QAAQ,IAAI,EAAE;YACZ,KAAK,GAAG;gBACN,OAAO,EAAE,IAAI,EAAE,GAAG,IAAI,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAA;YACnD,KAAK,GAAG;gBACN,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAA;YAChE,KAAK,GAAG;gBACN,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAA;YAChE,KAAK,GAAG;gBACN,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAA;YAChE;gBACE,OAAO,SAAS,CAAA;SACnB;KACF;IAED,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,EAAE;QAClB,OAAO,EAAE,IAAI,EAAE,GAAG,IAAI,GAAG,CAAC,GAAG,IAAI,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAA;KAC5D;IAED,OAAO,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,EAAE,iBAAiB,EAAE,IAAI,GAAG,CAAC,EAAE,CAAA;AAC1D,CAAC;AAhCD,4DAgCC;AAED,SAAgB,oBAAoB;IAClC,MAAM,QAAQ,GAAG,wBAAwB,CAAC,qBAAG,CAAC,oBAAoB,CAAC,CAAA;IACnE,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,YAAY,EAAE;QAC7C,0DAA0D;QAC1D,IAAI,EAAE,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,IAAI;QACpB,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,CAAA,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,iBAAiB,KAAI,CAAC;QAC1C,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,2BAA2B;QACpC,eAAe,EAAE,KAAK;KACvB,CAAC,CAAA;IAEF,OAAO,OAAO,CAAA;AAChB,CAAC;AAbD,oDAaC;AAED,SAAgB,gBAAgB;IAC9B,MAAM,OAAO,GAAG,EAAE,CAAA;IAClB,MAAM,WAAW,GAAG,WAAW,CAAC,2BAA2B,CAAC,CAAA;IAC5D,IAAI,YAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE;QAC9B,MAAM,WAAW,GAAG,YAAE,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAA;QACzD,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC5C,KAAK,MAAM,WAAW,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;YACrD,OAAO,CAAC,IAAI,CAAC,YAAE,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAA;SAC3C;KACF;IAED,OAAO,CAAC,IAAI,CAAC,YAAE,CAAC,YAAY,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;IAExD,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC9C,OAAO,eAAe,CAAA;AACxB,CAAC;AAfD,4CAeC"}

package/src/logging/system.ts

@@ -1,81 +0,0 @@
-import fs from "fs"
-import path from "path"
-import * as rfs from "rotating-file-stream"
-
-import env from "../environment"
-import { budibaseTempDir } from "../objectStore"
-
-const logsFileName = `budibase.log`
-const budibaseLogsHistoryFileName = "budibase-logs-history.txt"
-
-const logsPath = path.join(budibaseTempDir(), "systemlogs")
-
-function getFullPath(fileName: string) {
-  return path.join(logsPath, fileName)
-}
-
-export function getSingleFileMaxSizeInfo(totalMaxSize: string) {
-  const regex = /(\d+)([A-Za-z])/
-  const match = totalMaxSize?.match(regex)
-  if (!match) {
-    console.warn(`totalMaxSize does not have a valid value`, {
-      totalMaxSize,
-    })
-    return undefined
-  }
-
-  const size = +match[1]
-  const unit = match[2]
-  if (size === 1) {
-    switch (unit) {
-      case "B":
-        return { size: `${size}B`, totalHistoryFiles: 1 }
-      case "K":
-        return { size: `${(size * 1000) / 2}B`, totalHistoryFiles: 1 }
-      case "M":
-        return { size: `${(size * 1000) / 2}K`, totalHistoryFiles: 1 }
-      case "G":
-        return { size: `${(size * 1000) / 2}M`, totalHistoryFiles: 1 }
-      default:
-        return undefined
-    }
-  }
-
-  if (size % 2 === 0) {
-    return { size: `${size / 2}${unit}`, totalHistoryFiles: 1 }
-  }
-
-  return { size: `1${unit}`, totalHistoryFiles: size - 1 }
-}
-
-export function localFileDestination() {
-  const fileInfo = getSingleFileMaxSizeInfo(env.ROLLING_LOG_MAX_SIZE)
-  const outFile = rfs.createStream(logsFileName, {
-    // As we have a rolling size, we want to half the max size
-    size: fileInfo?.size,
-    path: logsPath,
-    maxFiles: fileInfo?.totalHistoryFiles || 1,
-    immutable: true,
-    history: budibaseLogsHistoryFileName,
-    initialRotation: false,
-  })
-
-  return outFile
-}
-
-export function getLogReadStream() {
-  const streams = []
-  const historyFile = getFullPath(budibaseLogsHistoryFileName)
-  if (fs.existsSync(historyFile)) {
-    const fileContent = fs.readFileSync(historyFile, "utf-8")
-    const historyFiles = fileContent.split("\n")
-    for (const historyFile of historyFiles.filter(x => x)) {
-      streams.push(fs.readFileSync(historyFile))
-    }
-  }
-
-  streams.push(fs.readFileSync(getFullPath(logsFileName)))
-
-  const combinedContent = Buffer.concat(streams)
-  return combinedContent
-}

package/src/logging/tests/system.spec.ts

@@ -1,61 +0,0 @@
-import { getSingleFileMaxSizeInfo } from "../system"
-
-describe("system", () => {
-  describe("getSingleFileMaxSizeInfo", () => {
-    it.each([
-      ["100B", "50B"],
-      ["200K", "100K"],
-      ["20M", "10M"],
-      ["4G", "2G"],
-    ])(
-      "Halving even number (%s) returns halved size and 1 history file (%s)",
-      (totalValue, expectedMaxSize) => {
-        const result = getSingleFileMaxSizeInfo(totalValue)
-        expect(result).toEqual({
-          size: expectedMaxSize,
-          totalHistoryFiles: 1,
-        })
-      }
-    )
-
-    it.each([
-      ["5B", "1B", 4],
-      ["17K", "1K", 16],
-      ["21M", "1M", 20],
-      ["3G", "1G", 2],
-    ])(
-      "Halving an odd number (%s) returns as many files as size (-1) (%s)",
-      (totalValue, expectedMaxSize, totalHistoryFiles) => {
-        const result = getSingleFileMaxSizeInfo(totalValue)
-        expect(result).toEqual({
-          size: expectedMaxSize,
-          totalHistoryFiles,
-        })
-      }
-    )
-
-    it.each([
-      ["1B", "1B"],
-      ["1K", "500B"],
-      ["1M", "500K"],
-      ["1G", "500M"],
-    ])(
-      "Halving '%s' returns halved unit (%s)",
-      (totalValue, expectedMaxSize) => {
-        const result = getSingleFileMaxSizeInfo(totalValue)
-        expect(result).toEqual({
-          size: expectedMaxSize,
-          totalHistoryFiles: 1,
-        })
-      }
-    )
-
-    it.each([[undefined], [""], ["50"], ["wrongvalue"]])(
-      "Halving wrongly formatted value ('%s') returns undefined",
-      totalValue => {
-        const result = getSingleFileMaxSizeInfo(totalValue!)
-        expect(result).toBeUndefined()
-      }
-    )
-  })
-})