@budibase/backend-core 2.5.6-alpha.6 → 2.5.6

package/src/db/utils.ts

@@ -1,12 +1,257 @@
+import { newid } from "../newid"
 import env from "../environment"
-import { DEFAULT_TENANT_ID, SEPARATOR, DocumentType } from "../constants"
+import {
+  DEFAULT_TENANT_ID,
+  SEPARATOR,
+  DocumentType,
+  UNICODE_MAX,
+  ViewName,
+  InternalTable,
+  APP_PREFIX,
+} from "../constants"
 import { getTenantId, getGlobalDBName } from "../context"
 import { doWithDB, directCouchAllDbs } from "./db"
 import { getAppMetadata } from "../cache/appMetadata"
-import { isDevApp, isDevAppID, getProdAppID } from "../docIds/conversions"
+import { isDevApp, isDevAppID, getProdAppID } from "./conversions"
 import { App, Database } from "@budibase/types"
-import { getStartEndKeyURL } from "../docIds"
-export * from "../docIds"
+
+/**
+ * Generates a new app ID.
+ * @returns {string} The new app ID which the app doc can be stored under.
+ */
+export const generateAppID = (tenantId?: string | null) => {
+  let id = APP_PREFIX
+  if (tenantId) {
+    id += `${tenantId}${SEPARATOR}`
+  }
+  return `${id}${newid()}`
+}
+
+/**
+ * If creating DB allDocs/query params with only a single top level ID this can be used, this
+ * is usually the case as most of our docs are top level e.g. tables, automations, users and so on.
+ * More complex cases such as link docs and rows which have multiple levels of IDs that their
+ * ID consists of need their own functions to build the allDocs parameters.
+ * @param {string} docType The type of document which input params are being built for, e.g. user,
+ * link, app, table and so on.
+ * @param {string|null} docId The ID of the document minus its type - this is only needed if looking
+ * for a singular document.
+ * @param {object} otherProps Add any other properties onto the request, e.g. include_docs.
+ * @returns {object} Parameters which can then be used with an allDocs request.
+ */
+export function getDocParams(
+  docType: string,
+  docId?: string | null,
+  otherProps: any = {}
+) {
+  if (docId == null) {
+    docId = ""
+  }
+  return {
+    ...otherProps,
+    startkey: `${docType}${SEPARATOR}${docId}`,
+    endkey: `${docType}${SEPARATOR}${docId}${UNICODE_MAX}`,
+  }
+}
+
+/**
+ * Gets the DB allDocs/query params for retrieving a row.
+ * @param {string|null} tableId The table in which the rows have been stored.
+ * @param {string|null} rowId The ID of the row which is being specifically queried for. This can be
+ * left null to get all the rows in the table.
+ * @param {object} otherProps Any other properties to add to the request.
+ * @returns {object} Parameters which can then be used with an allDocs request.
+ */
+export function getRowParams(
+  tableId?: string | null,
+  rowId?: string | null,
+  otherProps = {}
+) {
+  if (tableId == null) {
+    return getDocParams(DocumentType.ROW, null, otherProps)
+  }
+
+  const endOfKey = rowId == null ? `${tableId}${SEPARATOR}` : rowId
+
+  return getDocParams(DocumentType.ROW, endOfKey, otherProps)
+}
+
+/**
+ * Retrieve the correct index for a view based on default design DB.
+ */
+export function getQueryIndex(viewName: ViewName) {
+  return `database/${viewName}`
+}
+
+/**
+ * Gets a new row ID for the specified table.
+ * @param {string} tableId The table which the row is being created for.
+ * @param {string|null} id If an ID is to be used then the UUID can be substituted for this.
+ * @returns {string} The new ID which a row doc can be stored under.
+ */
+export function generateRowID(tableId: string, id?: string) {
+  id = id || newid()
+  return `${DocumentType.ROW}${SEPARATOR}${tableId}${SEPARATOR}${id}`
+}
+
+/**
+ * Check if a given ID is that of a table.
+ * @returns {boolean}
+ */
+export const isTableId = (id: string) => {
+  // this includes datasource plus tables
+  return (
+    id &&
+    (id.startsWith(`${DocumentType.TABLE}${SEPARATOR}`) ||
+      id.startsWith(`${DocumentType.DATASOURCE_PLUS}${SEPARATOR}`))
+  )
+}
+
+/**
+ * Check if a given ID is that of a datasource or datasource plus.
+ * @returns {boolean}
+ */
+export const isDatasourceId = (id: string) => {
+  // this covers both datasources and datasource plus
+  return id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`)
+}
+
+/**
+ * Generates a new workspace ID.
+ * @returns {string} The new workspace ID which the workspace doc can be stored under.
+ */
+export function generateWorkspaceID() {
+  return `${DocumentType.WORKSPACE}${SEPARATOR}${newid()}`
+}
+
+/**
+ * Gets parameters for retrieving workspaces.
+ */
+export function getWorkspaceParams(id = "", otherProps = {}) {
+  return {
+    ...otherProps,
+    startkey: `${DocumentType.WORKSPACE}${SEPARATOR}${id}`,
+    endkey: `${DocumentType.WORKSPACE}${SEPARATOR}${id}${UNICODE_MAX}`,
+  }
+}
+
+/**
+ * Generates a new global user ID.
+ * @returns {string} The new user ID which the user doc can be stored under.
+ */
+export function generateGlobalUserID(id?: any) {
+  return `${DocumentType.USER}${SEPARATOR}${id || newid()}`
+}
+
+/**
+ * Gets parameters for retrieving users.
+ */
+export function getGlobalUserParams(globalId: any, otherProps: any = {}) {
+  if (!globalId) {
+    globalId = ""
+  }
+  const startkey = otherProps?.startkey
+  return {
+    ...otherProps,
+    // need to include this incase pagination
+    startkey: startkey
+      ? startkey
+      : `${DocumentType.USER}${SEPARATOR}${globalId}`,
+    endkey: `${DocumentType.USER}${SEPARATOR}${globalId}${UNICODE_MAX}`,
+  }
+}
+
+/**
+ * Gets parameters for retrieving users, this is a utility function for the getDocParams function.
+ */
+export function getUserMetadataParams(userId?: string | null, otherProps = {}) {
+  return getRowParams(InternalTable.USER_METADATA, userId, otherProps)
+}
+
+/**
+ * Generates a new user ID based on the passed in global ID.
+ * @param {string} globalId The ID of the global user.
+ * @returns {string} The new user ID which the user doc can be stored under.
+ */
+export function generateUserMetadataID(globalId: string) {
+  return generateRowID(InternalTable.USER_METADATA, globalId)
+}
+
+/**
+ * Breaks up the ID to get the global ID.
+ */
+export function getGlobalIDFromUserMetadataID(id: string) {
+  const prefix = `${DocumentType.ROW}${SEPARATOR}${InternalTable.USER_METADATA}${SEPARATOR}`
+  if (!id || !id.includes(prefix)) {
+    return id
+  }
+  return id.split(prefix)[1]
+}
+
+export function getUsersByAppParams(appId: any, otherProps: any = {}) {
+  const prodAppId = getProdAppID(appId)
+  return {
+    ...otherProps,
+    startkey: prodAppId,
+    endkey: `${prodAppId}${UNICODE_MAX}`,
+  }
+}
+
+/**
+ * Generates a template ID.
+ * @param ownerId The owner/user of the template, this could be global or a workspace level.
+ */
+export function generateTemplateID(ownerId: any) {
+  return `${DocumentType.TEMPLATE}${SEPARATOR}${ownerId}${SEPARATOR}${newid()}`
+}
+
+export function generateAppUserID(prodAppId: string, userId: string) {
+  return `${prodAppId}${SEPARATOR}${userId}`
+}
+
+/**
+ * Gets parameters for retrieving templates. Owner ID must be specified, either global or a workspace level.
+ */
+export function getTemplateParams(
+  ownerId: any,
+  templateId: any,
+  otherProps = {}
+) {
+  if (!templateId) {
+    templateId = ""
+  }
+  let final
+  if (templateId) {
+    final = templateId
+  } else {
+    final = `${DocumentType.TEMPLATE}${SEPARATOR}${ownerId}${SEPARATOR}`
+  }
+  return {
+    ...otherProps,
+    startkey: final,
+    endkey: `${final}${UNICODE_MAX}`,
+  }
+}
+
+/**
+ * Generates a new role ID.
+ * @returns {string} The new role ID which the role doc can be stored under.
+ */
+export function generateRoleID(id?: any) {
+  return `${DocumentType.ROLE}${SEPARATOR}${id || newid()}`
+}
+
+/**
+ * Gets parameters for retrieving a role, this is a utility function for the getDocParams function.
+ */
+export function getRoleParams(roleId?: string | null, otherProps = {}) {
+  return getDocParams(DocumentType.ROLE, roleId, otherProps)
+}
+
+export function getStartEndKeyURL(baseKey: any, tenantId?: string) {
+  const tenancy = tenantId ? `${SEPARATOR}${tenantId}` : ""
+  return `startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"`
+}
 
 /**
  * if in production this will use the CouchDB _all_dbs call to retrieve a list of databases. If testing
@@ -166,8 +411,31 @@ export async function dbExists(dbName: any) {
   )
 }
 
-export function pagination<T>(
-  data: T[],
+/**
+ * Generates a new dev info document ID - this is scoped to a user.
+ * @returns {string} The new dev info ID which info for dev (like api key) can be stored under.
+ */
+export const generateDevInfoID = (userId: any) => {
+  return `${DocumentType.DEV_INFO}${SEPARATOR}${userId}`
+}
+
+/**
+ * Generates a new plugin ID - to be used in the global DB.
+ * @returns {string} The new plugin ID which a plugin metadata document can be stored under.
+ */
+export const generatePluginID = (name: string) => {
+  return `${DocumentType.PLUGIN}${SEPARATOR}${name}`
+}
+
+/**
+ * Gets parameters for retrieving automations, this is a utility function for the getDocParams function.
+ */
+export const getPluginParams = (pluginId?: string | null, otherProps = {}) => {
+  return getDocParams(DocumentType.PLUGIN, pluginId, otherProps)
+}
+
+export function pagination(
+  data: any[],
   pageSize: number,
   {
     paginate,
@@ -176,7 +444,7 @@ export function pagination<T>(
   }: {
     paginate: boolean
     property: string
-    getKey?: (doc: T) => string | undefined
+    getKey?: (doc: any) => string | undefined
   } = {
     paginate: true,
     property: "_id",

package/src/db/views.ts

@@ -42,11 +42,7 @@ async function removeDeprecated(db: Database, viewName: ViewName) {
   }
 }
 
-export async function createView(
-  db: any,
-  viewJs: string,
-  viewName: string
-): Promise<void> {
+export async function createView(db: any, viewJs: string, viewName: string) {
   let designDoc
   try {
     designDoc = (await db.get(DESIGN_DB)) as DesignDocument
@@ -61,15 +57,7 @@ export async function createView(
     ...designDoc.views,
     [viewName]: view,
   }
-  try {
-    await db.put(designDoc)
-  } catch (err: any) {
-    if (err.status === 409) {
-      return await createView(db, viewJs, viewName)
-    } else {
-      throw err
-    }
-  }
+  await db.put(designDoc)
 }
 
 export const createNewUserEmailView = async () => {
@@ -147,10 +135,6 @@ export const queryView = async <T>(
       await removeDeprecated(db, viewName)
       await createFunc()
       return queryView(viewName, params, db, createFunc, opts)
-    } else if (err.status === 409) {
-      // can happen when multiple queries occur at once, view couldn't be created
-      // other design docs being updated, re-run
-      return queryView(viewName, params, db, createFunc, opts)
     } else {
       throw err
     }

package/src/environment.ts

@@ -36,15 +36,6 @@ function getAPIEncryptionKey() {
     : process.env.JWT_SECRET // fallback to the JWT_SECRET used historically
 }
 
-function httpLogging() {
-  if (process.env.HTTP_LOGGING === undefined) {
-    // on by default unless otherwise specified
-    return true
-  }
-
-  return process.env.HTTP_LOGGING
-}
-
 const environment = {
   isTest,
   isJest,
@@ -99,11 +90,11 @@ const environment = {
   USE_COUCH: process.env.USE_COUCH || true,
   DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
   SERVICE: process.env.SERVICE || "budibase",
-  LOG_LEVEL: process.env.LOG_LEVEL || "info",
+  LOG_LEVEL: process.env.LOG_LEVEL,
   SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
   DEPLOYMENT_ENVIRONMENT:
     process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose",
-  HTTP_LOGGING: httpLogging(),
+  ENABLE_4XX_HTTP_LOGGING: process.env.ENABLE_4XX_HTTP_LOGGING || true,
   ENABLE_AUDIT_LOG_IP_ADDR: process.env.ENABLE_AUDIT_LOG_IP_ADDR,
   // smtp
   SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,

package/src/errors/errors.ts

@@ -97,11 +97,3 @@ export class InvalidAPIKeyError extends BudibaseError {
     )
   }
 }
-
-// USERS
-
-export class EmailUnavailableError extends Error {
-  constructor(email: string) {
-    super(`Email already in use: '${email}'`)
-  }
-}

package/src/events/events.ts

@@ -1,8 +1,7 @@
-import { Event } from "@budibase/types"
+import { Event, AuditedEventFriendlyName } from "@budibase/types"
 import { processors } from "./processors"
 import identification from "./identification"
 import * as backfill from "./backfill"
-import { publishAsyncEvent } from "./asyncEvents"
 
 export const publishEvent = async (
   event: Event,
@@ -15,14 +14,6 @@ export const publishEvent = async (
   const backfilling = await backfill.isBackfillingEvent(event)
   // no backfill - send the event and exit
   if (!backfilling) {
-    // send off async events if required
-    await publishAsyncEvent({
-      event,
-      identity,
-      properties,
-      timestamp,
-    })
-    // now handle the main sync event processing pipeline
     await processors.processEvent(event, identity, properties, timestamp)
     return
   }

package/src/events/identification.ts

@@ -65,7 +65,6 @@ const getCurrentIdentity = async (): Promise<Identity> => {
       hosting,
       installationId,
       tenantId,
-      realTenantId: context.getTenantId(),
       environment,
     }
   } else if (identityType === IdentityType.USER) {

package/src/events/index.ts

@@ -6,8 +6,6 @@ export * as backfillCache from "./backfill"
 
 import { processors } from "./processors"
 
-export function initAsyncEvents() {}
-
 export const shutdown = () => {
   processors.shutdown()
   console.log("Events shutdown")

package/src/events/processors/LoggingProcessor.ts

@@ -2,6 +2,14 @@ import { Event, Identity, Group } from "@budibase/types"
 import { EventProcessor } from "./types"
 import env from "../../environment"
 
+const getTimestampString = (timestamp?: string | number) => {
+  let timestampString = ""
+  if (timestamp) {
+    timestampString = `[timestamp=${new Date(timestamp).toISOString()}]`
+  }
+  return timestampString
+}
+
 const skipLogging = env.SELF_HOSTED && !env.isDev()
 
 export default class LoggingProcessor implements EventProcessor {
@@ -14,21 +22,32 @@ export default class LoggingProcessor implements EventProcessor {
     if (skipLogging) {
       return
     }
-    console.log(`[audit] [identityType=${identity.type}] ${event}`, properties)
+    let timestampString = getTimestampString(timestamp)
+    let message = `[audit] [identityType=${identity.type}] ${timestampString} ${event} `
+    if (env.isDev()) {
+      message = message + `[debug: [properties=${JSON.stringify(properties)}] ]`
+    }
+    console.log(message)
   }
 
   async identify(identity: Identity, timestamp?: string | number) {
     if (skipLogging) {
       return
     }
-    console.log(`[audit] identified`, identity)
+    let timestampString = getTimestampString(timestamp)
+    console.log(
+      `[audit] [${JSON.stringify(identity)}] ${timestampString} identified`
+    )
   }
 
   async identifyGroup(group: Group, timestamp?: string | number) {
     if (skipLogging) {
       return
     }
-    console.log(`[audit] group identified`, group)
+    let timestampString = getTimestampString(timestamp)
+    console.log(
+      `[audit] [${JSON.stringify(group)}] ${timestampString} group identified`
+    )
   }
 
   shutdown(): void {

package/src/events/processors/Processors.ts

@@ -25,9 +25,7 @@ export default class Processor implements EventProcessor {
     timestamp?: string | number
   ): Promise<void> {
     for (const eventProcessor of this.processors) {
-      if (eventProcessor.identify) {
-        await eventProcessor.identify(identity, timestamp)
-      }
+      await eventProcessor.identify(identity, timestamp)
     }
   }
 
@@ -36,17 +34,13 @@ export default class Processor implements EventProcessor {
     timestamp?: string | number
   ): Promise<void> {
     for (const eventProcessor of this.processors) {
-      if (eventProcessor.identifyGroup) {
-        await eventProcessor.identifyGroup(identity, timestamp)
-      }
+      await eventProcessor.identifyGroup(identity, timestamp)
     }
   }
 
   shutdown() {
     for (const eventProcessor of this.processors) {
-      if (eventProcessor.shutdown) {
-        eventProcessor.shutdown()
-      }
+      eventProcessor.shutdown()
     }
   }
 }

package/src/events/processors/posthog/tests/PosthogProcessor.spec.ts

@@ -1,4 +1,4 @@
-import { testEnv } from "../../../../../tests/extra"
+import { testEnv } from "../../../../../tests"
 import PosthogProcessor from "../PosthogProcessor"
 import { Event, IdentityType, Hosting } from "@budibase/types"
 const tk = require("timekeeper")

package/src/events/processors/types.ts

@@ -1 +1,18 @@
-export { EventProcessor } from "@budibase/types"
+import { Event, Identity, Group } from "@budibase/types"
+
+export enum EventProcessorType {
+  POSTHOG = "posthog",
+  LOGGING = "logging",
+}
+
+export interface EventProcessor {
+  processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string | number
+  ): Promise<void>
+  identify(identity: Identity, timestamp?: string | number): Promise<void>
+  identifyGroup(group: Group, timestamp?: string | number): Promise<void>
+  shutdown(): void
+}

package/src/events/publishers/group.ts

@@ -9,13 +9,12 @@ import {
   GroupUsersDeletedEvent,
   GroupAddedOnboardingEvent,
   GroupPermissionsEditedEvent,
+  UserGroupRoles,
 } from "@budibase/types"
-import { isScim } from "../../context"
 
 async function created(group: UserGroup, timestamp?: number) {
   const properties: GroupCreatedEvent = {
     groupId: group._id as string,
-    viaScim: isScim(),
     audited: {
       name: group.name,
     },
@@ -26,7 +25,6 @@ async function created(group: UserGroup, timestamp?: number) {
 async function updated(group: UserGroup) {
   const properties: GroupUpdatedEvent = {
     groupId: group._id as string,
-    viaScim: isScim(),
     audited: {
       name: group.name,
     },
@@ -37,7 +35,6 @@ async function updated(group: UserGroup) {
 async function deleted(group: UserGroup) {
   const properties: GroupDeletedEvent = {
     groupId: group._id as string,
-    viaScim: isScim(),
     audited: {
       name: group.name,
     },
@@ -49,7 +46,6 @@ async function usersAdded(count: number, group: UserGroup) {
   const properties: GroupUsersAddedEvent = {
     count,
     groupId: group._id as string,
-    viaScim: isScim(),
     audited: {
       name: group.name,
     },
@@ -61,7 +57,6 @@ async function usersDeleted(count: number, group: UserGroup) {
   const properties: GroupUsersDeletedEvent = {
     count,
     groupId: group._id as string,
-    viaScim: isScim(),
     audited: {
       name: group.name,
     },

package/src/events/publishers/user.ts

@@ -15,12 +15,10 @@ import {
   UserUpdatedEvent,
   UserOnboardingEvent,
 } from "@budibase/types"
-import { isScim } from "../../context"
 
 async function created(user: User, timestamp?: number) {
   const properties: UserCreatedEvent = {
     userId: user._id as string,
-    viaScim: isScim(),
     audited: {
       email: user.email,
     },
@@ -31,7 +29,6 @@ async function created(user: User, timestamp?: number) {
 async function updated(user: User) {
   const properties: UserUpdatedEvent = {
     userId: user._id as string,
-    viaScim: isScim(),
     audited: {
       email: user.email,
     },
@@ -42,7 +39,6 @@ async function updated(user: User) {
 async function deleted(user: User) {
   const properties: UserDeletedEvent = {
     userId: user._id as string,
-    viaScim: isScim(),
     audited: {
       email: user.email,
     },

package/src/index.ts

@@ -27,7 +27,6 @@ export * as errors from "./errors"
 export * as timers from "./timers"
 export { default as env } from "./environment"
 export * as blacklist from "./blacklist"
-export * as docUpdates from "./docUpdates"
 export { SearchParams } from "./db"
 // Add context to tenancy for backwards compatibility
 // only do this for external usages to prevent internal

package/src/logging.ts

@@ -0,0 +1,60 @@
+import { Header } from "./constants"
+import env from "./environment"
+const correlator = require("correlation-id")
+import { Options } from "pino-http"
+import { IncomingMessage } from "http"
+
+const NonErrors = ["AccountError"]
+
+function isSuppressed(e?: any) {
+  return e && e["suppressAlert"]
+}
+
+export function logAlert(message: string, e?: any) {
+  if (e && NonErrors.includes(e.name) && isSuppressed(e)) {
+    return
+  }
+  let errorJson = ""
+  if (e) {
+    errorJson = ": " + JSON.stringify(e, Object.getOwnPropertyNames(e))
+  }
+  console.error(`bb-alert: ${message} ${errorJson}`)
+}
+
+export function logAlertWithInfo(
+  message: string,
+  db: string,
+  id: string,
+  error: any
+) {
+  message = `${message} - db: ${db} - doc: ${id} - error: `
+  logAlert(message, error)
+}
+
+export function logWarn(message: string) {
+  console.warn(`bb-warn: ${message}`)
+}
+
+export function pinoSettings(): Options {
+  return {
+    prettyPrint: {
+      levelFirst: true,
+    },
+    genReqId: correlator.getId,
+    level: env.LOG_LEVEL || "error",
+    autoLogging: {
+      ignore: (req: IncomingMessage) => !!req.url?.includes("/health"),
+    },
+  }
+}
+
+const setCorrelationHeader = (headers: any) => {
+  const correlationId = correlator.getId()
+  if (correlationId) {
+    headers[Header.CORRELATION_ID] = correlationId
+  }
+}
+
+export const correlation = {
+  setHeader: setCorrelationHeader,
+}