@budibase/backend-core 2.24.2 → 2.26.0

package/src/cache/user.ts

@@ -69,7 +69,7 @@ async function populateUsersFromDB(
 export async function getUser(
   userId: string,
   tenantId?: string,
-  populateUser?: any
+  populateUser?: (userId: string, tenantId: string) => Promise<User>
 ) {
   if (!populateUser) {
     populateUser = populateFromDB
@@ -83,7 +83,7 @@ export async function getUser(
   }
   const client = await redis.getUserClient()
   // try cache
-  let user = await client.get(userId)
+  let user: User = await client.get(userId)
   if (!user) {
     user = await populateUser(userId, tenantId)
     await client.store(userId, user, EXPIRY_SECONDS)

package/src/context/mainContext.ts

@@ -281,7 +281,7 @@ export function doInScimContext(task: any) {
   return newContext(updates, task)
 }
 
-export async function ensureSnippetContext() {
+export async function ensureSnippetContext(enabled = !env.isTest()) {
   const ctx = getCurrentContext()
 
   // If we've already added snippets to context, continue
@@ -292,7 +292,7 @@ export async function ensureSnippetContext() {
   // Otherwise get snippets for this app and update context
   let snippets: Snippet[] | undefined
   const db = getAppDB()
-  if (db && !env.isTest()) {
+  if (db && enabled) {
     const app = await db.get<App>(DocumentType.APP_METADATA)
     snippets = app.snippets
   }

package/src/db/couch/DatabaseImpl.ts

@@ -3,11 +3,11 @@ import {
   AllDocsResponse,
   AnyDocument,
   Database,
-  DatabaseOpts,
-  DatabaseQueryOpts,
-  DatabasePutOpts,
   DatabaseCreateIndexOpts,
   DatabaseDeleteIndexOpts,
+  DatabaseOpts,
+  DatabasePutOpts,
+  DatabaseQueryOpts,
   Document,
   isDocument,
   RowResponse,
@@ -17,7 +17,7 @@ import {
 import { getCouchInfo } from "./connections"
 import { directCouchUrlCall } from "./utils"
 import { getPouchDB } from "./pouchDB"
-import { WriteStream, ReadStream } from "fs"
+import { ReadStream, WriteStream } from "fs"
 import { newid } from "../../docIds/newid"
 import { SQLITE_DESIGN_DOC_ID } from "../../constants"
 import { DDInstrumentedDatabase } from "../instrumentation"
@@ -38,6 +38,39 @@ function buildNano(couchInfo: { url: string; cookie: string }) {
 
 type DBCall<T> = () => Promise<T>
 
+class CouchDBError extends Error {
+  status: number
+  statusCode: number
+  reason: string
+  name: string
+  errid: string
+  error: string
+  description: string
+
+  constructor(
+    message: string,
+    info: {
+      status: number | undefined
+      statusCode: number | undefined
+      name: string
+      errid: string
+      description: string
+      reason: string
+      error: string
+    }
+  ) {
+    super(message)
+    const statusCode = info.status || info.statusCode || 500
+    this.status = statusCode
+    this.statusCode = statusCode
+    this.reason = info.reason
+    this.name = info.name
+    this.errid = info.errid
+    this.description = info.description
+    this.error = info.error
+  }
+}
+
 export function DatabaseWithConnection(
   dbName: string,
   connection: string,
@@ -119,7 +152,7 @@ export class DatabaseImpl implements Database {
       } catch (err: any) {
         // Handling race conditions
         if (err.statusCode !== 412) {
-          throw err
+          throw new CouchDBError(err.message, err)
         }
       }
     }
@@ -138,10 +171,9 @@ export class DatabaseImpl implements Database {
       if (err.statusCode === 404 && err.reason === DATABASE_NOT_FOUND) {
         await this.checkAndCreateDb()
         return await this.performCall(call)
-      } else if (err.statusCode) {
-        err.status = err.statusCode
       }
-      throw err
+      // stripping the error down the props which are safe/useful, drop everything else
+      throw new CouchDBError(`CouchDB error: ${err.message}`, err)
     }
   }
 
@@ -288,7 +320,7 @@ export class DatabaseImpl implements Database {
       if (err.statusCode === 404) {
         return
       } else {
-        throw { ...err, status: err.statusCode }
+        throw new CouchDBError(err.message, err)
       }
     }
   }

package/src/middleware/authenticated.ts

@@ -13,7 +13,7 @@ import { getGlobalDB, doInTenant } from "../context"
 import { decrypt } from "../security/encryption"
 import * as identity from "../context/identity"
 import env from "../environment"
-import { Ctx, EndpointMatcher, SessionCookie } from "@budibase/types"
+import { Ctx, EndpointMatcher, SessionCookie, User } from "@budibase/types"
 import { InvalidAPIKeyError, ErrorCode } from "../errors"
 import tracer from "dd-trace"
 
@@ -41,7 +41,10 @@ function finalise(ctx: any, opts: FinaliseOpts = {}) {
   ctx.version = opts.version
 }
 
-async function checkApiKey(apiKey: string, populateUser?: Function) {
+async function checkApiKey(
+  apiKey: string,
+  populateUser?: (userId: string, tenantId: string) => Promise<User>
+) {
   // check both the primary and the fallback internal api keys
   // this allows for rotation
   if (isValidInternalAPIKey(apiKey)) {
@@ -128,6 +131,7 @@ export default function (
           } else {
             user = await getUser(userId, session.tenantId)
           }
+          // @ts-ignore
           user.csrfToken = session.csrfToken
 
           if (session?.lastAccessedAt < timeMinusOneMinute()) {
@@ -167,19 +171,25 @@ export default function (
         authenticated = false
       }
 
-      if (user) {
+      const isUser = (
+        user: any
+      ): user is User & { budibaseAccess?: string } => {
+        return user && user.email
+      }
+
+      if (isUser(user)) {
         tracer.setUser({
-          id: user?._id,
-          tenantId: user?.tenantId,
-          budibaseAccess: user?.budibaseAccess,
-          status: user?.status,
+          id: user._id!,
+          tenantId: user.tenantId,
+          budibaseAccess: user.budibaseAccess,
+          status: user.status,
         })
       }
 
       // isAuthenticated is a function, so use a variable to be able to check authed state
       finalise(ctx, { authenticated, user, internal, version, publicEndpoint })
 
-      if (user && user.email) {
+      if (isUser(user)) {
         return identity.doInUserContext(user, ctx, next)
       } else {
         return next()

package/src/objectStore/objectStore.ts

@@ -13,13 +13,14 @@ import { bucketTTLConfig, budibaseTempDir } from "./utils"
 import { v4 } from "uuid"
 import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
 import fsp from "fs/promises"
+import { HeadObjectOutput } from "aws-sdk/clients/s3"
 
 const streamPipeline = promisify(stream.pipeline)
 // use this as a temporary store of buckets that are being created
 const STATE = {
   bucketCreationPromises: {},
 }
-const signedFilePrefix = "/files/signed"
+export const SIGNED_FILE_PREFIX = "/files/signed"
 
 type ListParams = {
   ContinuationToken?: string
@@ -40,8 +41,13 @@ type UploadParams = BaseUploadParams & {
   path?: string | PathLike
 }
 
-type StreamUploadParams = BaseUploadParams & {
-  stream: ReadStream
+export type StreamTypes =
+  | ReadStream
+  | NodeJS.ReadableStream
+  | ReadableStream<Uint8Array>
+
+export type StreamUploadParams = BaseUploadParams & {
+  stream?: StreamTypes
 }
 
 const CONTENT_TYPE_MAP: any = {
@@ -174,11 +180,9 @@ export async function upload({
   const objectStore = ObjectStore(bucketName)
   const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
 
-  if (ttl && (bucketCreated.created || bucketCreated.exists)) {
+  if (ttl && bucketCreated.created) {
     let ttlConfig = bucketTTLConfig(bucketName, ttl)
-    if (objectStore.putBucketLifecycleConfiguration) {
-      await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
-    }
+    await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
   }
 
   let contentType = type
@@ -222,11 +226,9 @@ export async function streamUpload({
   const objectStore = ObjectStore(bucketName)
   const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
 
-  if (ttl && (bucketCreated.created || bucketCreated.exists)) {
+  if (ttl && bucketCreated.created) {
     let ttlConfig = bucketTTLConfig(bucketName, ttl)
-    if (objectStore.putBucketLifecycleConfiguration) {
-      await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
-    }
+    await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
   }
 
   // Set content type for certain known extensions
@@ -333,7 +335,7 @@ export function getPresignedUrl(
     const signedUrl = new URL(url)
     const path = signedUrl.pathname
     const query = signedUrl.search
-    return `${signedFilePrefix}${path}${query}`
+    return `${SIGNED_FILE_PREFIX}${path}${query}`
   }
 }
 
@@ -521,6 +523,26 @@ export async function getReadStream(
   return client.getObject(params).createReadStream()
 }
 
+export async function getObjectMetadata(
+  bucket: string,
+  path: string
+): Promise<HeadObjectOutput> {
+  bucket = sanitizeBucket(bucket)
+  path = sanitizeKey(path)
+
+  const client = ObjectStore(bucket)
+  const params = {
+    Bucket: bucket,
+    Key: path,
+  }
+
+  try {
+    return await client.headObject(params).promise()
+  } catch (err: any) {
+    throw new Error("Unable to retrieve metadata from object")
+  }
+}
+
 /*
 Given a signed url like '/files/signed/tmp-files-attachments/app_123456/myfile.txt' extract
 the bucket and the path from it
@@ -530,7 +552,9 @@ export function extractBucketAndPath(
 ): { bucket: string; path: string } | null {
   const baseUrl = url.split("?")[0]
 
-  const regex = new RegExp(`^${signedFilePrefix}/(?<bucket>[^/]+)/(?<path>.+)$`)
+  const regex = new RegExp(
+    `^${SIGNED_FILE_PREFIX}/(?<bucket>[^/]+)/(?<path>.+)$`
+  )
   const match = baseUrl.match(regex)
 
   if (match && match.groups) {

package/src/objectStore/utils.ts

@@ -1,9 +1,14 @@
-import { join } from "path"
+import path, { join } from "path"
 import { tmpdir } from "os"
 import fs from "fs"
 import env from "../environment"
 import { PutBucketLifecycleConfigurationRequest } from "aws-sdk/clients/s3"
-
+import * as objectStore from "./objectStore"
+import {
+  AutomationAttachment,
+  AutomationAttachmentContent,
+  BucketedContent,
+} from "@budibase/types"
 /****************************************************
  *      NOTE: When adding a new bucket - name       *
  *     sure that S3 usages (like budibase-infra)    *
@@ -55,3 +60,50 @@ export const bucketTTLConfig = (
 
   return params
 }
+
+async function processUrlAttachment(
+  attachment: AutomationAttachment
+): Promise<AutomationAttachmentContent> {
+  const response = await fetch(attachment.url)
+  if (!response.ok || !response.body) {
+    throw new Error(`Unexpected response ${response.statusText}`)
+  }
+  const fallbackFilename = path.basename(new URL(attachment.url).pathname)
+  return {
+    filename: attachment.filename || fallbackFilename,
+    content: response.body,
+  }
+}
+
+export async function processObjectStoreAttachment(
+  attachment: AutomationAttachment
+): Promise<BucketedContent> {
+  const result = objectStore.extractBucketAndPath(attachment.url)
+
+  if (result === null) {
+    throw new Error("Invalid signed URL")
+  }
+
+  const { bucket, path: objectPath } = result
+  const readStream = await objectStore.getReadStream(bucket, objectPath)
+  const fallbackFilename = path.basename(objectPath)
+  return {
+    bucket,
+    path: objectPath,
+    filename: attachment.filename || fallbackFilename,
+    content: readStream,
+  }
+}
+
+export async function processAutomationAttachment(
+  attachment: AutomationAttachment
+): Promise<AutomationAttachmentContent | BucketedContent> {
+  const isFullyFormedUrl =
+    attachment.url?.startsWith("http://") ||
+    attachment.url?.startsWith("https://")
+  if (isFullyFormedUrl) {
+    return await processUrlAttachment(attachment)
+  } else {
+    return await processObjectStoreAttachment(attachment)
+  }
+}