@budibase/server 0.0.32 → 0.0.999-alpha.31

package/src/sdk/app/datasources/datasources.ts

@@ -0,0 +1,162 @@
+import { context } from "@budibase/backend-core"
+import { findHBSBlocks, processObjectSync } from "@budibase/string-templates"
+import {
+  Datasource,
+  DatasourceFieldType,
+  Integration,
+  PASSWORD_REPLACEMENT,
+  RestAuthConfig,
+  RestAuthType,
+  RestBasicAuthConfig,
+  SourceName,
+} from "@budibase/types"
+import { cloneDeep } from "lodash/fp"
+import { getEnvironmentVariables } from "../../utils"
+import { getDefinitions, getDefinition } from "../../../integrations"
+
+const ENV_VAR_PREFIX = "env."
+
+export function checkDatasourceTypes(schema: Integration, config: any) {
+  for (let key of Object.keys(config)) {
+    if (!schema.datasource[key]) {
+      continue
+    }
+    const type = schema.datasource[key].type
+    if (
+      type === DatasourceFieldType.NUMBER &&
+      typeof config[key] === "string"
+    ) {
+      config[key] = parseFloat(config[key])
+    }
+  }
+  return config
+}
+
+async function enrichDatasourceWithValues(datasource: Datasource) {
+  const cloned = cloneDeep(datasource)
+  const env = await getEnvironmentVariables()
+  const processed = processObjectSync(
+    cloned,
+    { env },
+    { onlyFound: true }
+  ) as Datasource
+  const definition = await getDefinition(processed.source)
+  processed.config = checkDatasourceTypes(definition, processed.config)
+  return {
+    datasource: processed,
+    envVars: env as Record<string, string>,
+  }
+}
+
+export async function enrich(datasource: Datasource) {
+  const { datasource: response } = await enrichDatasourceWithValues(datasource)
+  return response
+}
+
+export async function get(
+  datasourceId: string,
+  opts?: { enriched: boolean }
+): Promise<Datasource> {
+  const appDb = context.getAppDB()
+  const datasource = await appDb.get(datasourceId)
+  if (opts?.enriched) {
+    return (await enrichDatasourceWithValues(datasource)).datasource
+  } else {
+    return datasource
+  }
+}
+
+export async function getWithEnvVars(datasourceId: string) {
+  const appDb = context.getAppDB()
+  const datasource = await appDb.get(datasourceId)
+  return enrichDatasourceWithValues(datasource)
+}
+
+function hasAuthConfigs(datasource: Datasource) {
+  return datasource.source === SourceName.REST && datasource.config?.authConfigs
+}
+
+function useEnvVars(str: any) {
+  if (typeof str !== "string") {
+    return false
+  }
+  const blocks = findHBSBlocks(str)
+  return blocks.find(block => block.includes(ENV_VAR_PREFIX)) != null
+}
+
+export async function removeSecrets(datasources: Datasource[]) {
+  const definitions = await getDefinitions()
+  for (let datasource of datasources) {
+    const schema = definitions[datasource.source]
+    if (!schema) {
+      continue
+    }
+    if (datasource.config) {
+      // strip secrets from response, so they don't show in the network request
+      if (datasource.config.auth) {
+        delete datasource.config.auth
+      }
+      // specific to REST datasources, contains passwords
+      if (hasAuthConfigs(datasource)) {
+        const configs = datasource.config.authConfigs as RestAuthConfig[]
+        for (let config of configs) {
+          if (config.type !== RestAuthType.BASIC) {
+            continue
+          }
+          const basic = config.config as RestBasicAuthConfig
+          if (!useEnvVars(basic.password)) {
+            basic.password = PASSWORD_REPLACEMENT
+          }
+        }
+      }
+      // remove general passwords
+      for (let key of Object.keys(datasource.config)) {
+        if (
+          schema.datasource?.[key]?.type === DatasourceFieldType.PASSWORD &&
+          !useEnvVars(datasource.config[key])
+        ) {
+          datasource.config[key] = PASSWORD_REPLACEMENT
+        }
+      }
+    }
+  }
+  return datasources
+}
+
+export async function removeSecretSingle(datasource: Datasource) {
+  return (await removeSecrets([datasource]))[0]
+}
+
+export function mergeConfigs(update: Datasource, old: Datasource) {
+  if (!update.config) {
+    return update
+  }
+  // specific to REST datasources, fix the auth configs again if required
+  if (hasAuthConfigs(update)) {
+    const configs = update.config.authConfigs as RestAuthConfig[]
+    const oldConfigs = old.config?.authConfigs as RestAuthConfig[]
+    for (let config of configs) {
+      if (config.type !== RestAuthType.BASIC) {
+        continue
+      }
+      const basic = config.config as RestBasicAuthConfig
+      const oldBasic = oldConfigs.find(old => old.name === config.name)
+        ?.config as RestBasicAuthConfig
+      if (basic.password === PASSWORD_REPLACEMENT) {
+        basic.password = oldBasic.password
+      }
+    }
+  }
+  // update back to actual passwords for everything else
+  for (let [key, value] of Object.entries(update.config)) {
+    if (value !== PASSWORD_REPLACEMENT) {
+      continue
+    }
+    if (old.config?.[key]) {
+      update.config[key] = old.config?.[key]
+    } else {
+      delete update.config[key]
+    }
+  }
+  return update
+}

package/src/sdk/app/datasources/index.ts

@@ -0,0 +1,5 @@
+import * as datasources from "./datasources"
+
+export default {
+  ...datasources,
+}

package/src/sdk/app/queries/index.ts

@@ -0,0 +1,5 @@
+import * as queries from "./queries"
+
+export default {
+  ...queries,
+}

package/src/sdk/app/queries/queries.ts

@@ -0,0 +1,50 @@
+import { getEnvironmentVariables } from "../../utils"
+import { processStringSync } from "@budibase/string-templates"
+
+export async function enrichContext(
+  fields: Record<string, any>,
+  inputs = {}
+): Promise<Record<string, any>> {
+  const enrichedQuery: Record<string, any> = Array.isArray(fields) ? [] : {}
+  if (!fields || !inputs) {
+    return enrichedQuery
+  }
+  const env = await getEnvironmentVariables()
+  const parameters = { ...inputs, env }
+  // enrich the fields with dynamic parameters
+  for (let key of Object.keys(fields)) {
+    if (fields[key] == null) {
+      continue
+    }
+    if (typeof fields[key] === "object") {
+      // enrich nested fields object
+      enrichedQuery[key] = await enrichContext(fields[key], parameters)
+    } else if (typeof fields[key] === "string") {
+      // enrich string value as normal
+      enrichedQuery[key] = processStringSync(fields[key], parameters, {
+        noEscaping: true,
+        noHelpers: true,
+        escapeNewlines: true,
+      })
+    } else {
+      enrichedQuery[key] = fields[key]
+    }
+  }
+  if (
+    enrichedQuery.json ||
+    enrichedQuery.customData ||
+    enrichedQuery.requestBody
+  ) {
+    try {
+      enrichedQuery.json = JSON.parse(
+        enrichedQuery.json ||
+          enrichedQuery.customData ||
+          enrichedQuery.requestBody
+      )
+    } catch (err) {
+      // no json found, ignore
+    }
+    delete enrichedQuery.customData
+  }
+  return enrichedQuery
+}

package/src/sdk/app/rows/attachments.ts

@@ -0,0 +1,60 @@
+import { CouchFindOptions, Table, Row } from "@budibase/types"
+import { db as dbCore } from "@budibase/backend-core"
+import { DocumentType, SEPARATOR } from "../../../db/utils"
+import { FieldTypes } from "../../../constants"
+
+// default limit - seems to work well for performance
+export const FIND_LIMIT = 25
+
+function generateAttachmentFindParams(
+  tableId: string,
+  attachmentCols: string[],
+  bookmark: null | string
+) {
+  const params: CouchFindOptions = {
+    selector: {
+      $or: attachmentCols.map(col => ({ [col]: { $exists: true } })),
+      _id: {
+        $regex: `^${DocumentType.ROW}${SEPARATOR}${tableId}`,
+      },
+    },
+    limit: FIND_LIMIT,
+  }
+  if (bookmark) {
+    params.bookmark = bookmark
+  }
+  return params
+}
+
+export async function getRowsWithAttachments(appId: string, table: Table) {
+  // iterate through attachment documents and update them
+  const db = dbCore.getDB(appId)
+  const attachmentCols: string[] = []
+  for (let [key, column] of Object.entries(table.schema)) {
+    if (column.type === FieldTypes.ATTACHMENT) {
+      attachmentCols.push(key)
+    }
+  }
+  // no attachment columns, nothing to do
+  if (attachmentCols.length === 0) {
+    return { rows: [], columns: [] }
+  }
+  let bookmark: null | string = null,
+    rowsLength = 0,
+    rowList: Row[] = []
+  do {
+    const params = generateAttachmentFindParams(
+      table._id!,
+      attachmentCols,
+      bookmark
+    )
+    // use the CouchDB Mango query API to lookup rows that have attachments
+    const resp = await dbCore.directCouchFind(db.name, params)
+    bookmark = resp.bookmark
+    rowsLength = resp.rows.length
+    const rows = resp.rows
+    rowList = rowList.concat(rows)
+  } while (rowsLength === FIND_LIMIT)
+  // write back the updated attachments
+  return { rows: rowList, columns: attachmentCols }
+}

package/src/sdk/app/rows/index.ts

@@ -0,0 +1,7 @@
+import * as attachments from "./attachments"
+import * as rows from "./rows"
+
+export default {
+  ...attachments,
+  ...rows,
+}

package/src/sdk/app/rows/rows.ts

@@ -0,0 +1,18 @@
+import { db as dbCore, context } from "@budibase/backend-core"
+import { Database, Row } from "@budibase/types"
+import { getRowParams } from "../../../db/utils"
+
+export async function getAllInternalRows(appId?: string) {
+  let db: Database
+  if (appId) {
+    db = dbCore.getDB(appId)
+  } else {
+    db = context.getAppDB()
+  }
+  const response = await db.allDocs(
+    getRowParams(null, null, {
+      include_docs: true,
+    })
+  )
+  return response.rows.map(row => row.doc) as Row[]
+}

package/src/sdk/app/tables/index.ts

@@ -0,0 +1,63 @@
+import { context } from "@budibase/backend-core"
+import { BudibaseInternalDB, getTableParams } from "../../../db/utils"
+import {
+  breakExternalTableId,
+  isExternalTable,
+  isSQL,
+} from "../../../integrations/utils"
+import { Table, Database } from "@budibase/types"
+import datasources from "../datasources"
+
+async function getAllInternalTables(db?: Database): Promise<Table[]> {
+  if (!db) {
+    db = context.getAppDB()
+  }
+  const internalTables = await db.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+  return internalTables.rows.map((tableDoc: any) => ({
+    ...tableDoc.doc,
+    type: "internal",
+    sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id,
+  }))
+}
+
+async function getAllExternalTables(
+  datasourceId: any
+): Promise<Record<string, Table>> {
+  const db = context.getAppDB()
+  const datasource = await datasources.get(datasourceId, { enriched: true })
+  if (!datasource || !datasource.entities) {
+    throw "Datasource is not configured fully."
+  }
+  return datasource.entities
+}
+
+async function getExternalTable(
+  datasourceId: any,
+  tableName: any
+): Promise<Table> {
+  const entities = await getAllExternalTables(datasourceId)
+  return entities[tableName]
+}
+
+async function getTable(tableId: any): Promise<Table> {
+  const db = context.getAppDB()
+  if (isExternalTable(tableId)) {
+    let { datasourceId, tableName } = breakExternalTableId(tableId)
+    const datasource = await datasources.get(datasourceId!)
+    const table = await getExternalTable(datasourceId, tableName)
+    return { ...table, sql: isSQL(datasource) }
+  } else {
+    return db.get(tableId)
+  }
+}
+
+export default {
+  getAllInternalTables,
+  getAllExternalTables,
+  getExternalTable,
+  getTable,
+}

package/src/sdk/index.ts

@@ -0,0 +1,24 @@
+import { default as backups } from "./app/backups"
+import { default as tables } from "./app/tables"
+import { default as automations } from "./app/automations"
+import { default as applications } from "./app/applications"
+import { default as datasources } from "./app/datasources"
+import { default as queries } from "./app/queries"
+import { default as rows } from "./app/rows"
+import { default as users } from "./users"
+import { default as plugins } from "./plugins"
+
+const sdk = {
+  backups,
+  tables,
+  automations,
+  applications,
+  rows,
+  users,
+  datasources,
+  queries,
+  plugins,
+}
+
+// default export for TS
+export default sdk

package/src/sdk/plugins/index.ts

@@ -0,0 +1,5 @@
+import * as plugins from "./plugins"
+
+export default {
+  ...plugins,
+}

package/src/sdk/plugins/plugins.ts

@@ -0,0 +1,41 @@
+import { FileType, Plugin, PluginSource, PluginType } from "@budibase/types"
+import {
+  db as dbCore,
+  objectStore,
+  plugins as pluginCore,
+  tenancy,
+} from "@budibase/backend-core"
+import { fileUpload } from "../../api/controllers/plugin/file"
+import env from "../../environment"
+import { clientAppSocket } from "../../websockets"
+import { sdk as pro } from "@budibase/pro"
+
+export async function fetch(type?: PluginType) {
+  const db = tenancy.getGlobalDB()
+  const response = await db.allDocs(
+    dbCore.getPluginParams(null, {
+      include_docs: true,
+    })
+  )
+  let plugins = response.rows.map((row: any) => row.doc) as Plugin[]
+  plugins = objectStore.enrichPluginURLs(plugins)
+  if (type) {
+    return plugins.filter((plugin: Plugin) => plugin.schema?.type === type)
+  } else {
+    return plugins
+  }
+}
+
+export async function processUploaded(plugin: FileType, source?: PluginSource) {
+  const { metadata, directory } = await fileUpload(plugin)
+  pluginCore.validate(metadata?.schema)
+
+  // Only allow components in cloud
+  if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) {
+    throw new Error("Only component plugins are supported outside of self-host")
+  }
+
+  const doc = await pro.plugins.storePlugin(metadata, directory, source)
+  clientAppSocket.emit("plugin-update", { name: doc.name, hash: doc.hash })
+  return doc
+}

package/src/sdk/tests/attachments.spec.ts

@@ -0,0 +1,76 @@
+import newid from "../../db/newid"
+
+const attachment = {
+  size: 73479,
+  name: "2022-12-14 11_11_44-.png",
+  url: "/prod-budi-app-assets/app_bbb/attachments/a.png",
+  extension: "png",
+  key: "app_bbb/attachments/a.png",
+}
+
+const row = {
+  _id: "ro_ta_aaa",
+  photo: [attachment],
+  otherCol: "string",
+}
+
+const table = {
+  _id: "ta_aaa",
+  name: "photos",
+  schema: {
+    photo: {
+      type: "attachment",
+      name: "photo",
+    },
+    otherCol: {
+      type: "string",
+      name: "otherCol",
+    },
+  },
+}
+
+jest.mock("@budibase/backend-core", () => {
+  const core = jest.requireActual("@budibase/backend-core")
+  return {
+    ...core,
+    db: {
+      ...core.db,
+      directCouchFind: jest.fn(),
+    },
+  }
+})
+
+import { db as dbCore } from "@budibase/backend-core"
+import sdk from "../index"
+
+describe("should be able to re-write attachment URLs", () => {
+  it("it should update URLs on a number of rows over the limit", async () => {
+    const db = dbCore.getDB("app_aaa")
+    await db.put(table)
+    const limit = 30
+    let rows = []
+    for (let i = 0; i < limit; i++) {
+      const rowToWrite = {
+        ...row,
+        _id: `${row._id}_${newid()}`,
+      }
+      const { rev } = await db.put(rowToWrite)
+      rows.push({
+        ...rowToWrite,
+        _rev: rev,
+      })
+    }
+
+    dbCore.directCouchFind
+      // @ts-ignore
+      .mockReturnValueOnce({ rows: rows.slice(0, 25), bookmark: "aaa" })
+      .mockReturnValueOnce({ rows: rows.slice(25, limit), bookmark: "bbb" })
+    await sdk.backups.updateAttachmentColumns(db.name, db)
+    const finalRows = await sdk.rows.getAllInternalRows(db.name)
+    for (let rowToCheck of finalRows) {
+      expect(rowToCheck.otherCol).toBe(row.otherCol)
+      expect(rowToCheck.photo[0].url).toBe("")
+      expect(rowToCheck.photo[0].key).toBe(`${db.name}/attachments/a.png`)
+    }
+  })
+})

package/src/sdk/users/index.ts

@@ -0,0 +1,5 @@
+import * as utils from "./utils"
+
+export default {
+  ...utils,
+}

package/src/sdk/users/tests/utils.spec.ts

@@ -0,0 +1,128 @@
+import { db, roles } from "@budibase/backend-core"
+import { structures } from "@budibase/backend-core/tests"
+import { sdk as proSdk } from "@budibase/pro"
+
+import TestConfiguration from "../../../tests/utilities/TestConfiguration"
+import { rawUserMetadata, syncGlobalUsers } from "../utils"
+
+describe("syncGlobalUsers", () => {
+  const config = new TestConfiguration()
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)
+
+  it("the default user is synced", async () => {
+    await config.doInContext(config.appId, async () => {
+      await syncGlobalUsers()
+
+      const metadata = await rawUserMetadata()
+      expect(metadata).toHaveLength(1)
+      expect(metadata).toEqual([
+        expect.objectContaining({
+          _id: db.generateUserMetadataID(config.user._id),
+        }),
+      ])
+    })
+  })
+
+  it("admin and builders users are synced", async () => {
+    const user1 = await config.createUser({ admin: true })
+    const user2 = await config.createUser({ admin: false, builder: true })
+    await config.doInContext(config.appId, async () => {
+      expect(await rawUserMetadata()).toHaveLength(1)
+      await syncGlobalUsers()
+
+      const metadata = await rawUserMetadata()
+      expect(metadata).toHaveLength(3)
+      expect(metadata).toContainEqual(
+        expect.objectContaining({
+          _id: db.generateUserMetadataID(user1._id),
+        })
+      )
+      expect(metadata).toContainEqual(
+        expect.objectContaining({
+          _id: db.generateUserMetadataID(user2._id),
+        })
+      )
+    })
+  })
+
+  it("app users are not synced if not specified", async () => {
+    const user = await config.createUser({ admin: false, builder: false })
+    await config.doInContext(config.appId, async () => {
+      await syncGlobalUsers()
+
+      const metadata = await rawUserMetadata()
+      expect(metadata).toHaveLength(1)
+      expect(metadata).not.toContainEqual(
+        expect.objectContaining({
+          _id: db.generateUserMetadataID(user._id),
+        })
+      )
+    })
+  })
+
+  it("app users are added when group is assigned to app", async () => {
+    await config.doInTenant(async () => {
+      const group = await proSdk.groups.save(structures.userGroups.userGroup())
+      const user1 = await config.createUser({ admin: false, builder: false })
+      const user2 = await config.createUser({ admin: false, builder: false })
+      await proSdk.groups.addUsers(group.id, [user1._id, user2._id])
+
+      await config.doInContext(config.appId, async () => {
+        await syncGlobalUsers()
+        expect(await rawUserMetadata()).toHaveLength(1)
+
+        await proSdk.groups.updateGroupApps(group.id, {
+          appsToAdd: [
+            { appId: config.prodAppId!, roleId: roles.BUILTIN_ROLE_IDS.BASIC },
+          ],
+        })
+        await syncGlobalUsers()
+
+        const metadata = await rawUserMetadata()
+        expect(metadata).toHaveLength(3)
+        expect(metadata).toContainEqual(
+          expect.objectContaining({
+            _id: db.generateUserMetadataID(user1._id),
+          })
+        )
+        expect(metadata).toContainEqual(
+          expect.objectContaining({
+            _id: db.generateUserMetadataID(user2._id),
+          })
+        )
+      })
+    })
+  })
+
+  it("app users are removed when app is removed from user group", async () => {
+    await config.doInTenant(async () => {
+      const group = await proSdk.groups.save(structures.userGroups.userGroup())
+      const user1 = await config.createUser({ admin: false, builder: false })
+      const user2 = await config.createUser({ admin: false, builder: false })
+      await proSdk.groups.updateGroupApps(group.id, {
+        appsToAdd: [
+          { appId: config.prodAppId!, roleId: roles.BUILTIN_ROLE_IDS.BASIC },
+        ],
+      })
+      await proSdk.groups.addUsers(group.id, [user1._id, user2._id])
+
+      await config.doInContext(config.appId, async () => {
+        await syncGlobalUsers()
+        expect(await rawUserMetadata()).toHaveLength(3)
+
+        await proSdk.groups.updateGroupApps(group.id, {
+          appsToRemove: [{ appId: config.prodAppId! }],
+        })
+        await syncGlobalUsers()
+
+        const metadata = await rawUserMetadata()
+        expect(metadata).toHaveLength(0)
+      })
+    })
+  })
+})