@budibase/server 0.0.32 → 0.0.999-alpha.30

package/src/threads/query.ts

@@ -0,0 +1,305 @@
+import { default as threadUtils } from "./utils"
+threadUtils.threadSetup()
+import { WorkerCallback, QueryEvent, QueryVariable } from "./definitions"
+import ScriptRunner from "../utilities/scriptRunner"
+import { getIntegration } from "../integrations"
+import { processStringSync } from "@budibase/string-templates"
+import { context, cache, auth } from "@budibase/backend-core"
+import { getGlobalIDFromUserMetadataID } from "../db/utils"
+import sdk from "../sdk"
+import { cloneDeep } from "lodash/fp"
+
+import { isSQL } from "../integrations/utils"
+import { interpolateSQL } from "../integrations/queries/sql"
+
+class QueryRunner {
+  datasource: any
+  queryVerb: string
+  queryId: string
+  fields: any
+  parameters: any
+  pagination: any
+  transformer: any
+  cachedVariables: any[]
+  ctx: any
+  queryResponse: any
+  noRecursiveQuery: boolean
+  hasRerun: boolean
+  hasRefreshedOAuth: boolean
+  hasDynamicVariables: boolean
+
+  constructor(input: QueryEvent, flags = { noRecursiveQuery: false }) {
+    this.datasource = input.datasource
+    this.queryVerb = input.queryVerb
+    this.fields = input.fields
+    this.parameters = input.parameters
+    this.pagination = input.pagination
+    this.transformer = input.transformer
+    this.queryId = input.queryId
+    this.noRecursiveQuery = flags.noRecursiveQuery
+    this.cachedVariables = []
+    // Additional context items for enrichment
+    this.ctx = input.ctx
+    // allows the response from a query to be stored throughout this
+    // execution so that if it needs to be re-used for another variable
+    // it can be
+    this.queryResponse = {}
+    this.hasRerun = false
+    this.hasRefreshedOAuth = false
+    this.hasDynamicVariables = false
+  }
+
+  async execute(): Promise<any> {
+    let { datasource, fields, queryVerb, transformer } = this
+    let datasourceClone = cloneDeep(datasource)
+    let fieldsClone = cloneDeep(fields)
+
+    const Integration = await getIntegration(datasourceClone.source)
+    if (!Integration) {
+      throw "Integration type does not exist."
+    }
+
+    if (datasourceClone.config.authConfigs) {
+      const updatedConfigs = []
+      for (let config of datasourceClone.config.authConfigs) {
+        updatedConfigs.push(await sdk.queries.enrichContext(config, this.ctx))
+      }
+      datasourceClone.config.authConfigs = updatedConfigs
+    }
+
+    const integration = new Integration(datasourceClone.config)
+
+    // pre-query, make sure datasource variables are added to parameters
+    const parameters = await this.addDatasourceVariables()
+
+    // Enrich the parameters with the addition context items.
+    // 'user' is now a reserved variable key in mapping parameters
+    const enrichedParameters = await sdk.queries.enrichContext(
+      parameters,
+      this.ctx
+    )
+    const enrichedContext = { ...enrichedParameters, ...this.ctx }
+
+    // Parse global headers
+    if (datasourceClone.config.defaultHeaders) {
+      datasourceClone.config.defaultHeaders = await sdk.queries.enrichContext(
+        datasourceClone.config.defaultHeaders,
+        enrichedContext
+      )
+    }
+
+    let query
+    // handle SQL injections by interpolating the variables
+    if (isSQL(datasourceClone)) {
+      query = await interpolateSQL(fieldsClone, enrichedContext, integration)
+    } else {
+      query = await sdk.queries.enrichContext(fieldsClone, enrichedContext)
+    }
+
+    // Add pagination values for REST queries
+    if (this.pagination) {
+      query.paginationValues = this.pagination
+    }
+
+    let output = threadUtils.formatResponse(await integration[queryVerb](query))
+    let rows = output,
+      info = undefined,
+      extra = undefined,
+      pagination = undefined
+    if (threadUtils.hasExtraData(output)) {
+      rows = output.data
+      info = output.info
+      extra = output.extra
+      pagination = output.pagination
+    }
+
+    // transform as required
+    if (transformer) {
+      const runner = new ScriptRunner(transformer, {
+        data: rows,
+        params: enrichedParameters,
+      })
+      rows = runner.execute()
+    }
+
+    // if the request fails we retry once, invalidating the cached value
+    if (info && info.code >= 400 && !this.hasRerun) {
+      if (
+        this.ctx.user?.provider &&
+        info.code === 401 &&
+        !this.hasRefreshedOAuth
+      ) {
+        await this.refreshOAuth2(this.ctx)
+        // Attempt to refresh the access token from the provider
+        this.hasRefreshedOAuth = true
+      } else {
+        this.hasRerun = true
+      }
+
+      await threadUtils.invalidateDynamicVariables(this.cachedVariables)
+      return this.execute()
+    }
+
+    // check for undefined response
+    if (!rows) {
+      rows = []
+    }
+
+    // needs to an array for next step
+    if (!Array.isArray(rows)) {
+      rows = [rows]
+    }
+
+    // map into JSON if just raw primitive here
+    if (rows.find((row: any) => typeof row !== "object")) {
+      rows = rows.map((value: any) => ({ value }))
+    }
+
+    // get all the potential fields in the schema
+    let keys = rows.flatMap(Object.keys)
+
+    if (integration.end) {
+      integration.end()
+    }
+
+    return { rows, keys, info, extra, pagination }
+  }
+
+  async runAnotherQuery(queryId: string, parameters: any) {
+    const db = context.getAppDB()
+    const query = await db.get(queryId)
+    const datasource = await sdk.datasources.get(query.datasourceId, {
+      enriched: true,
+    })
+    return new QueryRunner(
+      {
+        datasource,
+        queryVerb: query.queryVerb,
+        fields: query.fields,
+        parameters,
+        transformer: query.transformer,
+        queryId,
+        ctx: this.ctx,
+      },
+      { noRecursiveQuery: true }
+    ).execute()
+  }
+
+  async refreshOAuth2(ctx: any) {
+    const { oauth2, providerType, _id } = ctx.user
+    const { configId } = ctx.auth
+
+    if (!providerType || !oauth2?.refreshToken) {
+      throw new Error("No refresh token found for authenticated user")
+    }
+
+    const resp: any = await auth.refreshOAuthToken(
+      oauth2.refreshToken,
+      providerType,
+      configId
+    )
+
+    // Refresh session flow. Should be in same location as refreshOAuthToken
+    // There are several other properties available in 'resp'
+    if (!resp.err) {
+      const globalUserId = getGlobalIDFromUserMetadataID(_id)
+      await auth.updateUserOAuth(globalUserId, resp)
+      this.ctx.user = await cache.user.getUser(globalUserId)
+    } else {
+      // In this event the user may have oAuth issues that
+      // could require re-authenticating with their provider.
+      let errorMessage = resp.err.data ? resp.err.data : resp.err.toString()
+      throw new Error(
+        "OAuth2 access token could not be refreshed: " + errorMessage
+      )
+    }
+
+    return resp
+  }
+
+  async getDynamicVariable(variable: QueryVariable) {
+    let { parameters } = this
+    const queryId = variable.queryId,
+      name = variable.name
+    let value = await threadUtils.checkCacheForDynamicVariable(queryId, name)
+    if (!value) {
+      value = this.queryResponse[queryId]
+        ? this.queryResponse[queryId]
+        : await this.runAnotherQuery(queryId, parameters)
+      // store incase this query is to be called again
+      this.queryResponse[queryId] = value
+      await threadUtils.storeDynamicVariable(queryId, name, value)
+    } else {
+      this.cachedVariables.push({ queryId, name })
+    }
+    return value
+  }
+
+  async addDatasourceVariables() {
+    let { datasource, parameters, fields } = this
+    if (!datasource || !datasource.config) {
+      return parameters
+    }
+    const staticVars = datasource.config.staticVariables || {}
+    const dynamicVars = datasource.config.dynamicVariables || []
+    for (let [key, value] of Object.entries(staticVars)) {
+      if (!parameters[key]) {
+        parameters[key] = value
+      }
+    }
+    if (!this.noRecursiveQuery) {
+      // need to see if this uses any variables
+      const stringFields = JSON.stringify(fields)
+      const foundVars = dynamicVars.filter((variable: QueryVariable) => {
+        // don't allow a query to use its own dynamic variable (loop)
+        if (variable.queryId === this.queryId) {
+          return false
+        }
+        // look for {{ variable }} but allow spaces between handlebars
+        const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`)
+        return regex.test(stringFields)
+      })
+      const dynamics = foundVars.map((dynVar: QueryVariable) =>
+        this.getDynamicVariable(dynVar)
+      )
+      const responses = await Promise.all(dynamics)
+      for (let i = 0; i < foundVars.length; i++) {
+        const variable = foundVars[i]
+        parameters[variable.name] = processStringSync(
+          variable.value,
+          {
+            data: responses[i].rows,
+            info: responses[i].extra,
+          },
+          {
+            escapeNewlines: true,
+          }
+        )
+        // make sure its known that this uses dynamic variables in case it fails
+        this.hasDynamicVariables = true
+      }
+    }
+    return parameters
+  }
+}
+
+export function execute(input: QueryEvent, callback: WorkerCallback) {
+  const run = async () => {
+    const Runner = new QueryRunner(input)
+    try {
+      const response = await Runner.execute()
+      callback(null, response)
+    } catch (err) {
+      callback(err)
+    }
+  }
+  context.doInAppContext(input.appId!, async () => {
+    if (input.environmentVariables) {
+      return context.doInEnvironmentContext(input.environmentVariables, () => {
+        return run()
+      })
+    } else {
+      return run()
+    }
+  })
+}

package/src/threads/utils.ts

@@ -0,0 +1,96 @@
+import { QueryVariable } from "./definitions"
+import env from "../environment"
+import * as db from "../db"
+import { redis, db as dbCore } from "@budibase/backend-core"
+
+const VARIABLE_TTL_SECONDS = 3600
+let client: any
+
+async function getClient() {
+  if (!client) {
+    client = await new redis.Client(redis.utils.Databases.QUERY_VARS).init()
+  }
+  return client
+}
+
+process.on("exit", async () => {
+  if (client) {
+    await client.finish()
+  }
+})
+
+function makeVariableKey(queryId: string, variable: string) {
+  return `${queryId}${dbCore.SEPARATOR}${variable}`
+}
+
+export function threadSetup() {
+  // don't run this if not threading
+  if (env.isTest() || env.DISABLE_THREADING) {
+    return
+  }
+  // when thread starts, make sure it is recorded
+  env.setInThread()
+  db.init()
+}
+
+export async function checkCacheForDynamicVariable(
+  queryId: string,
+  variable: string
+) {
+  const cache = await getClient()
+  return cache.get(makeVariableKey(queryId, variable))
+}
+
+export async function invalidateDynamicVariables(cachedVars: QueryVariable[]) {
+  const cache = await getClient()
+  let promises = []
+  for (let variable of cachedVars) {
+    promises.push(
+      cache.delete(makeVariableKey(variable.queryId, variable.name))
+    )
+  }
+  await Promise.all(promises)
+}
+
+export async function storeDynamicVariable(
+  queryId: string,
+  variable: string,
+  value: any
+) {
+  const cache = await getClient()
+  await cache.store(
+    makeVariableKey(queryId, variable),
+    value,
+    VARIABLE_TTL_SECONDS
+  )
+}
+
+export function formatResponse(resp: any) {
+  if (typeof resp === "string") {
+    try {
+      resp = JSON.parse(resp)
+    } catch (err) {
+      resp = { response: resp }
+    }
+  }
+  return resp
+}
+
+export function hasExtraData(response: any) {
+  return (
+    typeof response === "object" &&
+    !Array.isArray(response) &&
+    response &&
+    response.data != null &&
+    response.info != null
+  )
+}
+
+export default {
+  hasExtraData,
+  formatResponse,
+  storeDynamicVariable,
+  invalidateDynamicVariables,
+  checkCacheForDynamicVariable,
+  threadSetup,
+}

package/src/utilities/appDirectoryTemplate/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "{{ name }}",
+  "version": "1.0.0",
+  "description": "",
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@budibase/client": "0.x"
+  }
+}

package/src/utilities/budibaseDir.ts

@@ -0,0 +1,3 @@
+import { objectStore } from "@budibase/backend-core"
+
+export const budibaseTempDir = objectStore.budibaseTempDir

package/src/utilities/centralPath.ts

@@ -0,0 +1,22 @@
+import path from "path"
+
+// this simply runs all of our path join and resolve functions through
+// a central location incase we need to add some protection to file paths
+
+/**
+ * Exactly the same as path.join
+ * @param args Any number of string arguments to add to a path
+ * @returns {string} The final path ready to use
+ */
+export function join(...args: any) {
+  return path.join(...args)
+}
+
+/**
+ * Exactly the same as path.resolve
+ * @param args Any number of string arguments to add to a path
+ * @returns {string} The final path ready to use
+ */
+export function resolve(...args: any) {
+  return path.resolve(...args)
+}

package/src/utilities/fileSystem/app.ts

@@ -0,0 +1,79 @@
+import { budibaseTempDir } from "../budibaseDir"
+import fs from "fs"
+import { join } from "path"
+import { ObjectStoreBuckets } from "../../constants"
+import { updateClientLibrary } from "./clientLibrary"
+import env from "../../environment"
+import { objectStore, context } from "@budibase/backend-core"
+import { TOP_LEVEL_PATH } from "./filesystem"
+
+export const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
+
+/**
+ * Uploads the latest client library to the object store.
+ * @param {string} appId The ID of the app which is being created.
+ * @return {Promise<void>} once promise completes app resources should be ready in object store.
+ */
+export const createApp = async (appId: string) => {
+  await updateClientLibrary(appId)
+}
+
+/**
+ * Removes all of the assets created for an app in the object store.
+ * @param {string} appId The ID of the app which is being deleted.
+ * @return {Promise<void>} once promise completes the app resources will be removed from object store.
+ */
+export const deleteApp = async (appId: string) => {
+  await objectStore.deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`)
+}
+
+/**
+ * Retrieves component libraries from object store (or tmp symlink if in local)
+ */
+export const getComponentLibraryManifest = async (library: string) => {
+  const appId = context.getAppId()
+  const filename = "manifest.json"
+
+  if (env.isDev() || env.isTest()) {
+    const path = join(NODE_MODULES_PATH, "@budibase", "client", filename)
+    // always load from new so that updates are refreshed
+    delete require.cache[require.resolve(path)]
+    return require(path)
+  }
+
+  if (!appId) {
+    throw new Error("No app ID found - cannot get component libraries")
+  }
+
+  let resp
+  let path
+  try {
+    // Try to load the manifest from the new file location
+    path = join(appId, filename)
+    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
+  } catch (error) {
+    console.error(
+      `component-manifest-objectstore=failed appId=${appId} path=${path}`,
+      error
+    )
+    // Fallback to loading it from the old location for old apps
+    path = join(appId, "node_modules", library, "package", filename)
+    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
+  }
+  if (typeof resp !== "string") {
+    resp = resp.toString("utf8")
+  }
+  return JSON.parse(resp)
+}
+
+/**
+ * Given a set of app IDs makes sure file system is cleared of any of their temp info.
+ */
+export const cleanup = (appIds: string[]) => {
+  for (let appId of appIds) {
+    const path = join(budibaseTempDir(), appId)
+    if (fs.existsSync(path)) {
+      fs.rmdirSync(path, { recursive: true })
+    }
+  }
+}

package/src/utilities/fileSystem/clientLibrary.ts

@@ -0,0 +1,154 @@
+import { join } from "path"
+import { ObjectStoreBuckets } from "../../constants"
+import fs from "fs"
+import { objectStore } from "@budibase/backend-core"
+import { resolve } from "../centralPath"
+import env from "../../environment"
+import { TOP_LEVEL_PATH } from "./filesystem"
+
+/**
+ * Client library paths in the object store:
+ * Previously, the entire client library package was downloaded from NPM
+ * as a tarball and extracted to the object store, even though only the manifest
+ * was ever needed. Therefore we need to support old apps which may still have
+ * the manifest at this location for the first update.
+ *
+ * The new paths for the in-use version are:
+ * {appId}/manifest.json
+ * {appId}/budibase-client.js
+ *
+ * The paths for the backups are:
+ * {appId}/manifest.json.bak
+ * {appId}/budibase-client.js.bak
+ *
+ * We don't rely on NPM at all any more, as when updating to the latest version
+ * we pull both the manifest and client bundle from the server's dependencies
+ * in the local file system.
+ */
+
+/**
+ * Backs up the current client library version by copying both the manifest
+ * and client bundle to .bak extensions in the object store. Only the one
+ * previous version is stored as a backup, which can be reverted to.
+ * @param appId The app ID to backup
+ * @returns {Promise<void>}
+ */
+export async function backupClientLibrary(appId: string) {
+  // Copy existing manifest to tmp
+  let tmpManifestPath
+  try {
+    // Try to load the manifest from the new file location
+    tmpManifestPath = await objectStore.retrieveToTmp(
+      ObjectStoreBuckets.APPS,
+      join(appId, "manifest.json")
+    )
+  } catch (error) {
+    // Fallback to loading it from the old location for old apps
+    tmpManifestPath = await objectStore.retrieveToTmp(
+      ObjectStoreBuckets.APPS,
+      join(
+        appId,
+        "node_modules",
+        "budibase",
+        "standard-components",
+        "package",
+        "manifest.json"
+      )
+    )
+  }
+
+  // Copy existing client lib to tmp
+  const tmpClientPath = await objectStore.retrieveToTmp(
+    ObjectStoreBuckets.APPS,
+    join(appId, "budibase-client.js")
+  )
+
+  // Upload manifest and client library as backups
+  const manifestUpload = objectStore.upload({
+    bucket: ObjectStoreBuckets.APPS,
+    filename: join(appId, "manifest.json.bak"),
+    path: tmpManifestPath,
+    type: "application/json",
+  })
+  const clientUpload = objectStore.upload({
+    bucket: ObjectStoreBuckets.APPS,
+    filename: join(appId, "budibase-client.js.bak"),
+    path: tmpClientPath,
+    type: "application/javascript",
+  })
+  await Promise.all([manifestUpload, clientUpload])
+}
+
+/**
+ * Uploads the latest version of the component manifest and the client library
+ * to the object store, overwriting the existing version.
+ * @param appId The app ID to update
+ * @returns {Promise<void>}
+ */
+export async function updateClientLibrary(appId: string) {
+  let manifest, client
+
+  if (env.isDev()) {
+    // Load the symlinked version in dev which is always the newest
+    manifest = require.resolve("@budibase/client/manifest.json")
+    client = require.resolve("@budibase/client")
+  } else {
+    // Load the bundled version in prod
+    manifest = resolve(TOP_LEVEL_PATH, "client", "manifest.json")
+    client = resolve(TOP_LEVEL_PATH, "client", "budibase-client.js")
+  }
+
+  // Upload latest manifest and client library
+  const manifestUpload = objectStore.streamUpload(
+    ObjectStoreBuckets.APPS,
+    join(appId, "manifest.json"),
+    fs.createReadStream(manifest),
+    {
+      ContentType: "application/json",
+    }
+  )
+  const clientUpload = objectStore.streamUpload(
+    ObjectStoreBuckets.APPS,
+    join(appId, "budibase-client.js"),
+    fs.createReadStream(client),
+    {
+      ContentType: "application/javascript",
+    }
+  )
+  await Promise.all([manifestUpload, clientUpload])
+}
+
+/**
+ * Reverts the version of the client library and manifest to the previously
+ * used version for an app.
+ * @param appId The app ID to revert
+ * @returns {Promise<void>}
+ */
+export async function revertClientLibrary(appId: string) {
+  // Copy backups manifest to tmp directory
+  const tmpManifestPath = await objectStore.retrieveToTmp(
+    ObjectStoreBuckets.APPS,
+    join(appId, "manifest.json.bak")
+  )
+
+  // Copy backup client lib to tmp
+  const tmpClientPath = await objectStore.retrieveToTmp(
+    ObjectStoreBuckets.APPS,
+    join(appId, "budibase-client.js.bak")
+  )
+
+  // Upload backups as new versions
+  const manifestUpload = objectStore.upload({
+    bucket: ObjectStoreBuckets.APPS,
+    filename: join(appId, "manifest.json"),
+    path: tmpManifestPath,
+    type: "application/json",
+  })
+  const clientUpload = objectStore.upload({
+    bucket: ObjectStoreBuckets.APPS,
+    filename: join(appId, "budibase-client.js"),
+    path: tmpClientPath,
+    type: "application/javascript",
+  })
+  await Promise.all([manifestUpload, clientUpload])
+}