@budibase/server 1.0.27-alpha.10 → 1.0.27-alpha.14

package/src/api/routes/script.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/script")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/static.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionTypes,
   PermissionLevels,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 const env = require("../../environment")
 const { paramResource } = require("../../middleware/resourceId")
 

package/src/api/routes/table.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 const joiValidator = require("../../middleware/joi-validator")
 const Joi = require("joi")
 

package/src/api/routes/templates.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/templates")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/tests/datasource.spec.js

@@ -4,6 +4,7 @@ let setup = require("./utilities")
 let { basicDatasource } = setup.structures
 let { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const pg = require("pg")
+const { checkCacheForDynamicVariable } = require("../../../threads/utils")
 
 describe("/datasources", () => {
   let request = setup.getRequest()
@@ -31,6 +32,50 @@ describe("/datasources", () => {
     })
   })
 
+  describe("update", () => {
+    it("should update an existing datasource", async () => {
+      datasource.name = "Updated Test"
+      const res = await request
+        .put(`/api/datasources/${datasource._id}`)
+        .send(datasource)
+        .set(config.defaultHeaders())
+        .expect('Content-Type', /json/)
+        .expect(200)
+
+      expect(res.body.datasource.name).toEqual("Updated Test")
+      expect(res.body.errors).toBeUndefined()
+    })
+
+    describe("dynamic variables", () => {
+      async function preview(datasource, fields) {
+        return config.previewQuery(request, config, datasource, fields)
+      }
+
+      it("should invalidate changed or removed variables", async () => {
+        const { datasource, query } = await config.dynamicVariableDatasource()
+        // preview once to cache variables
+        await preview(datasource, { path: "www.test.com", queryString: "test={{ variable3 }}" })
+        // check variables in cache
+        let contents = await checkCacheForDynamicVariable(query._id, "variable3")
+        expect(contents.rows.length).toEqual(1)
+        
+        // update the datasource to remove the variables
+        datasource.config.dynamicVariables = []
+        const res = await request
+          .put(`/api/datasources/${datasource._id}`)
+          .send(datasource)
+          .set(config.defaultHeaders())
+          .expect('Content-Type', /json/)
+          .expect(200)
+        expect(res.body.errors).toBeUndefined()
+
+        // check variables no longer in cache
+        contents = await checkCacheForDynamicVariable(query._id, "variable3")
+        expect(contents).toBe(null)
+      })
+    })
+  })
+
   describe("fetch", () => {
     it("returns all the datasources from the server", async () => {
       const res = await request

package/src/api/routes/tests/permissions.spec.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const setup = require("./utilities")
 const { basicRow } = setup.structures
 

package/src/api/routes/tests/query.spec.js

@@ -4,7 +4,7 @@ jest.mock("node-fetch")
 
 // Mock isProdAppID to we can later mock the implementation and pretend we are
 // using prod app IDs
-const authDb = require("@budibase/auth/db")
+const authDb = require("@budibase/backend-core/db")
 const { isProdAppID } = authDb
 const mockIsProdAppID = jest.fn(isProdAppID)
 authDb.isProdAppID = mockIsProdAppID
@@ -229,52 +229,14 @@ describe("/queries", () => {
     })
   })
 
-  describe("test variables", () => {
-    async function restDatasource(cfg) {
-      return await config.createDatasource({
-        datasource: {
-          ...basicDatasource().datasource,
-          source: "REST",
-          config: cfg || {},
-        },
-      })
-    }
-
-    async function dynamicVariableDatasource() {
-      const datasource = await restDatasource()
-      const basedOnQuery = await config.createQuery({
-        ...basicQuery(datasource._id),
-        fields: {
-          path: "www.google.com",
-        },
-      })
-      await config.updateDatasource({
-        ...datasource,
-        config: {
-          dynamicVariables: [
-            { queryId: basedOnQuery._id, name: "variable3", value: "{{ data.0.[value] }}" }
-          ]
-        }
-      })
-      return { datasource, query: basedOnQuery }
-    }
+  describe("variables", () => {
 
     async function preview(datasource, fields) {
-      return await request
-        .post(`/api/queries/preview`)
-        .send({
-          datasourceId: datasource._id,
-          parameters: {},
-          fields,
-          queryVerb: "read",
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      return config.previewQuery(request, config, datasource, fields)
     }
 
     it("should work with static variables", async () => {
-      const datasource = await restDatasource({
+      const datasource = await config.restDatasource({
         staticVariables: {
           variable: "google",
           variable2: "1",
@@ -290,7 +252,7 @@ describe("/queries", () => {
     })
 
     it("should work with dynamic variables", async () => {
-      const { datasource } = await dynamicVariableDatasource()
+      const { datasource } = await config.dynamicVariableDatasource()
       const res = await preview(datasource, {
         path: "www.google.com",
         queryString: "test={{ variable3 }}",
@@ -300,7 +262,7 @@ describe("/queries", () => {
     })
 
     it("check that it automatically retries on fail with cached dynamics", async () => {
-      const { datasource, query: base } = await dynamicVariableDatasource()
+      const { datasource, query: base } = await config.dynamicVariableDatasource()
       // preview once to cache
       await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
       // check its in cache
@@ -313,5 +275,24 @@ describe("/queries", () => {
       expect(res.body.schemaFields).toEqual(["fails", "url", "opts"])
       expect(res.body.rows[0].fails).toEqual(1)
     })
+
+    it("deletes variables when linked query is deleted", async () => {
+      const { datasource, query: base } = await config.dynamicVariableDatasource()
+      // preview once to cache
+      await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
+      // check its in cache
+      let contents = await checkCacheForDynamicVariable(base._id, "variable3")
+      expect(contents.rows.length).toEqual(1)
+
+      // delete the query
+      await request
+        .delete(`/api/queries/${base._id}/${base._rev}`)
+        .set(config.defaultHeaders())
+        .expect(200)
+
+      // check variables no longer in cache
+      contents = await checkCacheForDynamicVariable(base._id, "variable3")
+      expect(contents).toBe(null)
+    })
   })
 })

package/src/api/routes/tests/role.spec.js

@@ -1,7 +1,7 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const {
   BUILTIN_PERMISSION_IDS,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 const setup = require("./utilities")
 const { basicRole } = setup.structures
 

package/src/api/routes/tests/routing.spec.js

@@ -1,7 +1,7 @@
 const setup = require("./utilities")
 const { basicScreen } = setup.structures
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 
 const route = "/test"
 

package/src/api/routes/tests/user.spec.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { checkPermissionsEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
 

package/src/api/routes/tests/utilities/TestFunctions.js

@@ -2,7 +2,7 @@ const rowController = require("../../../controllers/row")
 const appController = require("../../../controllers/application")
 const CouchDB = require("../../../../db")
 const { AppStatus } = require("../../../../db/utils")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { TENANT_ID } = require("../../../../tests/utilities/structures")
 
 function Request(appId, params) {

package/src/api/routes/user.js

@@ -4,7 +4,7 @@ const authorized = require("../../middleware/authorized")
 const {
   PermissionLevels,
   PermissionTypes,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/view.js

@@ -7,7 +7,7 @@ const {
   BUILDER,
   PermissionTypes,
   PermissionLevels,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

package/src/api/routes/webhook.js

@@ -2,7 +2,7 @@ const Router = require("@koa/router")
 const controller = require("../controllers/webhook")
 const authorized = require("../../middleware/authorized")
 const joiValidator = require("../../middleware/joi-validator")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 const Joi = require("joi")
 
 const router = Router()

package/src/app.ts

@@ -3,7 +3,7 @@ import { ExtendableContext } from "koa"
 
 const env = require("./environment")
 const CouchDB = require("./db")
-require("@budibase/auth").init(CouchDB)
+require("@budibase/backend-core").init(CouchDB)
 const Koa = require("koa")
 const destroyable = require("server-destroy")
 const koaBody = require("koa-body")

package/src/automations/bullboard.js

@@ -6,7 +6,7 @@ const Queue = env.isTest()
   ? require("../utilities/queue/inMemoryQueue")
   : require("bull")
 const { JobQueues } = require("../constants")
-const { utils } = require("@budibase/auth/redis")
+const { utils } = require("@budibase/backend-core/redis")
 const { opts, redisProtocolUrl } = utils.getRedisOptions()
 
 const CLEANUP_PERIOD_MS = 60 * 1000

package/src/automations/steps/executeQuery.js

@@ -72,7 +72,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   })
 
   try {
-    await queryController.execute(ctx)
+    await queryController.executeV1(ctx)
     const { data, ...rest } = ctx.body
     return {
       response: data,

package/src/automations/utils.js

@@ -6,7 +6,7 @@ const { queue } = require("./bullboard")
 const newid = require("../db/newid")
 const { updateEntityMetadata } = require("../utilities")
 const { MetadataTypes } = require("../constants")
-const { getDeployedAppID } = require("@budibase/auth/db")
+const { getDeployedAppID } = require("@budibase/backend-core/db")
 const { cloneDeep } = require("lodash/fp")
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId

package/src/constants/index.js

@@ -1,6 +1,6 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
-const { UserStatus } = require("@budibase/auth").constants
-const { ObjectStoreBuckets } = require("@budibase/auth").objectStore
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const { UserStatus } = require("@budibase/backend-core/constants")
+const { ObjectStoreBuckets } = require("@budibase/backend-core/objectStore")
 
 exports.JobQueues = {
   AUTOMATIONS: "automationQueue",

package/src/constants/screens.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { BASE_LAYOUT_PROP_IDS } = require("./layouts")
 
 exports.createHomeScreen = () => ({

package/src/db/client.js

@@ -1,5 +1,5 @@
 const PouchDB = require("pouchdb")
-const { getCouchUrl } = require("@budibase/auth/db")
+const { getCouchUrl } = require("@budibase/backend-core/db")
 const replicationStream = require("pouchdb-replication-stream")
 const allDbs = require("pouchdb-all-dbs")
 const find = require("pouchdb-find")

package/src/db/utils.js

@@ -9,7 +9,7 @@ const {
   StaticDatabases,
   isDevAppID,
   isProdAppID,
-} = require("@budibase/auth/db")
+} = require("@budibase/backend-core/db")
 
 const UNICODE_MAX = "\ufff0"
 

package/src/definitions/datasource.ts

@@ -232,6 +232,8 @@ export interface RestQueryFields {
   json: object
   method: string
   authConfigId: string
+  pagination: PaginationConfig | null
+  paginationValues: PaginationValues | null
 }
 
 export interface RestConfig {
@@ -252,6 +254,19 @@ export interface RestConfig {
   ]
 }
 
+export interface PaginationConfig {
+  type: string
+  location: string
+  pageParam: string
+  sizeParam: string | null
+  responseParam: string | null
+}
+
+export interface PaginationValues {
+  page: string | number | null
+  limit: number | null
+}
+
 export interface Query {
   _id?: string
   datasourceId: string

package/src/environment.js

@@ -65,6 +65,7 @@ module.exports = {
   DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,
+  QUERY_THREAD_TIMEOUT: process.env.QUERY_THREAD_TIMEOUT,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value

package/src/integrations/dynamodb.ts

@@ -80,6 +80,17 @@ module DynamoModule {
           },
         },
       },
+      describe: {
+        type: QueryTypes.FIELDS,
+        customisable: true,
+        readable: true,
+        fields: {
+          table: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+        },
+      },
       get: {
         type: QueryTypes.FIELDS,
         customisable: true,
@@ -180,6 +191,13 @@ module DynamoModule {
       return response
     }
 
+    async describe(query: { table: string }) {
+      const params = {
+        TableName: query.table,
+      }
+      return new AWS.DynamoDB().describeTable(params).promise()
+    }
+
     async get(query: { table: string; json: object }) {
       const params = {
         TableName: query.table,

package/src/integrations/rest.ts

@@ -4,9 +4,11 @@ import {
   QueryTypes,
   RestConfig,
   RestQueryFields as RestQuery,
+  PaginationConfig,
   AuthType,
   BasicAuthConfig,
   BearerAuthConfig,
+  PaginationValues,
 } from "../definitions/datasource"
 import { IntegrationBase } from "./base/IntegrationBase"
 
@@ -40,6 +42,9 @@ const coreFields = {
     type: DatasourceFieldTypes.STRING,
     enum: Object.values(BodyTypes),
   },
+  pagination: {
+    type: DatasourceFieldTypes.OBJECT
+  }
 }
 
 module RestModule {
@@ -115,7 +120,7 @@ module RestModule {
       this.config = config
     }
 
-    async parseResponse(response: any) {
+    async parseResponse(response: any, pagination: PaginationConfig | null) {
       let data, raw, headers
       const contentType = response.headers.get("content-type") || ""
       try {
@@ -154,6 +159,13 @@ module RestModule {
       for (let [key, value] of Object.entries(headers)) {
         headers[key] = Array.isArray(value) ? value[0] : value
       }
+
+      // Check if a pagination cursor exists in the response
+      let nextCursor = null
+      if (pagination?.responseParam) {
+        nextCursor = data?.[pagination.responseParam]
+      }
+
       return {
         data,
         info: {
@@ -165,10 +177,35 @@ module RestModule {
           raw,
           headers,
         },
+        pagination: {
+          cursor: nextCursor
+        }
       }
     }
 
-    getUrl(path: string, queryString: string): string {
+    getUrl(path: string, queryString: string, pagination: PaginationConfig | null, paginationValues: PaginationValues | null): string {
+      // Add pagination params to query string if required
+      if (pagination?.location === "query" && paginationValues) {
+        const { pageParam, sizeParam } = pagination
+        const params = new URLSearchParams()
+
+        // Append page number or cursor param if configured
+        if (pageParam && paginationValues.page != null) {
+          params.append(pageParam, paginationValues.page)
+        }
+
+        // Append page size param if configured
+        if (sizeParam && paginationValues.limit != null) {
+          params.append(sizeParam, paginationValues.limit)
+        }
+
+        // Prepend query string with pagination params
+        let paginationString = params.toString()
+        if (paginationString) {
+          queryString = `${paginationString}&${queryString}`
+        }
+      }
+
       const main = `${path}?${queryString}`
       let complete = main
       if (this.config.url && !main.startsWith("http")) {
@@ -180,20 +217,36 @@ module RestModule {
       return complete
     }
 
-    addBody(bodyType: string, body: string | any, input: any) {
-      let error, object, string
+    addBody(bodyType: string, body: string | any, input: any, pagination: PaginationConfig | null, paginationValues: PaginationValues | null) {
+      if (!input.headers) {
+        input.headers = {}
+      }
+      if (bodyType === BodyTypes.NONE) {
+        return input
+      }
+      let error, object: any = {}, string = ""
       try {
-        string = typeof body !== "string" ? JSON.stringify(body) : body
-        object = typeof body === "object" ? body : JSON.parse(body)
+        if (body) {
+          string = typeof body !== "string" ? JSON.stringify(body) : body
+          object = typeof body === "object" ? body : JSON.parse(body)
+        }
       } catch (err) {
         error = err
       }
-      if (!input.headers) {
-        input.headers = {}
+
+      // Util to add pagination values to a certain body type
+      const addPaginationToBody = (insertFn: Function) => {
+        if (pagination?.location === "body") {
+          if (pagination?.pageParam && paginationValues?.page != null) {
+            insertFn(pagination.pageParam, paginationValues.page)
+          }
+          if (pagination?.sizeParam && paginationValues?.limit != null) {
+            insertFn(pagination.sizeParam, paginationValues.limit)
+          }
+        }
       }
+
       switch (bodyType) {
-        case BodyTypes.NONE:
-          break
         case BodyTypes.TEXT:
           // content type defaults to plaintext
           input.body = string
@@ -203,6 +256,9 @@ module RestModule {
           for (let [key, value] of Object.entries(object)) {
             params.append(key, value)
           }
+          addPaginationToBody((key: string, value: any) => {
+            params.append(key, value)
+          })
           input.body = params
           break
         case BodyTypes.FORM_DATA:
@@ -210,6 +266,9 @@ module RestModule {
           for (let [key, value] of Object.entries(object)) {
             form.append(key, value)
           }
+          addPaginationToBody((key: string, value: any) => {
+            form.append(key, value)
+          })
           input.body = form
           break
         case BodyTypes.XML:
@@ -219,13 +278,15 @@ module RestModule {
           input.body = string
           input.headers["Content-Type"] = "application/xml"
           break
-        default:
         case BodyTypes.JSON:
           // if JSON error, throw it
           if (error) {
             throw "Invalid JSON for request body"
           }
-          input.body = string
+          addPaginationToBody((key: string, value: any) => {
+            object[key] = value
+          })
+          input.body = JSON.stringify(object)
           input.headers["Content-Type"] = "application/json"
           break
       }
@@ -271,6 +332,8 @@ module RestModule {
         bodyType,
         requestBody,
         authConfigId,
+        pagination,
+        paginationValues
       } = query
       const authHeaders = this.getAuthHeaders(authConfigId)
 
@@ -289,14 +352,12 @@ module RestModule {
       }
 
       let input: any = { method, headers: this.headers }
-      if (requestBody) {
-        input = this.addBody(bodyType, requestBody, input)
-      }
+      input = this.addBody(bodyType, requestBody, input, pagination, paginationValues)
 
       this.startTimeMs = performance.now()
-      const url = this.getUrl(path, queryString)
+      const url = this.getUrl(path, queryString, pagination, paginationValues)
       const response = await fetch(url, input)
-      return await this.parseResponse(response)
+      return await this.parseResponse(response, pagination)
     }
 
     async create(opts: RestQuery) {