@budibase/server 1.0.49-alpha.7 → 1.0.50-alpha.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.49-alpha.7",
+  "version": "1.0.50-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.49-alpha.7",
-    "@budibase/client": "^1.0.49-alpha.7",
-    "@budibase/string-templates": "^1.0.49-alpha.7",
+    "@budibase/backend-core": "^1.0.50-alpha.0",
+    "@budibase/client": "^1.0.50-alpha.0",
+    "@budibase/string-templates": "^1.0.50-alpha.0",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -168,5 +168,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "c7d23bbf7e30545a20c9452d720b7078dafb9947"
+  "gitHead": "a076d7ccf1deab0d300ae229a1aa3f4ce92aa1ba"
 }

package/scripts/integrations/mssql/data/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/mssql/server
+FROM mcr.microsoft.com/mssql/server:2017-latest
 
 ENV ACCEPT_EULA=Y
 ENV SA_PASSWORD=Passw0rd

package/scripts/integrations/mssql/data/setup.sql

@@ -48,7 +48,20 @@ INSERT tasks
 VALUES
   ('Processing', 1);
 
-INSERT people
-  (name, age)
-VALUES
-  ('Bob', '30');
+INSERT INTO people (name, age)
+VALUES ('Bob', '30'),
+  ('Bert', '10'),
+  ('Jack', '12'),
+  ('Mike', '31'),
+  ('Dave', '44'),
+  ('Jim', '43'),
+  ('Kerry', '32'),
+  ('Julie', '12'),
+  ('Kim', '55'),
+  ('Andy', '33'),
+  ('John', '22'),
+  ('Ruth', '66'),
+  ('Robert', '88'),
+  ('Bobert', '99'),
+  ('Jan', '22'),
+  ('Megan', '11');

package/scripts/integrations/postgres/init.sql

@@ -9,12 +9,16 @@ CREATE TABLE Persons (
 );
 CREATE TABLE Tasks (
     TaskID SERIAL PRIMARY KEY,
-    PersonID INT,
+    ExecutorID INT,
+    QaID INT,
     Completed BOOLEAN,
     TaskName varchar(255),
-    CONSTRAINT fkPersons
-        FOREIGN KEY(PersonID)
-            REFERENCES Persons(PersonID)
+    CONSTRAINT fkexecutor
+        FOREIGN KEY(ExecutorID)
+            REFERENCES Persons(PersonID),
+    CONSTRAINT fkqa
+        FOREIGN KEY(QaID)
+	    REFERENCES Persons(PersonID)
 );
 CREATE TABLE Products (
     ProductID SERIAL PRIMARY KEY,
@@ -32,8 +36,9 @@ CREATE TABLE Products_Tasks (
     PRIMARY KEY (ProductID, TaskID)
 );
 INSERT INTO Persons (FirstName, LastName, Address, City) VALUES ('Mike', 'Hughes', '123 Fake Street', 'Belfast');
-INSERT INTO Tasks (PersonID, TaskName, Completed) VALUES (1, 'assembling', TRUE);
-INSERT INTO Tasks (PersonID, TaskName, Completed) VALUES (1, 'processing', FALSE);
+INSERT INTO Persons (FirstName, LastName, Address, City) Values ('John', 'Smith', '64 Updown Road', 'Dublin');
+INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (1, 2, 'assembling', TRUE);
+INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (2, 1, 'processing', FALSE);
 INSERT INTO Products (ProductName) VALUES ('Computers');
 INSERT INTO Products (ProductName) VALUES ('Laptops');
 INSERT INTO Products (ProductName) VALUES ('Chairs');

package/src/api/controllers/row/ExternalRequest.ts

@@ -184,7 +184,7 @@ module External {
     thisRow._id = generateIdForRow(row, table)
     thisRow.tableId = table._id
     thisRow._rev = "rev"
-    return thisRow
+    return processFormulas(table, thisRow)
   }
 
   function fixArrayTypes(row: Row, table: Table) {
@@ -327,8 +327,12 @@ module External {
      * This iterates through the returned rows and works out what elements of the rows
      * actually match up to another row (based on primary keys) - this is pretty specific
      * to SQL and the way that SQL relationships are returned based on joins.
+     * This is complicated, but the idea is that when a SQL query returns all the relations
+     * will be separate rows, with all of the data in each row. We have to decipher what comes
+     * from where (which tables) and how to convert that into budibase columns.
      */
     updateRelationshipColumns(
+      table: Table,
       row: Row,
       rows: { [key: string]: Row },
       relationships: RelationshipsJson[]
@@ -339,6 +343,13 @@ module External {
         if (!linkedTable) {
           continue
         }
+        const fromColumn = `${table.name}.${relationship.from}`
+        const toColumn = `${linkedTable.name}.${relationship.to}`
+        // this is important when working with multiple relationships
+        // between the same tables, don't want to overlap/multiply the relations
+        if (!relationship.through && row[fromColumn] !== row[toColumn]) {
+          continue
+        }
         let linked = basicProcessing(row, linkedTable)
         if (!linked._id) {
           continue
@@ -386,6 +397,7 @@ module External {
         // this is a relationship of some sort
         if (finalRows[rowId]) {
           finalRows = this.updateRelationshipColumns(
+            table,
             row,
             finalRows,
             relationships
@@ -399,6 +411,7 @@ module External {
         finalRows[thisRow._id] = thisRow
         // do this at end once its been added to the final rows
         finalRows = this.updateRelationshipColumns(
+          table,
           row,
           finalRows,
           relationships

package/src/api/controllers/row/external.js

@@ -31,23 +31,21 @@ async function handleRequest(operation, tableId, opts = {}) {
 exports.handleRequest = handleRequest
 
 exports.patch = async ctx => {
-  const appId = ctx.appId
   const inputs = ctx.request.body
   const tableId = ctx.params.tableId
   const id = breakRowIdField(inputs._id)
   // don't save the ID to db
   delete inputs._id
-  return handleRequest(appId, DataSourceOperation.UPDATE, tableId, {
+  return handleRequest(DataSourceOperation.UPDATE, tableId, {
     id,
     row: inputs,
   })
 }
 
 exports.save = async ctx => {
-  const appId = ctx.appId
   const inputs = ctx.request.body
   const tableId = ctx.params.tableId
-  return handleRequest(appId, DataSourceOperation.CREATE, tableId, {
+  return handleRequest(DataSourceOperation.CREATE, tableId, {
     row: inputs,
   })
 }
@@ -61,49 +59,35 @@ exports.fetchView = async ctx => {
 }
 
 exports.fetch = async ctx => {
-  const appId = ctx.appId
   const tableId = ctx.params.tableId
-  return handleRequest(appId, DataSourceOperation.READ, tableId)
+  return handleRequest(DataSourceOperation.READ, tableId)
 }
 
 exports.find = async ctx => {
-  const appId = ctx.appId
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
-  const response = await handleRequest(
-    appId,
-    DataSourceOperation.READ,
-    tableId,
-    {
-      id,
-    }
-  )
+  const response = await handleRequest(DataSourceOperation.READ, tableId, {
+    id,
+  })
   return response ? response[0] : response
 }
 
 exports.destroy = async ctx => {
-  const appId = ctx.appId
   const tableId = ctx.params.tableId
   const id = ctx.request.body._id
-  const { row } = await handleRequest(
-    appId,
-    DataSourceOperation.DELETE,
-    tableId,
-    {
-      id,
-    }
-  )
+  const { row } = await handleRequest(DataSourceOperation.DELETE, tableId, {
+    id,
+  })
   return { response: { ok: true }, row }
 }
 
 exports.bulkDestroy = async ctx => {
-  const appId = ctx.appId
   const { rows } = ctx.request.body
   const tableId = ctx.params.tableId
   let promises = []
   for (let row of rows) {
     promises.push(
-      handleRequest(appId, DataSourceOperation.DELETE, tableId, {
+      handleRequest(DataSourceOperation.DELETE, tableId, {
         id: breakRowIdField(row._id),
       })
     )
@@ -113,7 +97,6 @@ exports.bulkDestroy = async ctx => {
 }
 
 exports.search = async ctx => {
-  const appId = ctx.appId
   const tableId = ctx.params.tableId
   const { paginate, query, ...params } = ctx.request.body
   let { bookmark, limit } = params
@@ -143,26 +126,21 @@ exports.search = async ctx => {
       [params.sort]: direction,
     }
   }
-  const rows = await handleRequest(appId, DataSourceOperation.READ, tableId, {
+  const rows = await handleRequest(DataSourceOperation.READ, tableId, {
     filters: query,
     sort,
     paginate: paginateObj,
   })
   let hasNextPage = false
   if (paginate && rows.length === limit) {
-    const nextRows = await handleRequest(
-      appId,
-      DataSourceOperation.READ,
-      tableId,
-      {
-        filters: query,
-        sort,
-        paginate: {
-          limit: 1,
-          page: bookmark * limit + 1,
-        },
-      }
-    )
+    const nextRows = await handleRequest(DataSourceOperation.READ, tableId, {
+      filters: query,
+      sort,
+      paginate: {
+        limit: 1,
+        page: bookmark * limit + 1,
+      },
+    })
     hasNextPage = nextRows.length > 0
   }
   // need wrapper object for bookmarks etc when paginating
@@ -175,7 +153,6 @@ exports.validate = async () => {
 }
 
 exports.fetchEnrichedRow = async ctx => {
-  const appId = ctx.appId
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
   const { datasourceId, tableName } = breakExternalTableId(tableId)
@@ -185,15 +162,10 @@ exports.fetchEnrichedRow = async ctx => {
     ctx.throw(400, "Datasource has not been configured for plus API.")
   }
   const tables = datasource.entities
-  const response = await handleRequest(
-    appId,
-    DataSourceOperation.READ,
-    tableId,
-    {
-      id,
-      datasource,
-    }
-  )
+  const response = await handleRequest(DataSourceOperation.READ, tableId, {
+    id,
+    datasource,
+  })
   const table = tables[tableName]
   const row = response[0]
   // this seems like a lot of work, but basically we need to dig deeper for the enrich
@@ -212,7 +184,6 @@ exports.fetchEnrichedRow = async ctx => {
     // don't support composite keys right now
     const linkedIds = links.map(link => breakRowIdField(link._id)[0])
     row[fieldName] = await handleRequest(
-      appId,
       DataSourceOperation.READ,
       linkedTableId,
       {

package/src/api/controllers/static/index.js

@@ -16,7 +16,7 @@ const { clientLibraryPath } = require("../../../utilities")
 const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentTypes } = require("../../../db/utils")
-const { getAppDB } = require("@budibase/backend-core/context")
+const { getAppDB, updateAppId } = require("@budibase/backend-core/context")
 const AWS = require("aws-sdk")
 const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"
 
@@ -49,11 +49,9 @@ async function getAppIdFromUrl(ctx) {
     a => a.url && a.url.toLowerCase() === possibleAppUrl
   )[0]
 
-  if (app && app.appId) {
-    return app.appId
-  } else {
-    return ctx.params.appId
-  }
+  const appId = app && app.appId ? app.appId : ctx.params.appId
+  updateAppId(appId)
+  return appId
 }
 
 exports.serveBuilder = async function (ctx) {

package/src/api/controllers/user.js

@@ -14,7 +14,7 @@ const {
   dbExists,
 } = require("@budibase/backend-core/db")
 const { UserStatus } = require("@budibase/backend-core/constants")
-const { getAppDB } = require("@budibase/backend-core/context")
+const { getAppDB, doInAppContext } = require("@budibase/backend-core/context")
 
 async function rawMetadata() {
   const db = getAppDB()
@@ -105,34 +105,36 @@ exports.syncUser = async function (ctx) {
       if (!(await dbExists(appId))) {
         continue
       }
-      const db = getAppDB()
-      const metadataId = generateUserMetadataID(userId)
-      let metadata
-      try {
-        metadata = await db.get(metadataId)
-      } catch (err) {
-        if (deleting) {
-          continue
+      await doInAppContext(appId, async () => {
+        const db = getAppDB()
+        const metadataId = generateUserMetadataID(userId)
+        let metadata
+        try {
+          metadata = await db.get(metadataId)
+        } catch (err) {
+          if (deleting) {
+            return
+          }
+          metadata = {
+            tableId: InternalTables.USER_METADATA,
+          }
         }
-        metadata = {
-          tableId: InternalTables.USER_METADATA,
+        // assign the roleId for the metadata doc
+        if (roleId) {
+          metadata.roleId = roleId
         }
-      }
-      // assign the roleId for the metadata doc
-      if (roleId) {
-        metadata.roleId = roleId
-      }
-      let combined = !deleting
-        ? combineMetadataAndUser(user, metadata)
-        : {
-            ...metadata,
-            status: UserStatus.INACTIVE,
-            metadata: BUILTIN_ROLE_IDS.PUBLIC,
-          }
-      // if its null then there was no updates required
-      if (combined) {
-        await db.put(combined)
-      }
+        let combined = !deleting
+          ? combineMetadataAndUser(user, metadata)
+          : {
+              ...metadata,
+              status: UserStatus.INACTIVE,
+              metadata: BUILTIN_ROLE_IDS.PUBLIC,
+            }
+        // if its null then there was no updates required
+        if (combined) {
+          await db.put(combined)
+        }
+      })
     }
   }
   ctx.body = {

package/src/integrations/base/sql.ts

@@ -166,15 +166,13 @@ class InternalBuilder {
 
   addSorting(query: KnexQuery, json: QueryJson): KnexQuery {
     let { sort, paginate } = json
-    if (!sort) {
-      return query
-    }
     const table = json.meta?.table
-    for (let [key, value] of Object.entries(sort)) {
-      const direction = value === SortDirection.ASCENDING ? "asc" : "desc"
-      query = query.orderBy(`${table?.name}.${key}`, direction)
-    }
-    if (this.client === SqlClients.MS_SQL && !sort && paginate?.limit) {
+    if (sort) {
+      for (let [key, value] of Object.entries(sort)) {
+        const direction = value === SortDirection.ASCENDING ? "asc" : "desc"
+        query = query.orderBy(`${table?.name}.${key}`, direction)
+      }
+    } else if (this.client === SqlClients.MS_SQL && paginate?.limit) {
       // @ts-ignore
       query = query.orderBy(`${table?.name}.${table?.primary[0]}`)
     }
@@ -191,29 +189,70 @@ class InternalBuilder {
     if (!relationships) {
       return query
     }
+    const tableSets: Record<string, [any]> = {}
+    // aggregate into table sets (all the same to tables)
     for (let relationship of relationships) {
-      const from = relationship.from,
-        to = relationship.to,
-        toTable = relationship.tableName
-      if (!relationship.through) {
+      const keyObj: { toTable: string; throughTable: string | undefined } = {
+        toTable: relationship.tableName,
+        throughTable: undefined,
+      }
+      if (relationship.through) {
+        keyObj.throughTable = relationship.through
+      }
+      const key = JSON.stringify(keyObj)
+      if (tableSets[key]) {
+        tableSets[key].push(relationship)
+      } else {
+        tableSets[key] = [relationship]
+      }
+    }
+    for (let [key, relationships] of Object.entries(tableSets)) {
+      const { toTable, throughTable } = JSON.parse(key)
+      if (!throughTable) {
         // @ts-ignore
-        query = query.leftJoin(
+        query = query.join(
           toTable,
-          `${fromTable}.${from}`,
-          `${toTable}.${to}`
+          function () {
+            for (let relationship of relationships) {
+              const from = relationship.from,
+                to = relationship.to
+              // @ts-ignore
+              this.orOn(`${fromTable}.${from}`, "=", `${toTable}.${to}`)
+            }
+          },
+          "left"
         )
       } else {
-        const throughTable = relationship.through
-        const fromPrimary = relationship.fromPrimary
-        const toPrimary = relationship.toPrimary
         query = query
           // @ts-ignore
-          .leftJoin(
+          .join(
             throughTable,
-            `${fromTable}.${fromPrimary}`,
-            `${throughTable}.${from}`
+            function () {
+              for (let relationship of relationships) {
+                const fromPrimary = relationship.fromPrimary
+                const from = relationship.from
+                // @ts-ignore
+                this.orOn(
+                  `${fromTable}.${fromPrimary}`,
+                  "=",
+                  `${throughTable}.${from}`
+                )
+              }
+            },
+            "left"
+          )
+          .join(
+            toTable,
+            function () {
+              for (let relationship of relationships) {
+                const toPrimary = relationship.toPrimary
+                const to = relationship.to
+                // @ts-ignore
+                this.orOn(`${toTable}.${toPrimary}`, `${throughTable}.${to}`)
+              }
+            },
+            "left"
           )
-          .leftJoin(toTable, `${toTable}.${toPrimary}`, `${throughTable}.${to}`)
       }
     }
     return query.limit(BASE_LIMIT)

package/src/threads/query.js

@@ -5,9 +5,6 @@ const { integrations } = require("../integrations")
 const { processStringSync } = require("@budibase/string-templates")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
 
-const IS_TRIPLE_BRACE = new RegExp(/^{{3}.*}{3}$/)
-const IS_HANDLEBARS = new RegExp(/^{{2}.*}{2}$/)
-
 class QueryRunner {
   constructor(input, flags = { noRecursiveQuery: false }) {
     this.datasource = input.datasource
@@ -188,12 +185,8 @@ class QueryRunner {
         enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters)
       } else if (typeof fields[key] === "string") {
         // enrich string value as normal
-        let value = fields[key]
-        // add triple brace to avoid escaping e.g. '=' in cookie header
-        if (IS_HANDLEBARS.test(value) && !IS_TRIPLE_BRACE.test(value)) {
-          value = `{${value}}`
-        }
-        enrichedQuery[key] = processStringSync(value, parameters, {
+        enrichedQuery[key] = processStringSync(fields[key], parameters, {
+          noEscaping: true,
           noHelpers: true,
         })
       } else {