@budibase/server 1.0.56 → 1.0.58-alpha.2

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.56",
+  "version": "1.0.58-alpha.2",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.56",
-    "@budibase/client": "^1.0.56",
-    "@budibase/string-templates": "^1.0.56",
+    "@budibase/backend-core": "^1.0.58-alpha.2",
+    "@budibase/client": "^1.0.58-alpha.2",
+    "@budibase/string-templates": "^1.0.58-alpha.2",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -168,5 +168,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "dc9d8b926198fc581281654cbb756fa58b17e3f8"
+  "gitHead": "d3f4c4a68b8abef15ef812381965b99ff6425d63"
 }

package/scripts/dev/manage.js

@@ -22,21 +22,21 @@ const Commands = {
 }
 
 async function init() {
-  // generate envoy file, always do this incase it has changed
+  // generate nginx file, always do this incase it has changed
   const hostingPath = path.join(process.cwd(), "..", "..", "hosting")
-  const envoyHbsPath = path.join(hostingPath, "envoy.dev.yaml.hbs")
-  const envoyOutputPath = path.join(hostingPath, ".generated-envoy.dev.yaml")
-  const contents = fs.readFileSync(envoyHbsPath, "utf8")
+  const nginxHbsPath = path.join(hostingPath, "nginx.dev.conf.hbs")
+  const nginxOutputPath = path.join(hostingPath, ".generated-nginx.dev.conf")
+  const contents = fs.readFileSync(nginxHbsPath, "utf8")
   const config = {
     address: isLinux() ? "172.17.0.1" : "host.docker.internal",
   }
-  fs.writeFileSync(envoyOutputPath, processStringSync(contents, config))
+  fs.writeFileSync(nginxOutputPath, processStringSync(contents, config))
 
   const envFilePath = path.join(process.cwd(), ".env")
   if (!fs.existsSync(envFilePath)) {
     const envFileJson = {
       PORT: 4001,
-      MINIO_URL: "http://localhost:10000/",
+      MINIO_URL: "http://localhost:4004",
       COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/",
       REDIS_URL: "localhost:6379",
       WORKER_URL: "http://localhost:4002",

package/scripts/integrations/mssql/data/setup.sql

@@ -36,13 +36,9 @@ CREATE TABLE people
 INSERT products
   (name, description)  
 VALUES  
-  ('Bananas', 'Fruit thing');  
-  
-INSERT products
-  (name, description)  
-VALUES  
+  ('Bananas', 'Fruit thing'),
   ('Meat', 'Animal thing');
-
+  
 INSERT tasks
   (taskname, productid)
 VALUES

package/scripts/integrations/mysql/init.sql

@@ -19,6 +19,12 @@ CREATE TABLE Tasks (
         FOREIGN KEY(PersonID)
 	    REFERENCES Persons(PersonID)
 );
+CREATE TABLE Products (
+    id serial primary key,
+    name text,
+    updated time
+);
 INSERT INTO Persons (FirstName, LastName, Age, Address, City, CreatedAt) VALUES ('Mike', 'Hughes', 28.2, '123 Fake Street', 'Belfast', '2021-01-19 03:14:07');
 INSERT INTO Tasks (PersonID, TaskName) VALUES (1, 'assembling');
 INSERT INTO Tasks (PersonID, TaskName) VALUES (1, 'processing');
+INSERT INTO Products (name, updated) VALUES ('Meat', '11:00:22'), ('Fruit', '10:00:00');

package/scripts/likeCypress.ts

@@ -17,7 +17,7 @@ process.env.JWT_SECRET = "budibase"
 process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
 process.env.SELF_HOSTED = "1"
 process.env.WORKER_URL = `http://localhost:${WORKER_PORT}/`
-process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/`
+process.env.MINIO_URL = `http://localhost:4004`
 process.env.MINIO_ACCESS_KEY = "budibase"
 process.env.MINIO_SECRET_KEY = "budibase"
 process.env.COUCH_DB_USER = "budibase"

package/src/api/controllers/integration.js

@@ -1,7 +1,6 @@
 const { definitions } = require("../../integrations")
 
 exports.fetch = async function (ctx) {
-  // TODO: fetch these from a github repo etc
   ctx.status = 200
   ctx.body = definitions
 }

package/src/api/controllers/query/validation.js

@@ -31,8 +31,14 @@ exports.generateQueryValidation = () => {
 exports.generateQueryPreviewValidation = () => {
   // prettier-ignore
   return joiValidator.body(Joi.object({
+    _id: Joi.string().optional(),
+    _rev: Joi.string().optional(),
+    readable: Joi.boolean().optional(),
     fields: Joi.object().required(),
     queryVerb: Joi.string().allow().required(),
+    name: Joi.string().required(),
+    flags: Joi.object().optional(),
+    schema: Joi.object().optional(),
     extra: Joi.object().optional(),
     datasourceId: Joi.string().required(),
     transformer: Joi.string().optional(),

package/src/api/controllers/row/ExternalRequest.ts

@@ -541,7 +541,7 @@ module External {
         if (!linkTable || !linkPrimary) {
           return
         }
-        const rows = related[key].rows || []
+        const rows = related[key]?.rows || []
         const found = rows.find(
           (row: { [key: string]: any }) =>
             row[linkPrimary] === relationship.id ||

package/src/api/controllers/view/exporters.js

@@ -5,8 +5,11 @@ exports.csv = function (headers, rows) {
     csv = `${csv}\n${headers
       .map(header => {
         let val = row[header]
-        val = typeof val === "object" ? JSON.stringify(val) : val
-        return `"${val}"`.trim()
+        val =
+          typeof val === "object"
+            ? `"${JSON.stringify(val).replace(/"/g, "'")}"`
+            : `"${val}"`
+        return val.trim()
       })
       .join(",")}`
   }

package/src/api/routes/tests/query.spec.js

@@ -169,6 +169,7 @@ describe("/queries", () => {
           parameters: {},
           fields: {},
           queryVerb: "read",
+          name: datasource.name,
         })
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
@@ -230,7 +231,6 @@ describe("/queries", () => {
   })
 
   describe("variables", () => {
-
     async function preview(datasource, fields) {
       return config.previewQuery(request, config, datasource, fields)
     }
@@ -262,9 +262,13 @@ describe("/queries", () => {
     })
 
     it("check that it automatically retries on fail with cached dynamics", async () => {
-      const { datasource, query: base } = await config.dynamicVariableDatasource()
+      const { datasource, query: base } =
+        await config.dynamicVariableDatasource()
       // preview once to cache
-      await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
+      await preview(datasource, {
+        path: "www.google.com",
+        queryString: "test={{ variable3 }}",
+      })
       // check its in cache
       const contents = await checkCacheForDynamicVariable(base._id, "variable3")
       expect(contents.rows.length).toEqual(1)
@@ -277,9 +281,13 @@ describe("/queries", () => {
     })
 
     it("deletes variables when linked query is deleted", async () => {
-      const { datasource, query: base } = await config.dynamicVariableDatasource()
+      const { datasource, query: base } =
+        await config.dynamicVariableDatasource()
       // preview once to cache
-      await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
+      await preview(datasource, {
+        path: "www.google.com",
+        queryString: "test={{ variable3 }}",
+      })
       // check its in cache
       let contents = await checkCacheForDynamicVariable(base._id, "variable3")
       expect(contents.rows.length).toEqual(1)

package/src/integrations/base/sql.ts

@@ -210,49 +210,37 @@ class InternalBuilder {
       const { toTable, throughTable } = JSON.parse(key)
       if (!throughTable) {
         // @ts-ignore
-        query = query.join(
-          toTable,
-          function () {
-            for (let relationship of relationships) {
-              const from = relationship.from,
-                to = relationship.to
-              // @ts-ignore
-              this.orOn(`${fromTable}.${from}`, "=", `${toTable}.${to}`)
-            }
-          },
-          "left"
-        )
+        query = query.leftJoin(toTable, function () {
+          for (let relationship of relationships) {
+            const from = relationship.from,
+              to = relationship.to
+            // @ts-ignore
+            this.orOn(`${fromTable}.${from}`, "=", `${toTable}.${to}`)
+          }
+        })
       } else {
         query = query
           // @ts-ignore
-          .join(
-            throughTable,
-            function () {
-              for (let relationship of relationships) {
-                const fromPrimary = relationship.fromPrimary
-                const from = relationship.from
-                // @ts-ignore
-                this.orOn(
-                  `${fromTable}.${fromPrimary}`,
-                  "=",
-                  `${throughTable}.${from}`
-                )
-              }
-            },
-            "left"
-          )
-          .join(
-            toTable,
-            function () {
-              for (let relationship of relationships) {
-                const toPrimary = relationship.toPrimary
-                const to = relationship.to
-                // @ts-ignore
-                this.orOn(`${toTable}.${toPrimary}`, `${throughTable}.${to}`)
-              }
-            },
-            "left"
-          )
+          .leftJoin(throughTable, function () {
+            for (let relationship of relationships) {
+              const fromPrimary = relationship.fromPrimary
+              const from = relationship.from
+              // @ts-ignore
+              this.orOn(
+                `${fromTable}.${fromPrimary}`,
+                "=",
+                `${throughTable}.${from}`
+              )
+            }
+          })
+          .leftJoin(toTable, function () {
+            for (let relationship of relationships) {
+              const toPrimary = relationship.toPrimary
+              const to = relationship.to
+              // @ts-ignore
+              this.orOn(`${toTable}.${toPrimary}`, `${throughTable}.${to}`)
+            }
+          })
       }
     }
     return query.limit(BASE_LIMIT)

package/src/integrations/dynamodb.ts

@@ -169,13 +169,11 @@ module DynamoModule {
         IndexName: query.index ? query.index : undefined,
         ...query.json,
       }
-      if (query.index) {
-        const response = await this.client.query(params).promise()
-        if (response.Items) {
-          return response.Items
-        }
-        return response
+      const response = await this.client.query(params).promise()
+      if (response.Items) {
+        return response.Items
       }
+      return response
     }
 
     async scan(query: { table: string; json: object; index: null | string }) {

package/src/integrations/index.ts

@@ -11,6 +11,7 @@ const arangodb = require("./arangodb")
 const rest = require("./rest")
 const googlesheets = require("./googlesheets")
 const { SourceNames } = require("../definitions/datasource")
+const environment = require("../environment")
 
 const DEFINITIONS = {
   [SourceNames.POSTGRES]: postgres.schema,
@@ -24,7 +25,6 @@ const DEFINITIONS = {
   [SourceNames.MYSQL]: mysql.schema,
   [SourceNames.ARANGODB]: arangodb.schema,
   [SourceNames.REST]: rest.schema,
-  [SourceNames.GOOGLE_SHEETS]: googlesheets.schema,
 }
 
 const INTEGRATIONS = {
@@ -39,7 +39,6 @@ const INTEGRATIONS = {
   [SourceNames.MYSQL]: mysql.integration,
   [SourceNames.ARANGODB]: arangodb.integration,
   [SourceNames.REST]: rest.integration,
-  [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed
@@ -49,6 +48,11 @@ if (!(process.arch === "arm64" && process.platform === "darwin")) {
   INTEGRATIONS[SourceNames.ORACLE] = oracle.integration
 }
 
+if (environment.SELF_HOSTED) {
+  DEFINITIONS[SourceNames.GOOGLE_SHEETS] = googlesheets.schema
+  INTEGRATIONS[SourceNames.GOOGLE_SHEETS] = googlesheets.integration
+}
+
 module.exports = {
   definitions: DEFINITIONS,
   integrations: INTEGRATIONS,

package/src/tests/utilities/TestConfiguration.js

@@ -393,6 +393,7 @@ class TestConfiguration {
         parameters: {},
         fields,
         queryVerb: "read",
+        name: datasource.name,
       })
       .set(config.defaultHeaders())
       .expect("Content-Type", /json/)

package/src/threads/query.js

@@ -161,10 +161,16 @@ class QueryRunner {
       const responses = await Promise.all(dynamics)
       for (let i = 0; i < foundVars.length; i++) {
         const variable = foundVars[i]
-        parameters[variable.name] = processStringSync(variable.value, {
-          data: responses[i].rows,
-          info: responses[i].extra,
-        })
+        parameters[variable.name] = processStringSync(
+          variable.value,
+          {
+            data: responses[i].rows,
+            info: responses[i].extra,
+          },
+          {
+            escapeNewlines: true,
+          }
+        )
         // make sure its known that this uses dynamic variables in case it fails
         this.hasDynamicVariables = true
       }
@@ -188,6 +194,7 @@ class QueryRunner {
         enrichedQuery[key] = processStringSync(fields[key], parameters, {
           noEscaping: true,
           noHelpers: true,
+          escapeNewlines: true,
         })
       } else {
         enrichedQuery[key] = fields[key]