@budibase/server 2.1.17 → 2.1.19-alpha.0

package/dist/utilities/index.js

@@ -14,7 +14,6 @@ const { sanitizeKey } = require("@budibase/backend-core/objectStore");
 const { generateMetadataID } = require("../db/utils");
 const Readable = require("stream").Readable;
 const { getAppDB } = require("@budibase/backend-core/context");
-const BB_CDN = "https://cdn.budi.live";
 exports.wait = ms => new Promise(resolve => setTimeout(resolve, ms));
 exports.isDev = env.isDev;
 exports.NUMBER_REGEX = /^[+-]?([0-9]*[.])?[0-9]+$/g;
@@ -44,7 +43,7 @@ exports.objectStoreUrl = () => {
         return OBJ_STORE_DIRECTORY;
     }
     else {
-        return BB_CDN;
+        return env.CDN_URL;
     }
 };
 /**

package/dist/utilities/plugins.js

@@ -12,7 +12,7 @@ exports.enrichPluginURLs = plugins => {
         const jsFileName = "plugin.min.js";
         // In self host we need to prefix the path, as the bucket name is not part
         // of the bucket path. In cloud, it's already part of the bucket path.
-        let jsUrl = cloud ? "https://cdn.budi.live/" : `/${bucket}/`;
+        let jsUrl = cloud ? `${env.CDN_URL}/` : `/${bucket}/`;
         jsUrl += ProPlugins.getBucketPath(plugin.name);
         jsUrl += jsFileName;
         return Object.assign(Object.assign({}, plugin), { jsUrl });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.1.17",
+  "version": "2.1.19-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^2.1.17",
-    "@budibase/client": "^2.1.17",
-    "@budibase/pro": "2.1.16",
-    "@budibase/string-templates": "^2.1.17",
-    "@budibase/types": "^2.1.17",
+    "@budibase/backend-core": "2.1.19-alpha.0",
+    "@budibase/client": "2.1.19-alpha.0",
+    "@budibase/pro": "2.1.18",
+    "@budibase/string-templates": "2.1.19-alpha.0",
+    "@budibase/types": "2.1.19-alpha.0",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -199,5 +199,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "f13fc9cd82db61af9474e1070071cafefdfed42d"
+  "gitHead": "8159f0503b4cfcf1a307ac221ccf5609239be71a"
 }

package/src/api/controllers/row/ExternalRequest.ts

@@ -282,9 +282,11 @@ module External {
         const linkTablePrimary = linkTable.primary[0]
         // one to many
         if (isOneSide(field)) {
-          newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(
-            row[key][0]
-          )[0]
+          let id = row[key][0]
+          if (typeof row[key] === "string") {
+            id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1]
+          }
+          newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(id)[0]
         }
         // many to many
         else if (field.through) {

package/src/api/controllers/row/index.ts

@@ -30,16 +30,24 @@ export async function patch(ctx: any): Promise<any> {
   if (body && !body._id) {
     return save(ctx)
   }
-  const { row, table } = await quotas.addQuery(
-    () => pickApi(tableId).patch(ctx),
-    {
-      datasourceId: tableId,
+  try {
+    const { row, table } = await quotas.addQuery(
+      () => pickApi(tableId).patch(ctx),
+      {
+        datasourceId: tableId,
+      }
+    )
+    if (!row) {
+      ctx.throw(404, "Row not found")
     }
-  )
-  ctx.status = 200
-  ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:update`, appId, row, table)
-  ctx.message = `${table.name} updated successfully.`
-  ctx.body = row
+    ctx.status = 200
+    ctx.eventEmitter &&
+      ctx.eventEmitter.emitRow(`row:update`, appId, row, table)
+    ctx.message = `${table.name} updated successfully.`
+    ctx.body = row
+  } catch (err) {
+    ctx.throw(400, err)
+  }
 }
 
 export const save = async (ctx: any) => {

package/src/environment.js

@@ -46,6 +46,7 @@ module.exports = {
   AWS_REGION: process.env.AWS_REGION,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
+  CDN_URL: process.env.CDN_URL || "https://cdn.budi.live",
   REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,

package/src/integrations/elasticsearch.ts

@@ -5,10 +5,13 @@ import {
   IntegrationBase,
 } from "@budibase/types"
 
-const { Client } = require("@elastic/elasticsearch")
+import { Client, ClientOptions } from "@elastic/elasticsearch"
 
 interface ElasticsearchConfig {
   url: string
+  ssl?: boolean
+  ca?: string
+  rejectUnauthorized?: boolean
 }
 
 const SCHEMA: Integration = {
@@ -23,6 +26,21 @@ const SCHEMA: Integration = {
       required: true,
       default: "http://localhost:9200",
     },
+    ssl: {
+      type: DatasourceFieldType.BOOLEAN,
+      default: false,
+      required: false,
+    },
+    rejectUnauthorized: {
+      type: DatasourceFieldType.BOOLEAN,
+      default: true,
+      required: false,
+    },
+    ca: {
+      type: DatasourceFieldType.LONGFORM,
+      default: false,
+      required: false,
+    },
   },
   query: {
     create: {
@@ -81,7 +99,19 @@ class ElasticSearchIntegration implements IntegrationBase {
 
   constructor(config: ElasticsearchConfig) {
     this.config = config
-    this.client = new Client({ node: config.url })
+
+    const clientConfig: ClientOptions = {
+      node: this.config.url,
+    }
+
+    if (this.config.ssl) {
+      clientConfig.ssl = {
+        rejectUnauthorized: this.config.rejectUnauthorized,
+        ca: this.config.ca || undefined,
+      }
+    }
+
+    this.client = new Client(clientConfig)
   }
 
   async create(query: { index: string; json: object }) {

package/src/integrations/s3.ts

@@ -1,5 +1,12 @@
-import { Integration, QueryType, IntegrationBase } from "@budibase/types"
+import {
+  Integration,
+  QueryType,
+  IntegrationBase,
+  DatasourceFieldType,
+} from "@budibase/types"
+
 const AWS = require("aws-sdk")
+const csv = require("csvtojson")
 
 interface S3Config {
   region: string
@@ -40,16 +47,106 @@ const SCHEMA: Integration = {
     },
   },
   query: {
+    create: {
+      type: QueryType.FIELDS,
+      fields: {
+        bucket: {
+          display: "New Bucket",
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        location: {
+          required: true,
+          default: "us-east-1",
+          type: DatasourceFieldType.STRING,
+        },
+        grantFullControl: {
+          display: "Grant full control",
+          type: DatasourceFieldType.STRING,
+        },
+        grantRead: {
+          display: "Grant read",
+          type: DatasourceFieldType.STRING,
+        },
+        grantReadAcp: {
+          display: "Grant read ACP",
+          type: DatasourceFieldType.STRING,
+        },
+        grantWrite: {
+          display: "Grant write",
+          type: DatasourceFieldType.STRING,
+        },
+        grantWriteAcp: {
+          display: "Grant write ACP",
+          type: DatasourceFieldType.STRING,
+        },
+      },
+    },
     read: {
       type: QueryType.FIELDS,
       fields: {
         bucket: {
-          type: "string",
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        delimiter: {
+          type: DatasourceFieldType.STRING,
+        },
+        marker: {
+          type: DatasourceFieldType.STRING,
+        },
+        maxKeys: {
+          type: DatasourceFieldType.NUMBER,
+          display: "Max Keys",
+        },
+        prefix: {
+          type: DatasourceFieldType.STRING,
+        },
+      },
+    },
+    readCsv: {
+      displayName: "Read CSV",
+      type: QueryType.FIELDS,
+      fields: {
+        bucket: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        key: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+      },
+    },
+    delete: {
+      type: QueryType.FIELDS,
+      fields: {
+        bucket: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        delete: {
+          type: DatasourceFieldType.JSON,
           required: true,
         },
       },
     },
   },
+  extra: {
+    acl: {
+      required: false,
+      displayName: "ACL",
+      type: DatasourceFieldType.LIST,
+      data: {
+        create: [
+          "private",
+          "public-read",
+          "public-read-write",
+          "authenticated-read",
+        ],
+      },
+    },
+  },
 }
 
 class S3Integration implements IntegrationBase {
@@ -67,14 +164,93 @@ class S3Integration implements IntegrationBase {
     this.client = new AWS.S3(this.config)
   }
 
-  async read(query: { bucket: string }) {
+  async create(query: {
+    bucket: string
+    location: string
+    grantFullControl: string
+    grantRead: string
+    grantReadAcp: string
+    grantWrite: string
+    grantWriteAcp: string
+    extra: {
+      acl: string
+    }
+  }) {
+    let params: any = {
+      Bucket: query.bucket,
+      ACL: query.extra?.acl,
+      GrantFullControl: query.grantFullControl,
+      GrantRead: query.grantRead,
+      GrantReadACP: query.grantReadAcp,
+      GrantWrite: query.grantWrite,
+      GrantWriteACP: query.grantWriteAcp,
+    }
+    if (query.location) {
+      params["CreateBucketConfiguration"] = {
+        LocationConstraint: query.location,
+      }
+    }
+    return await this.client.createBucket(params).promise()
+  }
+
+  async read(query: {
+    bucket: string
+    delimiter: string
+    expectedBucketOwner: string
+    marker: string
+    maxKeys: number
+    prefix: string
+  }) {
     const response = await this.client
       .listObjects({
         Bucket: query.bucket,
+        Delimiter: query.delimiter,
+        Marker: query.marker,
+        MaxKeys: query.maxKeys,
+        Prefix: query.prefix,
       })
       .promise()
     return response.Contents
   }
+
+  async readCsv(query: { bucket: string; key: string }) {
+    const stream = this.client
+      .getObject({
+        Bucket: query.bucket,
+        Key: query.key,
+      })
+      .createReadStream()
+
+    let csvError = false
+    return new Promise((resolve, reject) => {
+      stream.on("error", (err: Error) => {
+        reject(err)
+      })
+      const response = csv()
+        .fromStream(stream)
+        .on("error", () => {
+          csvError = true
+        })
+      stream.on("finish", () => {
+        resolve(response)
+      })
+    }).catch(err => {
+      if (csvError) {
+        throw new Error("Could not read CSV")
+      } else {
+        throw err
+      }
+    })
+  }
+
+  async delete(query: { bucket: string; delete: string }) {
+    return await this.client
+      .deleteObjects({
+        Bucket: query.bucket,
+        Delete: JSON.parse(query.delete),
+      })
+      .promise()
+  }
 }
 
 export default {

package/src/integrations/tests/s3.spec.ts

@@ -18,11 +18,95 @@ describe("S3 Integration", () => {
   })
 
   it("calls the read method with the correct params", async () => {
-    const response = await config.integration.read({
+    await config.integration.read({
       bucket: "test",
+      delimiter: "/",
+      marker: "file.txt",
+      maxKeys: 999,
+      prefix: "directory/",
     })
     expect(config.integration.client.listObjects).toHaveBeenCalledWith({
       Bucket: "test",
+      Delimiter: "/",
+      Marker: "file.txt",
+      MaxKeys: 999,
+      Prefix: "directory/",
+    })
+  })
+
+  it("calls the create method with the correct params", async () => {
+    await config.integration.create({
+      bucket: "test",
+      location: "af-south-1",
+      grantFullControl: "me",
+      grantRead: "him",
+      grantReadAcp: "her",
+      grantWrite: "she",
+      grantWriteAcp: "he",
+      objectLockEnabledForBucket: true,
+      extra: {
+        acl: "private",
+      },
+    })
+    expect(config.integration.client.createBucket).toHaveBeenCalledWith({
+      Bucket: "test",
+      CreateBucketConfiguration: {
+        LocationConstraint: "af-south-1",
+      },
+      GrantFullControl: "me",
+      GrantRead: "him",
+      GrantReadACP: "her",
+      GrantWrite: "she",
+      GrantWriteACP: "he",
+      ACL: "private",
+    })
+  })
+
+  it("does not add undefined location constraint when calling the create method", async () => {
+    await config.integration.create({
+      bucket: "test",
+    })
+    expect(config.integration.client.createBucket).toHaveBeenCalledWith({
+      Bucket: "test",
+      GrantFullControl: undefined,
+      GrantRead: undefined,
+      GrantReadACP: undefined,
+      GrantWrite: undefined,
+      GrantWriteACP: undefined,
+      ACL: undefined,
+    })
+  })
+
+  it("calls the delete method with the correct params ", async () => {
+    await config.integration.delete({
+      bucket: "test",
+      delete: `{
+        "Objects": [
+          {
+         "Key": "HappyFace.jpg", 
+         "VersionId": "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b"
+        }, 
+          {
+         "Key": "HappyFace.jpg", 
+         "VersionId": "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd"
+        }
+       ]
+      }`,
+    })
+    expect(config.integration.client.deleteObjects).toHaveBeenCalledWith({
+      Bucket: "test",
+      Delete: {
+        Objects: [
+          {
+            Key: "HappyFace.jpg",
+            VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b",
+          },
+          {
+            Key: "HappyFace.jpg",
+            VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd",
+          },
+        ],
+      },
     })
   })
 })

package/src/utilities/index.js

@@ -5,8 +5,6 @@ const { generateMetadataID } = require("../db/utils")
 const Readable = require("stream").Readable
 const { getAppDB } = require("@budibase/backend-core/context")
 
-const BB_CDN = "https://cdn.budi.live"
-
 exports.wait = ms => new Promise(resolve => setTimeout(resolve, ms))
 
 exports.isDev = env.isDev
@@ -40,7 +38,7 @@ exports.objectStoreUrl = () => {
     // can use a relative url for this as all goes through the proxy (this is hosted in minio)
     return OBJ_STORE_DIRECTORY
   } else {
-    return BB_CDN
+    return env.CDN_URL
   }
 }
 

package/src/utilities/plugins.js

@@ -13,7 +13,7 @@ exports.enrichPluginURLs = plugins => {
 
     // In self host we need to prefix the path, as the bucket name is not part
     // of the bucket path. In cloud, it's already part of the bucket path.
-    let jsUrl = cloud ? "https://cdn.budi.live/" : `/${bucket}/`
+    let jsUrl = cloud ? `${env.CDN_URL}/` : `/${bucket}/`
     jsUrl += ProPlugins.getBucketPath(plugin.name)
     jsUrl += jsFileName
     return { ...plugin, jsUrl }