@budibase/server 0.9.140-alpha.4 → 0.9.140-alpha.5

package/dist/utilities/rowProcessor/index.js

@@ -96,10 +96,11 @@ const TYPE_TRANSFORM_MAP = {
  * @param {Object} user The user to be used for an appId as well as the createdBy and createdAt fields.
  * @param {Object} table The table which is to be used for the schema, as well as handling auto IDs incrementing.
  * @param {Object} row The row which is to be updated with information for the auto columns.
+ * @param {Object} opts specific options for function to carry out optional features.
  * @returns {{row: Object, table: Object}} The updated row and table, the table may need to be updated
  * for automatic ID purposes.
  */
-function processAutoColumn(user, table, row) {
+function processAutoColumn(user, table, row, opts = { reprocessing: false, noAutoRelationships: false }) {
     let now = new Date().toISOString();
     // if a row doesn't have a revision then it doesn't exist yet
     const creating = !row._rev;
@@ -109,7 +110,7 @@ function processAutoColumn(user, table, row) {
         }
         switch (schema.subtype) {
             case AutoFieldSubTypes.CREATED_BY:
-                if (creating) {
+                if (creating && !opts.reprocessing && !opts.noAutoRelationships) {
                     row[key] = [user.userId];
                 }
                 break;
@@ -119,7 +120,9 @@ function processAutoColumn(user, table, row) {
                 }
                 break;
             case AutoFieldSubTypes.UPDATED_BY:
-                row[key] = [user.userId];
+                if (!opts.reprocessing && !opts.noAutoRelationships) {
+                    row[key] = [user.userId];
+                }
                 break;
             case AutoFieldSubTypes.UPDATED_AT:
                 row[key] = now;
@@ -134,6 +137,7 @@ function processAutoColumn(user, table, row) {
     }
     return { table, row };
 }
+exports.processAutoColumn = processAutoColumn;
 /**
  * This will coerce a value to the correct types based on the type transform map
  * @param {object} row The value to coerce
@@ -156,9 +160,10 @@ exports.coerce = (row, type) => {
  * @param {object} user the user which is performing the input.
  * @param {object} row the row which is being created/updated.
  * @param {object} table the table which the row is being saved to.
+ * @param {object} opts some input processing options (like disabling auto-column relationships).
  * @returns {object} the row which has been prepared to be written to the DB.
  */
-exports.inputProcessing = (user = {}, table, row) => {
+exports.inputProcessing = (user = {}, table, row, opts = { noAutoRelationships: false }) => {
     let clonedRow = cloneDeep(row);
     // need to copy the table so it can be differenced on way out
     const copiedTable = cloneDeep(table);
@@ -182,7 +187,7 @@ exports.inputProcessing = (user = {}, table, row) => {
         }
     }
     // handle auto columns - this returns an object like {table, row}
-    return processAutoColumn(user, copiedTable, clonedRow);
+    return processAutoColumn(user, copiedTable, clonedRow, opts);
 };
 /**
  * This function enriches the input rows with anything they are supposed to contain, for example

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.140-alpha.4",
+  "version": "0.9.140-alpha.5",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -62,9 +62,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.140-alpha.4",
-    "@budibase/client": "^0.9.140-alpha.4",
-    "@budibase/string-templates": "^0.9.140-alpha.4",
+    "@budibase/auth": "^0.9.140-alpha.5",
+    "@budibase/client": "^0.9.140-alpha.5",
+    "@budibase/string-templates": "^0.9.140-alpha.5",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -102,6 +102,7 @@
     "pg": "8.5.1",
     "pino-pretty": "4.0.0",
     "pouchdb": "7.2.1",
+    "pouchdb-adapter-memory": "^7.2.1",
     "pouchdb-all-dbs": "1.0.2",
     "pouchdb-find": "^7.2.2",
     "pouchdb-replication-stream": "1.2.9",
@@ -131,7 +132,6 @@
     "express": "^4.17.1",
     "jest": "^27.0.5",
     "nodemon": "^2.0.4",
-    "pouchdb-adapter-memory": "^7.2.1",
     "prettier": "^2.3.1",
     "rimraf": "^3.0.2",
     "supertest": "^4.0.2",
@@ -140,5 +140,5 @@
     "typescript": "^4.3.5",
     "update-dotenv": "^1.1.1"
   },
-  "gitHead": "0c9203d539e1f542fab6f2671a2cf612c9e609c1"
+  "gitHead": "15cd5e45144b43db3af75d58bbe695dc497b7cbd"
 }

package/scripts/integrations/service-vehicles/docker-compose.yml

@@ -0,0 +1,28 @@
+version: "3.8"
+services:
+  db:
+    container_name: postgres-vehicle
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_USER: root
+      POSTGRES_PASSWORD: root
+      POSTGRES_DB: main
+    ports:
+      - "5432:5432"
+    volumes:
+      #- pg_data:/var/lib/postgresql/data/
+      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
+
+  pgadmin:
+    container_name: pgadmin
+    image: dpage/pgadmin4
+    restart: always
+    environment:
+      PGADMIN_DEFAULT_EMAIL: root@root.com
+      PGADMIN_DEFAULT_PASSWORD: root
+    ports:
+      - "5050:80"
+
+#volumes:
+#  pg_data:

package/scripts/integrations/service-vehicles/init.sql

@@ -0,0 +1,52 @@
+SELECT 'CREATE DATABASE main'
+WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
+CREATE TABLE Vehicles (
+    id bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ),
+    Registration text COLLATE pg_catalog."default",
+    Make text COLLATE pg_catalog."default",
+    Model text COLLATE pg_catalog."default",
+    Colour text COLLATE pg_catalog."default",
+    Year smallint,
+    CONSTRAINT Vehicles_pkey PRIMARY KEY (id)
+);
+
+CREATE TABLE ServiceLog (
+    id bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ),
+    Description text COLLATE pg_catalog."default",
+    VehicleId bigint,
+    ServiceDate timestamp without time zone,
+    Category text COLLATE pg_catalog."default",
+    Mileage bigint,
+    CONSTRAINT ServiceLog_pkey PRIMARY KEY (id),
+    CONSTRAINT vehicle_foreign_key FOREIGN KEY (VehicleId)
+        REFERENCES Vehicles (id) MATCH SIMPLE
+        ON UPDATE NO ACTION
+        ON DELETE NO ACTION
+);
+
+INSERT INTO Vehicles (Registration, Make, Model, Colour, Year)
+VALUES ('FAZ 9837','Volkswagen','Polo','White',2002);
+INSERT INTO Vehicles (Registration, Make, Model, Colour, Year)
+VALUES ('JHI 8827','BMW','M3','Black',2013);
+INSERT INTO Vehicles (Registration, Make, Model, Colour, Year)
+VALUES ('D903PI','Volvo','XC40','Grey',2014);
+INSERT INTO Vehicles (Registration, Make, Model, Colour, Year)
+VALUES ('YFI002','Volkswagen','Golf','Dark Blue',2018);
+INSERT INTO Vehicles (Registration, Make, Model, Colour, Year)
+VALUES ('HGT5677','Skoda','Octavia','Graphite',2009);
+INSERT INTO Vehicles (Registration, Make, Model, Colour, Year)
+VALUES ('PPF9276','Skoda','Octavia','Graphite',2021);
+INSERT INTO Vehicles (Registration, Make, Model, Colour, Year)
+VALUES ('J893FT','Toyota','Corolla','Red',2015);
+INSERT INTO Vehicles (Registration, Make, Model, Colour, Year)
+VALUES ('MJK776','Honda','HR-V','Silver',2015);
+
+
+INSERT INTO ServiceLog (Description, VehicleId, ServiceDate, Category, Mileage)
+VALUES ('Change front brakes', 1, '2021-05-04', 'Brakes', 20667);
+INSERT INTO ServiceLog (Description, VehicleId, ServiceDate, Category, Mileage)
+VALUES ('Tyres - full set', 1, '2021-05-04', 'Brakes', 20667);
+INSERT INTO ServiceLog (Description, VehicleId, ServiceDate, Category, Mileage)
+VALUES ('Engine tune up', 2, '2021-07-14', 'Brakes', 50889);
+INSERT INTO ServiceLog (Description, VehicleId, ServiceDate, Category, Mileage)
+VALUES ('Replace transmission', 3, '2021-09-26', 'Transmission', 98002);

package/scripts/integrations/service-vehicles/reset.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker-compose down
+docker volume prune -f

package/src/api/controllers/datasource.js

@@ -51,7 +51,7 @@ exports.buildSchemaFromDb = async function (ctx) {
   await connector.buildSchema(datasource._id, datasource.entities)
   datasource.entities = connector.tables
 
-  const response = await db.post(datasource)
+  const response = await db.put(datasource)
   datasource._rev = response.rev
 
   ctx.body = datasource
@@ -89,7 +89,7 @@ exports.save = async function (ctx) {
     ...ctx.request.body,
   }
 
-  const response = await db.post(datasource)
+  const response = await db.put(datasource)
   datasource._rev = response.rev
 
   // Drain connection pools when configuration is changed

package/src/api/controllers/row/ExternalRequest.ts

@@ -437,7 +437,11 @@ module External {
       for (let [colName, { isMany, rows, tableId }] of Object.entries(
         related
       )) {
-        const table = this.getTable(tableId)
+        const table: Table = this.getTable(tableId)
+        // if its not the foreign key skip it, nothing to do
+        if (table.primary && table.primary.indexOf(colName) !== -1) {
+          continue
+        }
         for (let row of rows) {
           const filters = buildFilters(generateIdForRow(row, table), {}, table)
           // safety check, if there are no filters on deletion bad things happen

package/src/api/controllers/row/internal.js

@@ -5,17 +5,22 @@ const {
   generateRowID,
   DocumentTypes,
   InternalTables,
+  generateMemoryViewID,
 } = require("../../../db/utils")
 const userController = require("../user")
 const {
   inputProcessing,
   outputProcessing,
+  processAutoColumn,
 } = require("../../../utilities/rowProcessor")
 const { FieldTypes } = require("../../../constants")
 const { isEqual } = require("lodash")
 const { validate, findRow } = require("./utils")
 const { fullSearch, paginatedSearch } = require("./internalSearch")
 const { getGlobalUsersFromMetadata } = require("../../../utilities/global")
+const inMemoryViews = require("../../../db/inMemoryView")
+const env = require("../../../environment")
+const { migrateToInMemoryView } = require("../view/utils")
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -25,17 +30,84 @@ const CALCULATION_TYPES = {
 
 async function storeResponse(ctx, db, row, oldTable, table) {
   row.type = "row"
-  const response = await db.put(row)
   // don't worry about rev, tables handle rev/lastID updates
+  // if another row has been written since processing this will
+  // handle the auto ID clash
   if (!isEqual(oldTable, table)) {
-    await db.put(table)
+    try {
+      await db.put(table)
+    } catch (err) {
+      if (err.status === 409) {
+        const updatedTable = await db.get(table._id)
+        let response = processAutoColumn(null, updatedTable, row, {
+          reprocessing: true,
+        })
+        await db.put(response.table)
+        row = response.row
+      } else {
+        throw err
+      }
+    }
   }
+  const response = await db.put(row)
   row._rev = response.rev
   // process the row before return, to include relationships
   row = await outputProcessing(ctx, table, row, { squash: false })
   return { row, table }
 }
 
+// doesn't do the outputProcessing
+async function getRawTableData(ctx, db, tableId) {
+  let rows
+  if (tableId === InternalTables.USER_METADATA) {
+    await userController.fetchMetadata(ctx)
+    rows = ctx.body
+  } else {
+    const response = await db.allDocs(
+      getRowParams(tableId, null, {
+        include_docs: true,
+      })
+    )
+    rows = response.rows.map(row => row.doc)
+  }
+  return rows
+}
+
+async function getView(db, viewName) {
+  let viewInfo
+  async function getFromDesignDoc() {
+    const designDoc = await db.get("_design/database")
+    viewInfo = designDoc.views[viewName]
+    return viewInfo
+  }
+  let migrate = false
+  if (env.SELF_HOSTED) {
+    viewInfo = await getFromDesignDoc()
+  } else {
+    try {
+      viewInfo = await db.get(generateMemoryViewID(viewName))
+      if (viewInfo) {
+        viewInfo = viewInfo.view
+      }
+    } catch (err) {
+      // check if it can be retrieved from design doc (needs migrated)
+      if (err.status !== 404) {
+        viewInfo = null
+      } else {
+        viewInfo = await getFromDesignDoc()
+        migrate = !!viewInfo
+      }
+    }
+  }
+  if (migrate) {
+    await migrateToInMemoryView(db, viewName)
+  }
+  if (!viewInfo) {
+    throw "View does not exist."
+  }
+  return viewInfo
+}
+
 exports.patch = async ctx => {
   const appId = ctx.appId
   const db = new CouchDB(appId)
@@ -139,15 +211,18 @@ exports.fetchView = async ctx => {
 
   const db = new CouchDB(appId)
   const { calculation, group, field } = ctx.query
-  const designDoc = await db.get("_design/database")
-  const viewInfo = designDoc.views[viewName]
-  if (!viewInfo) {
-    throw "View does not exist."
+  const viewInfo = await getView(db, viewName)
+  let response
+  if (env.SELF_HOSTED) {
+    response = await db.query(`database/${viewName}`, {
+      include_docs: !calculation,
+      group: !!group,
+    })
+  } else {
+    const tableId = viewInfo.meta.tableId
+    const data = await getRawTableData(ctx, db, tableId)
+    response = await inMemoryViews.runView(viewInfo, calculation, group, data)
   }
-  const response = await db.query(`database/${viewName}`, {
-    include_docs: !calculation,
-    group: !!group,
-  })
 
   let rows
   if (!calculation) {
@@ -191,19 +266,9 @@ exports.fetch = async ctx => {
   const appId = ctx.appId
   const db = new CouchDB(appId)
 
-  let rows,
-    table = await db.get(ctx.params.tableId)
-  if (ctx.params.tableId === InternalTables.USER_METADATA) {
-    await userController.fetchMetadata(ctx)
-    rows = ctx.body
-  } else {
-    const response = await db.allDocs(
-      getRowParams(ctx.params.tableId, null, {
-        include_docs: true,
-      })
-    )
-    rows = response.rows.map(row => row.doc)
-  }
+  const tableId = ctx.params.tableId
+  let table = await db.get(tableId)
+  let rows = await getRawTableData(ctx, db, tableId)
   return outputProcessing(ctx, table, rows)
 }
 

package/src/api/controllers/table/index.js

@@ -145,7 +145,7 @@ exports.save = async function (ctx) {
   if (updatedRows && updatedRows.length !== 0) {
     await db.bulkDocs(updatedRows)
   }
-  const result = await db.post(tableToSave)
+  const result = await db.put(tableToSave)
   tableToSave._rev = result.rev
 
   tableToSave = await tableSaveFunctions.after(tableToSave)

package/src/api/controllers/table/utils.js

@@ -68,23 +68,17 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
     // Populate the table with rows imported from CSV in a bulk update
     const data = await csvParser.transform(dataImport)
 
+    let finalData = []
     for (let i = 0; i < data.length; i++) {
       let row = data[i]
       row._id = generateRowID(table._id)
       row.tableId = table._id
-      const processed = inputProcessing(user, table, row)
+      const processed = inputProcessing(user, table, row, {
+        noAutoRelationships: true,
+      })
       table = processed.table
       row = processed.row
 
-      // make sure link rows are up to date
-      row = await linkRows.updateLinks({
-        appId,
-        eventType: linkRows.EventType.ROW_SAVE,
-        row,
-        tableId: row.tableId,
-        table,
-      })
-
       for (let [fieldName, schema] of Object.entries(table.schema)) {
         // check whether the options need to be updated for inclusion as part of the data import
         if (
@@ -98,10 +92,20 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
           ]
         }
       }
-      data[i] = row
+
+      // make sure link rows are up to date
+      finalData.push(
+        linkRows.updateLinks({
+          appId,
+          eventType: linkRows.EventType.ROW_SAVE,
+          row,
+          tableId: row.tableId,
+          table,
+        })
+      )
     }
 
-    await db.bulkDocs(data)
+    await db.bulkDocs(await Promise.all(finalData))
     let response = await db.put(table)
     table._rev = response._rev
   }

package/src/api/controllers/view/index.js

@@ -2,127 +2,93 @@ const CouchDB = require("../../../db")
 const viewTemplate = require("./viewBuilder")
 const { apiFileReturn } = require("../../../utilities/fileSystem")
 const exporters = require("./exporters")
+const { saveView, getView, getViews, deleteView } = require("./utils")
 const { fetchView } = require("../row")
-const { ViewNames } = require("../../../db/utils")
 
-const controller = {
-  fetch: async ctx => {
-    const db = new CouchDB(ctx.appId)
-    const designDoc = await db.get("_design/database")
-    const response = []
-
-    for (let name of Object.keys(designDoc.views)) {
-      // Only return custom views, not built ins
-      if (Object.values(ViewNames).indexOf(name) !== -1) {
-        continue
-      }
-      response.push({
-        name,
-        ...designDoc.views[name],
-      })
-    }
-
-    ctx.body = response
-  },
-  save: async ctx => {
-    const db = new CouchDB(ctx.appId)
-    const { originalName, ...viewToSave } = ctx.request.body
-    const designDoc = await db.get("_design/database")
-    const view = viewTemplate(viewToSave)
-
-    if (!viewToSave.name) {
-      ctx.throw(400, "Cannot create view without a name")
-    }
-
-    designDoc.views = {
-      ...designDoc.views,
-      [viewToSave.name]: view,
-    }
-
-    // view has been renamed
-    if (originalName) {
-      delete designDoc.views[originalName]
-    }
-
-    await db.put(designDoc)
-
-    // add views to table document
-    const table = await db.get(ctx.request.body.tableId)
-    if (!table.views) table.views = {}
-    if (!view.meta.schema) {
-      view.meta.schema = table.schema
-    }
-    table.views[viewToSave.name] = view.meta
-
-    if (originalName) {
-      delete table.views[originalName]
-    }
-
-    await db.put(table)
-
-    ctx.body = {
-      ...table.views[viewToSave.name],
-      name: viewToSave.name,
-    }
-  },
-  destroy: async ctx => {
-    const db = new CouchDB(ctx.appId)
-    const designDoc = await db.get("_design/database")
-    const viewName = decodeURI(ctx.params.viewName)
-    const view = designDoc.views[viewName]
-    delete designDoc.views[viewName]
-
-    await db.put(designDoc)
-
-    const table = await db.get(view.meta.tableId)
-    delete table.views[viewName]
-    await db.put(table)
-
-    ctx.body = view
-  },
-  exportView: async ctx => {
-    const db = new CouchDB(ctx.appId)
-    const designDoc = await db.get("_design/database")
-    const viewName = decodeURI(ctx.query.view)
+exports.fetch = async ctx => {
+  const db = new CouchDB(ctx.appId)
+  ctx.body = await getViews(db)
+}
 
-    const view = designDoc.views[viewName]
-    const format = ctx.query.format
-    if (!format) {
-      ctx.throw(400, "Format must be specified, either csv or json")
-    }
+exports.save = async ctx => {
+  const db = new CouchDB(ctx.appId)
+  const { originalName, ...viewToSave } = ctx.request.body
+  const view = viewTemplate(viewToSave)
+
+  if (!viewToSave.name) {
+    ctx.throw(400, "Cannot create view without a name")
+  }
+
+  await saveView(db, originalName, viewToSave.name, view)
+
+  // add views to table document
+  const table = await db.get(ctx.request.body.tableId)
+  if (!table.views) table.views = {}
+  if (!view.meta.schema) {
+    view.meta.schema = table.schema
+  }
+  table.views[viewToSave.name] = view.meta
+  if (originalName) {
+    delete table.views[originalName]
+  }
+  await db.put(table)
+
+  ctx.body = {
+    ...table.views[viewToSave.name],
+    name: viewToSave.name,
+  }
+}
 
-    if (view) {
-      ctx.params.viewName = viewName
-      // Fetch view rows
-      ctx.query = {
-        group: view.meta.groupBy,
-        calculation: view.meta.calculation,
-        stats: !!view.meta.field,
-        field: view.meta.field,
-      }
-    } else {
-      // table all_ view
-      /* istanbul ignore next */
-      ctx.params.viewName = viewName
-    }
+exports.destroy = async ctx => {
+  const db = new CouchDB(ctx.appId)
+  const viewName = decodeURI(ctx.params.viewName)
+  const view = await deleteView(db, viewName)
+  const table = await db.get(view.meta.tableId)
+  delete table.views[viewName]
+  await db.put(table)
 
-    await fetchView(ctx)
+  ctx.body = view
+}
 
-    let schema = view && view.meta && view.meta.schema
-    if (!schema) {
-      const tableId = ctx.params.tableId || view.meta.tableId
-      const table = await db.get(tableId)
-      schema = table.schema
+exports.exportView = async ctx => {
+  const db = new CouchDB(ctx.appId)
+  const viewName = decodeURI(ctx.query.view)
+  const view = await getView(db, viewName)
+
+  const format = ctx.query.format
+  if (!format) {
+    ctx.throw(400, "Format must be specified, either csv or json")
+  }
+
+  if (view) {
+    ctx.params.viewName = viewName
+    // Fetch view rows
+    ctx.query = {
+      group: view.meta.groupBy,
+      calculation: view.meta.calculation,
+      stats: !!view.meta.field,
+      field: view.meta.field,
     }
-
-    // Export part
-    let headers = Object.keys(schema)
-    const exporter = exporters[format]
-    const filename = `${viewName}.${format}`
-    // send down the file
-    ctx.attachment(filename)
-    ctx.body = apiFileReturn(exporter(headers, ctx.body))
-  },
+  } else {
+    // table all_ view
+    /* istanbul ignore next */
+    ctx.params.viewName = viewName
+  }
+
+  await fetchView(ctx)
+
+  let schema = view && view.meta && view.meta.schema
+  if (!schema) {
+    const tableId = ctx.params.tableId || view.meta.tableId
+    const table = await db.get(tableId)
+    schema = table.schema
+  }
+
+  // Export part
+  let headers = Object.keys(schema)
+  const exporter = exporters[format]
+  const filename = `${viewName}.${format}`
+  // send down the file
+  ctx.attachment(filename)
+  ctx.body = apiFileReturn(exporter(headers, ctx.body))
 }
-
-module.exports = controller