@budibase/server 0.9.140-alpha.4 → 0.9.140-alpha.5

package/src/api/controllers/view/utils.js

@@ -0,0 +1,109 @@
+const {
+  ViewNames,
+  generateMemoryViewID,
+  getMemoryViewParams,
+} = require("../../../db/utils")
+const env = require("../../../environment")
+
+exports.getView = async (db, viewName) => {
+  if (env.SELF_HOSTED) {
+    const designDoc = await db.get("_design/database")
+    return designDoc.views[viewName]
+  } else {
+    const viewDoc = await db.get(generateMemoryViewID(viewName))
+    return viewDoc.view
+  }
+}
+
+exports.getViews = async db => {
+  const response = []
+  if (env.SELF_HOSTED) {
+    const designDoc = await db.get("_design/database")
+    for (let name of Object.keys(designDoc.views)) {
+      // Only return custom views, not built ins
+      if (Object.values(ViewNames).indexOf(name) !== -1) {
+        continue
+      }
+      response.push({
+        name,
+        ...designDoc.views[name],
+      })
+    }
+  } else {
+    const views = (
+      await db.allDocs(
+        getMemoryViewParams({
+          include_docs: true,
+        })
+      )
+    ).rows.map(row => row.doc)
+    for (let viewDoc of views) {
+      response.push({
+        name: viewDoc.name,
+        ...viewDoc.view,
+      })
+    }
+  }
+  return response
+}
+
+exports.saveView = async (db, originalName, viewName, viewTemplate) => {
+  if (env.SELF_HOSTED) {
+    const designDoc = await db.get("_design/database")
+    designDoc.views = {
+      ...designDoc.views,
+      [viewName]: viewTemplate,
+    }
+    // view has been renamed
+    if (originalName) {
+      delete designDoc.views[originalName]
+    }
+    await db.put(designDoc)
+  } else {
+    const id = generateMemoryViewID(viewName)
+    const originalId = originalName ? generateMemoryViewID(originalName) : null
+    const viewDoc = {
+      _id: id,
+      view: viewTemplate,
+      name: viewName,
+      tableId: viewTemplate.meta.tableId,
+    }
+    try {
+      const old = await db.get(id)
+      if (originalId) {
+        const originalDoc = await db.get(originalId)
+        await db.remove(originalDoc._id, originalDoc._rev)
+      }
+      if (old && old._rev) {
+        viewDoc._rev = old._rev
+      }
+    } catch (err) {
+      // didn't exist, just skip
+    }
+    await db.put(viewDoc)
+  }
+}
+
+exports.deleteView = async (db, viewName) => {
+  if (env.SELF_HOSTED) {
+    const designDoc = await db.get("_design/database")
+    const view = designDoc.views[viewName]
+    delete designDoc.views[viewName]
+    await db.put(designDoc)
+    return view
+  } else {
+    const id = generateMemoryViewID(viewName)
+    const viewDoc = await db.get(id)
+    await db.remove(viewDoc._id, viewDoc._rev)
+    return viewDoc.view
+  }
+}
+
+exports.migrateToInMemoryView = async (db, viewName) => {
+  // delete the view initially
+  const designDoc = await db.get("_design/database")
+  const view = designDoc.views[viewName]
+  delete designDoc.views[viewName]
+  await db.put(designDoc)
+  await exports.saveView(db, null, viewName, view)
+}

package/src/api/routes/tests/view.spec.js

@@ -205,7 +205,7 @@ describe("/views", () => {
   })
 
   describe("exportView", () => {
-    it("should be able to delete a view", async () => {
+    it("should be able to export a view", async () => {
       await config.createTable(priceTable())
       await config.createRow()
       const view = await config.createView()

package/src/automations/steps/createRow.js

@@ -2,6 +2,7 @@ const rowController = require("../../api/controllers/row")
 const automationUtils = require("../automationUtils")
 const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
+const { buildCtx } = require("./utils")
 
 exports.definition = {
   name: "Create Row",
@@ -69,16 +70,12 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) {
     }
   }
   // have to clean up the row, remove the table from it
-  const ctx = {
+  const ctx = buildCtx(appId, emitter, {
+    body: inputs.row,
     params: {
       tableId: inputs.row.tableId,
     },
-    request: {
-      body: inputs.row,
-    },
-    appId,
-    eventEmitter: emitter,
-  }
+  })
 
   try {
     inputs.row = await automationUtils.cleanUpRow(
@@ -86,7 +83,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) {
       inputs.row.tableId,
       inputs.row
     )
-    if (env.isProd()) {
+    if (env.USE_QUOTAS) {
       await usage.update(apiKey, usage.Properties.ROW, 1)
     }
     await rowController.save(ctx)

package/src/automations/steps/deleteRow.js

@@ -1,6 +1,7 @@
 const rowController = require("../../api/controllers/row")
 const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
+const { buildCtx } = require("./utils")
 
 exports.definition = {
   description: "Delete a row from your database",
@@ -60,19 +61,16 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) {
       },
     }
   }
-  let ctx = {
+
+  let ctx = buildCtx(appId, emitter, {
+    body: {
+      _id: inputs.id,
+      _rev: inputs.revision,
+    },
     params: {
       tableId: inputs.tableId,
     },
-    request: {
-      body: {
-        _id: inputs.id,
-        _rev: inputs.revision,
-      },
-    },
-    appId,
-    eventEmitter: emitter,
-  }
+  })
 
   try {
     if (env.isProd()) {

package/src/automations/steps/queryRows.js

@@ -1,6 +1,7 @@
 const rowController = require("../../api/controllers/row")
 const tableController = require("../../api/controllers/table")
 const { FieldTypes } = require("../../constants")
+const { buildCtx } = require("./utils")
 
 const SortOrders = {
   ASCENDING: "ascending",
@@ -70,12 +71,11 @@ exports.definition = {
 }
 
 async function getTable(appId, tableId) {
-  const ctx = {
+  const ctx = buildCtx(appId, null, {
     params: {
       id: tableId,
     },
-    appId,
-  }
+  })
   await tableController.find(ctx)
   return ctx.body
 }
@@ -89,21 +89,18 @@ exports.run = async function ({ inputs, appId }) {
     sortType =
       fieldType === FieldTypes.NUMBER ? FieldTypes.NUMBER : FieldTypes.STRING
   }
-  const ctx = {
+  const ctx = buildCtx(appId, null, {
     params: {
       tableId,
     },
-    request: {
-      body: {
-        sortOrder,
-        sortType,
-        sort: sortColumn,
-        query: filters || {},
-        limit,
-      },
+    body: {
+      sortOrder,
+      sortType,
+      sort: sortColumn,
+      query: filters || {},
+      limit,
     },
-    appId,
-  }
+  })
   try {
     await rowController.search(ctx)
     return {

package/src/automations/steps/updateRow.js

@@ -1,5 +1,6 @@
 const rowController = require("../../api/controllers/row")
 const automationUtils = require("../automationUtils")
+const { buildCtx } = require("./utils")
 
 exports.definition = {
   name: "Update Row",
@@ -72,19 +73,15 @@ exports.run = async function ({ inputs, appId, emitter }) {
   }
 
   // have to clean up the row, remove the table from it
-  const ctx = {
+  const ctx = buildCtx(appId, emitter, {
+    body: {
+      ...inputs.row,
+      _id: inputs.rowId,
+    },
     params: {
       rowId: inputs.rowId,
     },
-    request: {
-      body: {
-        ...inputs.row,
-        _id: inputs.rowId,
-      },
-    },
-    appId,
-    eventEmitter: emitter,
-  }
+  })
 
   try {
     inputs.row = await automationUtils.cleanUpRowById(

package/src/db/inMemoryView.js

@@ -0,0 +1,48 @@
+const PouchDB = require("pouchdb")
+const memory = require("pouchdb-adapter-memory")
+const newid = require("./newid")
+
+PouchDB.plugin(memory)
+const Pouch = PouchDB.defaults({
+  prefix: undefined,
+  adapter: "memory",
+})
+
+exports.runView = async (view, calculation, group, data) => {
+  // use a different ID each time for the DB, make sure they
+  // are always unique for each query, don't want overlap
+  // which could cause 409s
+  const db = new Pouch(newid())
+  // write all the docs to the in memory Pouch (remove revs)
+  await db.bulkDocs(
+    data.map(row => ({
+      ...row,
+      _rev: undefined,
+    }))
+  )
+  let fn = (doc, emit) => emit(doc._id)
+  eval("fn = " + view.map.replace("function (doc)", "function (doc, emit)"))
+  const queryFns = {
+    meta: view.meta,
+    map: fn,
+  }
+  if (view.reduce) {
+    queryFns.reduce = view.reduce
+  }
+  const response = await db.query(queryFns, {
+    include_docs: !calculation,
+    group: !!group,
+  })
+  // need to fix the revs to be totally accurate
+  for (let row of response.rows) {
+    if (!row._rev || !row._id) {
+      continue
+    }
+    const found = data.find(possible => possible._id === row._id)
+    if (found) {
+      row._rev = found._rev
+    }
+  }
+  await db.destroy()
+  return response
+}

package/src/db/linkedRows/index.js

@@ -76,9 +76,12 @@ async function getFullLinkedDocs(ctx, appId, links) {
   // create DBs
   const db = new CouchDB(appId)
   const linkedRowIds = links.map(link => link.id)
-  let linked = (await db.allDocs(getMultiIDParams(linkedRowIds))).rows.map(
+  const uniqueRowIds = [...new Set(linkedRowIds)]
+  let dbRows = (await db.allDocs(getMultiIDParams(uniqueRowIds))).rows.map(
     row => row.doc
   )
+  // convert the unique db rows back to a full list of linked rows
+  const linked = linkedRowIds.map(id => dbRows.find(row => row._id === id))
   // need to handle users as specific cases
   let [users, other] = partition(linked, linkRow =>
     linkRow._id.startsWith(USER_METDATA_PREFIX)
@@ -112,7 +115,7 @@ exports.updateLinks = async function (args) {
   let linkController = new LinkController(args)
   try {
     if (
-      !(await linkController.doesTableHaveLinkedFields()) &&
+      !(await linkController.doesTableHaveLinkedFields(table)) &&
       (oldTable == null ||
         !(await linkController.doesTableHaveLinkedFields(oldTable)))
     ) {

package/src/db/utils.js

@@ -39,6 +39,7 @@ const DocumentTypes = {
   QUERY: "query",
   DEPLOYMENTS: "deployments",
   METADATA: "metadata",
+  MEM_VIEW: "view",
 }
 
 const ViewNames = {
@@ -348,6 +349,14 @@ exports.getMetadataParams = (type, entityId = null, otherProps = {}) => {
   return getDocParams(DocumentTypes.METADATA, docId, otherProps)
 }
 
+exports.generateMemoryViewID = viewName => {
+  return `${DocumentTypes.MEM_VIEW}${SEPARATOR}${viewName}`
+}
+
+exports.getMemoryViewParams = (otherProps = {}) => {
+  return getDocParams(DocumentTypes.MEM_VIEW, null, otherProps)
+}
+
 /**
  * This can be used with the db.allDocs to get a list of IDs
  */

package/src/environment.js

@@ -26,7 +26,7 @@ module.exports = {
   COUCH_DB_URL: process.env.COUCH_DB_URL,
   MINIO_URL: process.env.MINIO_URL,
   WORKER_URL: process.env.WORKER_URL,
-  SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
+  SELF_HOSTED: process.env.SELF_HOSTED,
   AWS_REGION: process.env.AWS_REGION,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
@@ -66,3 +66,10 @@ module.exports = {
     return !isDev()
   },
 }
+
+// convert any strings to numbers if required, like "0" would be true otherwise
+for (let [key, value] of Object.entries(module.exports)) {
+  if (typeof value === "string" && !isNaN(parseInt(value))) {
+    module.exports[key] = parseInt(value)
+  }
+}

package/src/integrations/mysql.ts

@@ -12,7 +12,7 @@ import { getSqlQuery } from "./utils"
 module MySQLModule {
   const mysql = require("mysql")
   const Sql = require("./base/sql")
-  const { buildExternalTableId, convertType } = require("./utils")
+  const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils")
   const { FieldTypes } = require("../constants")
 
   interface MySQLConfig {
@@ -194,18 +194,7 @@ module MySQLModule {
           }
         }
 
-        // add the existing relationships from the entities if they exist, to prevent them from being overridden
-        if (entities && entities[tableName]) {
-          const existingTableSchema = entities[tableName].schema
-          for (let key in existingTableSchema) {
-            if (!existingTableSchema.hasOwnProperty(key)) {
-              continue
-            }
-            if (existingTableSchema[key].type === "link") {
-              tables[tableName].schema[key] = existingTableSchema[key]
-            }
-          }
-        }
+        copyExistingPropsOver(tableName, tables, entities)
       }
 
       this.client.end()

package/src/integrations/postgres.ts

@@ -12,7 +12,7 @@ module PostgresModule {
   const { Pool } = require("pg")
   const Sql = require("./base/sql")
   const { FieldTypes } = require("../constants")
-  const { buildExternalTableId, convertType } = require("./utils")
+  const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils")
 
   interface PostgresConfig {
     host: string
@@ -173,31 +173,24 @@ module PostgresModule {
             name: tableName,
             schema: {},
           }
-
-          // add the existing relationships from the entities if they exist, to prevent them from being overridden
-          if (entities && entities[tableName]) {
-            const existingTableSchema = entities[tableName].schema
-            for (let key in existingTableSchema) {
-              if (!existingTableSchema.hasOwnProperty(key)) {
-                continue
-              }
-              if (existingTableSchema[key].type === "link") {
-                tables[tableName].schema[key] = existingTableSchema[key]
-              }
-            }
-          }
         }
 
         const type: string = convertType(column.data_type, TYPE_MAP)
-        const isAuto: boolean =
-          typeof column.column_default === "string" &&
+        const identity = !!(column.identity_generation || column.identity_start || column.identity_increment)
+        const hasDefault = typeof column.column_default === "string" &&
           column.column_default.startsWith("nextval")
+        const isGenerated = column.is_generated && column.is_generated !== "NEVER"
+        const isAuto: boolean = hasDefault || identity || isGenerated
         tables[tableName].schema[columnName] = {
           autocolumn: isAuto,
           name: columnName,
           type,
         }
       }
+
+      for (let tableName of Object.keys(tables)) {
+        copyExistingPropsOver(tableName, tables, entities)
+      }
       this.tables = tables
     }
 

package/src/integrations/utils.ts

@@ -82,3 +82,21 @@ export function isIsoDateString(str: string) {
   let d = new Date(str)
   return d.toISOString() === str
 }
+
+// add the existing relationships from the entities if they exist, to prevent them from being overridden
+export function copyExistingPropsOver(tableName: string, tables: { [key: string]: any }, entities: { [key: string]: any }) {
+  if (entities && entities[tableName]) {
+    if (entities[tableName].primaryDisplay) {
+      tables[tableName].primaryDisplay = entities[tableName].primaryDisplay
+    }
+    const existingTableSchema = entities[tableName].schema
+    for (let key in existingTableSchema) {
+      if (!existingTableSchema.hasOwnProperty(key)) {
+        continue
+      }
+      if (existingTableSchema[key].type === "link") {
+        tables[tableName].schema[key] = existingTableSchema[key]
+      }
+    }
+  }
+}

package/src/utilities/rowProcessor/index.js

@@ -89,10 +89,16 @@ const TYPE_TRANSFORM_MAP = {
  * @param {Object} user The user to be used for an appId as well as the createdBy and createdAt fields.
  * @param {Object} table The table which is to be used for the schema, as well as handling auto IDs incrementing.
  * @param {Object} row The row which is to be updated with information for the auto columns.
+ * @param {Object} opts specific options for function to carry out optional features.
  * @returns {{row: Object, table: Object}} The updated row and table, the table may need to be updated
  * for automatic ID purposes.
  */
-function processAutoColumn(user, table, row) {
+function processAutoColumn(
+  user,
+  table,
+  row,
+  opts = { reprocessing: false, noAutoRelationships: false }
+) {
   let now = new Date().toISOString()
   // if a row doesn't have a revision then it doesn't exist yet
   const creating = !row._rev
@@ -102,7 +108,7 @@ function processAutoColumn(user, table, row) {
     }
     switch (schema.subtype) {
       case AutoFieldSubTypes.CREATED_BY:
-        if (creating) {
+        if (creating && !opts.reprocessing && !opts.noAutoRelationships) {
           row[key] = [user.userId]
         }
         break
@@ -112,7 +118,9 @@ function processAutoColumn(user, table, row) {
         }
         break
       case AutoFieldSubTypes.UPDATED_BY:
-        row[key] = [user.userId]
+        if (!opts.reprocessing && !opts.noAutoRelationships) {
+          row[key] = [user.userId]
+        }
         break
       case AutoFieldSubTypes.UPDATED_AT:
         row[key] = now
@@ -127,6 +135,7 @@ function processAutoColumn(user, table, row) {
   }
   return { table, row }
 }
+exports.processAutoColumn = processAutoColumn
 
 /**
  * This will coerce a value to the correct types based on the type transform map
@@ -151,9 +160,15 @@ exports.coerce = (row, type) => {
  * @param {object} user the user which is performing the input.
  * @param {object} row the row which is being created/updated.
  * @param {object} table the table which the row is being saved to.
+ * @param {object} opts some input processing options (like disabling auto-column relationships).
  * @returns {object} the row which has been prepared to be written to the DB.
  */
-exports.inputProcessing = (user = {}, table, row) => {
+exports.inputProcessing = (
+  user = {},
+  table,
+  row,
+  opts = { noAutoRelationships: false }
+) => {
   let clonedRow = cloneDeep(row)
   // need to copy the table so it can be differenced on way out
   const copiedTable = cloneDeep(table)
@@ -176,7 +191,7 @@ exports.inputProcessing = (user = {}, table, row) => {
     }
   }
   // handle auto columns - this returns an object like {table, row}
-  return processAutoColumn(user, copiedTable, clonedRow)
+  return processAutoColumn(user, copiedTable, clonedRow, opts)
 }
 
 /**