@budibase/server 0.9.187 → 0.9.190-alpha.1

package/src/integrations/base/sql.ts

@@ -1,265 +1,333 @@
 import { Knex, knex } from "knex"
-const BASE_LIMIT = 5000
 import {
+  Operation,
   QueryJson,
-  SearchFilters,
   QueryOptions,
-  SortDirection,
-  Operation,
   RelationshipsJson,
+  SearchFilters,
+  SortDirection,
 } from "../../definitions/datasource"
-import { isIsoDateString } from "../utils"
+import { isIsoDateString, SqlClients } from "../utils"
+import SqlTableQueryBuilder from "./sqlTable"
+
+const BASE_LIMIT = 5000
 
 type KnexQuery = Knex.QueryBuilder | Knex
+// these are invalid dates sent by the client, need to convert them to a real max date
+const MIN_ISO_DATE = "0000-00-00T00:00:00.000Z"
+const MAX_ISO_DATE = "9999-00-00T00:00:00.000Z"
+
+function parse(input: any) {
+  if (Array.isArray(input)) {
+    return JSON.stringify(input)
+  }
+  if (input == undefined) {
+    return null
+  }
+  if (typeof input !== "string") {
+    return input
+  }
+  if (input === MAX_ISO_DATE) {
+    return new Date(8640000000000000)
+  }
+  if (input === MIN_ISO_DATE) {
+    return new Date(-8640000000000000)
+  }
+  if (isIsoDateString(input)) {
+    return new Date(input)
+  }
+  return input
+}
 
 function parseBody(body: any) {
   for (let [key, value] of Object.entries(body)) {
-    if (typeof value !== "string") {
-      continue
-    }
-    if (isIsoDateString(value)) {
-      body[key] = new Date(value)
-    }
+    body[key] = parse(value)
   }
   return body
 }
 
-// right now we only do filters on the specific table being queried
-function addFilters(
-  tableName: string,
-  query: KnexQuery,
-  filters: SearchFilters | undefined
-): KnexQuery {
-  function iterate(
-    structure: { [key: string]: any },
-    fn: (key: string, value: any) => void
-  ) {
-    for (let [key, value] of Object.entries(structure)) {
-      fn(`${tableName}.${key}`, value)
-    }
-  }
+function parseFilters(filters: SearchFilters | undefined): SearchFilters {
   if (!filters) {
-    return query
-  }
-  // if all or specified in filters, then everything is an or
-  const allOr = filters.allOr
-  if (filters.oneOf) {
-    iterate(filters.oneOf, (key, array) => {
-      const fnc = allOr ? "orWhereIn" : "whereIn"
-      query = query[fnc](key, array)
-    })
+    return {}
   }
-  if (filters.string) {
-    iterate(filters.string, (key, value) => {
-      const fnc = allOr ? "orWhere" : "where"
-      query = query[fnc](key, "ilike", `${value}%`)
-    })
+  for (let [key, value] of Object.entries(filters)) {
+    let parsed
+    if (typeof value === "object") {
+      parsed = parseFilters(value)
+    } else {
+      parsed = parse(value)
+    }
+    // @ts-ignore
+    filters[key] = parsed
   }
-  if (filters.fuzzy) {
-    iterate(filters.fuzzy, (key, value) => {
-      const fnc = allOr ? "orWhere" : "where"
-      query = query[fnc](key, "ilike", `%${value}%`)
-    })
+  return filters
+}
+
+class InternalBuilder {
+  private readonly client: string
+
+  constructor(client: string) {
+    this.client = client
   }
-  if (filters.range) {
-    iterate(filters.range, (key, value) => {
-      if (!value.high || !value.low) {
-        return
+
+  // right now we only do filters on the specific table being queried
+  addFilters(
+    tableName: string,
+    query: KnexQuery,
+    filters: SearchFilters | undefined
+  ): KnexQuery {
+    function iterate(
+      structure: { [key: string]: any },
+      fn: (key: string, value: any) => void
+    ) {
+      for (let [key, value] of Object.entries(structure)) {
+        fn(`${tableName}.${key}`, value)
       }
-      const fnc = allOr ? "orWhereBetween" : "whereBetween"
-      query = query[fnc](key, [value.low, value.high])
-    })
-  }
-  if (filters.equal) {
-    iterate(filters.equal, (key, value) => {
-      const fnc = allOr ? "orWhere" : "where"
-      query = query[fnc]({ [key]: value })
-    })
-  }
-  if (filters.notEqual) {
-    iterate(filters.notEqual, (key, value) => {
-      const fnc = allOr ? "orWhereNot" : "whereNot"
-      query = query[fnc]({ [key]: value })
-    })
-  }
-  if (filters.empty) {
-    iterate(filters.empty, key => {
-      const fnc = allOr ? "orWhereNull" : "whereNull"
-      query = query[fnc](key)
-    })
-  }
-  if (filters.notEmpty) {
-    iterate(filters.notEmpty, key => {
-      const fnc = allOr ? "orWhereNotNull" : "whereNotNull"
-      query = query[fnc](key)
-    })
+    }
+    if (!filters) {
+      return query
+    }
+    filters = parseFilters(filters)
+    // if all or specified in filters, then everything is an or
+    const allOr = filters.allOr
+    if (filters.oneOf) {
+      iterate(filters.oneOf, (key, array) => {
+        const fnc = allOr ? "orWhereIn" : "whereIn"
+        query = query[fnc](key, array)
+      })
+    }
+    if (filters.string) {
+      iterate(filters.string, (key, value) => {
+        const fnc = allOr ? "orWhere" : "where"
+        // postgres supports ilike, nothing else does
+        if (this.client === SqlClients.POSTGRES) {
+          query = query[fnc](key, "ilike", `${value}%`)
+        } else {
+          const rawFnc = `${fnc}Raw`
+          // @ts-ignore
+          query = query[rawFnc](`LOWER(${key}) LIKE ?`, [`${value}%`])
+        }
+      })
+    }
+    if (filters.fuzzy) {
+      iterate(filters.fuzzy, (key, value) => {
+        const fnc = allOr ? "orWhere" : "where"
+        // postgres supports ilike, nothing else does
+        if (this.client === SqlClients.POSTGRES) {
+          query = query[fnc](key, "ilike", `%${value}%`)
+        } else {
+          const rawFnc = `${fnc}Raw`
+          // @ts-ignore
+          query = query[rawFnc](`LOWER(${key}) LIKE ?`, [`%${value}%`])
+        }
+      })
+    }
+    if (filters.range) {
+      iterate(filters.range, (key, value) => {
+        if (!value.high || !value.low) {
+          return
+        }
+        const fnc = allOr ? "orWhereBetween" : "whereBetween"
+        query = query[fnc](key, [value.low, value.high])
+      })
+    }
+    if (filters.equal) {
+      iterate(filters.equal, (key, value) => {
+        const fnc = allOr ? "orWhere" : "where"
+        query = query[fnc]({ [key]: value })
+      })
+    }
+    if (filters.notEqual) {
+      iterate(filters.notEqual, (key, value) => {
+        const fnc = allOr ? "orWhereNot" : "whereNot"
+        query = query[fnc]({ [key]: value })
+      })
+    }
+    if (filters.empty) {
+      iterate(filters.empty, key => {
+        const fnc = allOr ? "orWhereNull" : "whereNull"
+        query = query[fnc](key)
+      })
+    }
+    if (filters.notEmpty) {
+      iterate(filters.notEmpty, key => {
+        const fnc = allOr ? "orWhereNotNull" : "whereNotNull"
+        query = query[fnc](key)
+      })
+    }
+    return query
   }
-  return query
-}
 
-function addRelationships(
-  knex: Knex,
-  query: KnexQuery,
-  fields: string | string[],
-  fromTable: string,
-  relationships: RelationshipsJson[] | undefined
-): KnexQuery {
-  if (!relationships) {
+  addSorting(query: KnexQuery, json: QueryJson): KnexQuery {
+    let { sort, paginate } = json
+    if (!sort) {
+      return query
+    }
+    const table = json.meta?.table
+    for (let [key, value] of Object.entries(sort)) {
+      const direction = value === SortDirection.ASCENDING ? "asc" : "desc"
+      query = query.orderBy(`${table?.name}.${key}`, direction)
+    }
+    if (this.client === SqlClients.MS_SQL && !sort && paginate?.limit) {
+      // @ts-ignore
+      query = query.orderBy(`${table?.name}.${table?.primary[0]}`)
+    }
     return query
   }
-  for (let relationship of relationships) {
-    const from = relationship.from,
-      to = relationship.to,
-      toTable = relationship.tableName
-    if (!relationship.through) {
-      // @ts-ignore
-      query = query.leftJoin(
-        toTable,
-        `${fromTable}.${from}`,
-        `${toTable}.${to}`
-      )
-    } else {
-      const throughTable = relationship.through
-      const fromPrimary = relationship.fromPrimary
-      const toPrimary = relationship.toPrimary
-      query = query
+
+  addRelationships(
+    knex: Knex,
+    query: KnexQuery,
+    fields: string | string[],
+    fromTable: string,
+    relationships: RelationshipsJson[] | undefined
+  ): KnexQuery {
+    if (!relationships) {
+      return query
+    }
+    for (let relationship of relationships) {
+      const from = relationship.from,
+        to = relationship.to,
+        toTable = relationship.tableName
+      if (!relationship.through) {
         // @ts-ignore
-        .leftJoin(
-          throughTable,
-          `${fromTable}.${fromPrimary}`,
-          `${throughTable}.${from}`
+        query = query.leftJoin(
+          toTable,
+          `${fromTable}.${from}`,
+          `${toTable}.${to}`
         )
-        .leftJoin(toTable, `${toTable}.${toPrimary}`, `${throughTable}.${to}`)
+      } else {
+        const throughTable = relationship.through
+        const fromPrimary = relationship.fromPrimary
+        const toPrimary = relationship.toPrimary
+        query = query
+          // @ts-ignore
+          .leftJoin(
+            throughTable,
+            `${fromTable}.${fromPrimary}`,
+            `${throughTable}.${from}`
+          )
+          .leftJoin(toTable, `${toTable}.${toPrimary}`, `${throughTable}.${to}`)
+      }
     }
+    return query.limit(BASE_LIMIT)
   }
-  return query.limit(BASE_LIMIT)
-}
 
-function buildCreate(
-  knex: Knex,
-  json: QueryJson,
-  opts: QueryOptions
-): KnexQuery {
-  const { endpoint, body } = json
-  let query: KnexQuery = knex(endpoint.entityId)
-  const parsedBody = parseBody(body)
-  // make sure no null values in body for creation
-  for (let [key, value] of Object.entries(parsedBody)) {
-    if (value == null) {
-      delete parsedBody[key]
+  create(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
+    const { endpoint, body } = json
+    let query: KnexQuery = knex(endpoint.entityId)
+    const parsedBody = parseBody(body)
+    // make sure no null values in body for creation
+    for (let [key, value] of Object.entries(parsedBody)) {
+      if (value == null) {
+        delete parsedBody[key]
+      }
+    }
+    // mysql can't use returning
+    if (opts.disableReturning) {
+      return query.insert(parsedBody)
+    } else {
+      return query.insert(parsedBody).returning("*")
     }
   }
-  // mysql can't use returning
-  if (opts.disableReturning) {
+
+  bulkCreate(knex: Knex, json: QueryJson): KnexQuery {
+    const { endpoint, body } = json
+    let query: KnexQuery = knex(endpoint.entityId)
+    if (!Array.isArray(body)) {
+      return query
+    }
+    const parsedBody = body.map(row => parseBody(row))
     return query.insert(parsedBody)
-  } else {
-    return query.insert(parsedBody).returning("*")
   }
-}
 
-function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery {
-  let { endpoint, resource, filters, sort, paginate, relationships } = json
-  const tableName = endpoint.entityId
-  // select all if not specified
-  if (!resource) {
-    resource = { fields: [] }
-  }
-  let selectStatement: string | string[] = "*"
-  // handle select
-  if (resource.fields && resource.fields.length > 0) {
-    // select the resources as the format "table.columnName" - this is what is provided
-    // by the resource builder further up
-    selectStatement = resource.fields.map(field => `${field} as ${field}`)
-  }
-  let foundLimit = limit || BASE_LIMIT
-  // handle pagination
-  let foundOffset: number | null = null
-  if (paginate && paginate.page && paginate.limit) {
+  read(knex: Knex, json: QueryJson, limit: number): KnexQuery {
+    let { endpoint, resource, filters, sort, paginate, relationships } = json
+    const tableName = endpoint.entityId
+    // select all if not specified
+    if (!resource) {
+      resource = { fields: [] }
+    }
+    let selectStatement: string | string[] = "*"
+    // handle select
+    if (resource.fields && resource.fields.length > 0) {
+      // select the resources as the format "table.columnName" - this is what is provided
+      // by the resource builder further up
+      selectStatement = resource.fields.map(field => `${field} as ${field}`)
+    }
+    let foundLimit = limit || BASE_LIMIT
+    // handle pagination
+    let foundOffset: number | null = null
+    if (paginate && paginate.page && paginate.limit) {
+      // @ts-ignore
+      const page = paginate.page <= 1 ? 0 : paginate.page - 1
+      const offset = page * paginate.limit
+      foundLimit = paginate.limit
+      foundOffset = offset
+    } else if (paginate && paginate.limit) {
+      foundLimit = paginate.limit
+    }
+    // start building the query
+    let query: KnexQuery = knex(tableName).limit(foundLimit)
+    if (foundOffset) {
+      query = query.offset(foundOffset)
+    }
+    query = this.addFilters(tableName, query, filters)
+    // add sorting to pre-query
+    query = this.addSorting(query, json)
     // @ts-ignore
-    const page = paginate.page <= 1 ? 0 : paginate.page - 1
-    const offset = page * paginate.limit
-    foundLimit = paginate.limit
-    foundOffset = offset
-  } else if (paginate && paginate.limit) {
-    foundLimit = paginate.limit
-  }
-  // start building the query
-  let query: KnexQuery = knex(tableName).limit(foundLimit)
-  if (foundOffset) {
-    query = query.offset(foundOffset)
-  }
-  if (sort) {
-    for (let [key, value] of Object.entries(sort)) {
-      const direction = value === SortDirection.ASCENDING ? "asc" : "desc"
-      query = query.orderBy(key, direction)
+    let preQuery: KnexQuery = knex({
+      // @ts-ignore
+      [tableName]: query,
+    }).select(selectStatement)
+    // have to add after as well (this breaks MS-SQL)
+    if (this.client !== SqlClients.MS_SQL) {
+      preQuery = this.addSorting(preQuery, json)
     }
+    // handle joins
+    return this.addRelationships(
+      knex,
+      preQuery,
+      selectStatement,
+      tableName,
+      relationships
+    )
   }
-  query = addFilters(tableName, query, filters)
-  // @ts-ignore
-  let preQuery: KnexQuery = knex({
-    // @ts-ignore
-    [tableName]: query,
-  }).select(selectStatement)
-  // handle joins
-  return addRelationships(
-    knex,
-    preQuery,
-    selectStatement,
-    tableName,
-    relationships
-  )
-}
 
-function buildUpdate(
-  knex: Knex,
-  json: QueryJson,
-  opts: QueryOptions
-): KnexQuery {
-  const { endpoint, body, filters } = json
-  let query: KnexQuery = knex(endpoint.entityId)
-  const parsedBody = parseBody(body)
-  query = addFilters(endpoint.entityId, query, filters)
-  // mysql can't use returning
-  if (opts.disableReturning) {
-    return query.update(parsedBody)
-  } else {
-    return query.update(parsedBody).returning("*")
+  update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
+    const { endpoint, body, filters } = json
+    let query: KnexQuery = knex(endpoint.entityId)
+    const parsedBody = parseBody(body)
+    query = this.addFilters(endpoint.entityId, query, filters)
+    // mysql can't use returning
+    if (opts.disableReturning) {
+      return query.update(parsedBody)
+    } else {
+      return query.update(parsedBody).returning("*")
+    }
   }
-}
 
-function buildDelete(
-  knex: Knex,
-  json: QueryJson,
-  opts: QueryOptions
-): KnexQuery {
-  const { endpoint, filters } = json
-  let query: KnexQuery = knex(endpoint.entityId)
-  query = addFilters(endpoint.entityId, query, filters)
-  // mysql can't use returning
-  if (opts.disableReturning) {
-    return query.delete()
-  } else {
-    return query.delete().returning("*")
+  delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
+    const { endpoint, filters } = json
+    let query: KnexQuery = knex(endpoint.entityId)
+    query = this.addFilters(endpoint.entityId, query, filters)
+    // mysql can't use returning
+    if (opts.disableReturning) {
+      return query.delete()
+    } else {
+      return query.delete().returning("*")
+    }
   }
 }
 
-class SqlQueryBuilder {
-  private readonly sqlClient: string
+class SqlQueryBuilder extends SqlTableQueryBuilder {
   private readonly limit: number
   // pass through client to get flavour of SQL
   constructor(client: string, limit: number = BASE_LIMIT) {
-    this.sqlClient = client
+    super(client)
     this.limit = limit
   }
 
-  /**
-   * @param json the input JSON structure from which an SQL query will be built.
-   * @return {string} the operation that was found in the JSON.
-   */
-  _operation(json: QueryJson): Operation {
-    return json.endpoint.operation
-  }
-
   /**
    * @param json The JSON query DSL which is to be converted to SQL.
    * @param opts extra options which are to be passed into the query builder, e.g. disableReturning
@@ -267,21 +335,30 @@ class SqlQueryBuilder {
    * @return {{ sql: string, bindings: object }} the query ready to be passed to the driver.
    */
   _query(json: QueryJson, opts: QueryOptions = {}) {
-    const client = knex({ client: this.sqlClient })
+    const sqlClient = this.getSqlClient()
+    const client = knex({ client: sqlClient })
     let query
+    const builder = new InternalBuilder(sqlClient)
     switch (this._operation(json)) {
       case Operation.CREATE:
-        query = buildCreate(client, json, opts)
+        query = builder.create(client, json, opts)
         break
       case Operation.READ:
-        query = buildRead(client, json, this.limit)
+        query = builder.read(client, json, this.limit)
         break
       case Operation.UPDATE:
-        query = buildUpdate(client, json, opts)
+        query = builder.update(client, json, opts)
         break
       case Operation.DELETE:
-        query = buildDelete(client, json, opts)
+        query = builder.delete(client, json, opts)
         break
+      case Operation.BULK_CREATE:
+        query = builder.bulkCreate(client, json)
+        break
+      case Operation.CREATE_TABLE:
+      case Operation.UPDATE_TABLE:
+      case Operation.DELETE_TABLE:
+        return this._tableQuery(json)
       default:
         throw `Operation type is not supported by SQL query builder`
     }
@@ -289,6 +366,88 @@ class SqlQueryBuilder {
     // @ts-ignore
     return query.toSQL().toNative()
   }
+
+  async getReturningRow(queryFn: Function, json: QueryJson) {
+    if (!json.extra || !json.extra.idFilter) {
+      return {}
+    }
+    const input = this._query({
+      endpoint: {
+        ...json.endpoint,
+        operation: Operation.READ,
+      },
+      resource: {
+        fields: [],
+      },
+      filters: json.extra.idFilter,
+      paginate: {
+        limit: 1,
+      },
+      meta: json.meta,
+    })
+    return queryFn(input, Operation.READ)
+  }
+
+  // when creating if an ID has been inserted need to make sure
+  // the id filter is enriched with it before trying to retrieve the row
+  checkLookupKeys(id: any, json: QueryJson) {
+    if (!id || !json.meta?.table || !json.meta.table.primary) {
+      return json
+    }
+    const primaryKey = json.meta.table.primary?.[0]
+    json.extra = {
+      idFilter: {
+        equal: {
+          [primaryKey]: id,
+        },
+      },
+    }
+    return json
+  }
+
+  // this function recreates the returning functionality of postgres
+  async queryWithReturning(
+    json: QueryJson,
+    queryFn: Function,
+    processFn: Function = (result: any) => result
+  ) {
+    const sqlClient = this.getSqlClient()
+    const operation = this._operation(json)
+    const input = this._query(json, { disableReturning: true })
+    if (Array.isArray(input)) {
+      const responses = []
+      for (let query of input) {
+        responses.push(await queryFn(query, operation))
+      }
+      return responses
+    }
+    let row
+    // need to manage returning, a feature mySQL can't do
+    if (operation === Operation.DELETE) {
+      row = processFn(await this.getReturningRow(queryFn, json))
+    }
+    const response = await queryFn(input, operation)
+    const results = processFn(response)
+    // same as delete, manage returning
+    if (operation === Operation.CREATE || operation === Operation.UPDATE) {
+      let id
+      if (sqlClient === SqlClients.MS_SQL) {
+        id = results?.[0].id
+      } else if (sqlClient === SqlClients.MY_SQL) {
+        id = results?.insertId
+      } else if (sqlClient === SqlClients.ORACLE) {
+        id = response.outBinds[0][0]
+      }
+      row = processFn(
+        await this.getReturningRow(queryFn, this.checkLookupKeys(id, json))
+      )
+    }
+    if (operation !== Operation.READ) {
+      return row
+    }
+    return results.length ? results : [{ [operation.toLowerCase()]: true }]
+  }
 }
 
+export default SqlQueryBuilder
 module.exports = SqlQueryBuilder