@budibase/frontend-core 2.4.12-alpha.0 → 2.4.12

package/package.json

@@ -1,15 +1,14 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.4.12-alpha.0",
+  "version": "2.4.12",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "2.4.12-alpha.0",
-    "@budibase/shared-core": "2.4.12-alpha.0",
+    "@budibase/bbui": "^2.4.12",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   },
-  "gitHead": "b5b20e239ee8e5d1172dca2a16538587a58a91e4"
+  "gitHead": "ac4c0b3dd60c901691c7644c2f9aa6a5458b03f4"
 }

package/src/constants.js

@@ -1,7 +1,56 @@
 /**
  * Operator options for lucene queries
  */
-export { OperatorOptions, SqlNumberTypeRangeMap } from "@budibase/shared-core"
+export const OperatorOptions = {
+  Equals: {
+    value: "equal",
+    label: "Equals",
+  },
+  NotEquals: {
+    value: "notEqual",
+    label: "Not equals",
+  },
+  Empty: {
+    value: "empty",
+    label: "Is empty",
+  },
+  NotEmpty: {
+    value: "notEmpty",
+    label: "Is not empty",
+  },
+  StartsWith: {
+    value: "string",
+    label: "Starts with",
+  },
+  Like: {
+    value: "fuzzy",
+    label: "Like",
+  },
+  MoreThan: {
+    value: "rangeLow",
+    label: "More than or equal to",
+  },
+  LessThan: {
+    value: "rangeHigh",
+    label: "Less than or equal to",
+  },
+  Contains: {
+    value: "contains",
+    label: "Contains",
+  },
+  NotContains: {
+    value: "notContains",
+    label: "Does not contain",
+  },
+  In: {
+    value: "oneOf",
+    label: "Is in",
+  },
+  ContainsAny: {
+    value: "containsAny",
+    label: "Has any",
+  },
+}
 
 // Cookie names
 export const Cookies = {
@@ -78,6 +127,27 @@ export const Roles = {
   PUBLIC: "PUBLIC",
   BUILDER: "BUILDER",
 }
+/**
+ * Maximum minimum range for SQL number values
+ */
+export const SqlNumberTypeRangeMap = {
+  integer: {
+    max: 2147483647,
+    min: -2147483648,
+  },
+  int: {
+    max: 2147483647,
+    min: -2147483648,
+  },
+  smallint: {
+    max: 32767,
+    min: -32768,
+  },
+  mediumint: {
+    max: 8388607,
+    min: -8388608,
+  },
+}
 
 export const Themes = [
   {

package/src/fetch/DataFetch.js

@@ -1,11 +1,13 @@
 import { writable, derived, get } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
-import { LuceneUtils } from "../utils"
+import {
+  buildLuceneQuery,
+  luceneLimit,
+  runLuceneQuery,
+  luceneSort,
+} from "../utils/lucene"
 import { convertJSONSchemaToTableSchema } from "../utils/json"
 
-const { buildLuceneQuery, luceneLimit, runLuceneQuery, luceneSort } =
-  LuceneUtils
-
 /**
  * Parent class which handles the implementation of fetching data from an
  * internal table or datasource plus.

package/src/utils/index.js

@@ -1,4 +1,4 @@
-export { dataFilters as LuceneUtils } from "@budibase/shared-core"
+export * as LuceneUtils from "./lucene"
 export * as JSONUtils from "./json"
 export * as CookieUtils from "./cookies"
 export * as RoleUtils from "./roles"

package/src/utils/lucene.js

@@ -0,0 +1,360 @@
+import { Helpers } from "@budibase/bbui"
+import { OperatorOptions, SqlNumberTypeRangeMap } from "../constants"
+
+const HBS_REGEX = /{{([^{].*?)}}/g
+
+/**
+ * Returns the valid operator options for a certain data type
+ * @param type the data type
+ */
+export const getValidOperatorsForType = (type, field, datasource) => {
+  const Op = OperatorOptions
+  const stringOps = [
+    Op.Equals,
+    Op.NotEquals,
+    Op.StartsWith,
+    Op.Like,
+    Op.Empty,
+    Op.NotEmpty,
+    Op.In,
+  ]
+  const numOps = [
+    Op.Equals,
+    Op.NotEquals,
+    Op.MoreThan,
+    Op.LessThan,
+    Op.Empty,
+    Op.NotEmpty,
+    Op.In,
+  ]
+  let ops = []
+  if (type === "string") {
+    ops = stringOps
+  } else if (type === "number") {
+    ops = numOps
+  } else if (type === "options") {
+    ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In]
+  } else if (type === "array") {
+    ops = [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty, Op.ContainsAny]
+  } else if (type === "boolean") {
+    ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
+  } else if (type === "longform") {
+    ops = stringOps
+  } else if (type === "datetime") {
+    ops = numOps
+  } else if (type === "formula") {
+    ops = stringOps.concat([Op.MoreThan, Op.LessThan])
+  }
+
+  // Filter out "like" for internal tables
+  const externalTable = datasource?.tableId?.includes("datasource_plus")
+  if (datasource?.type === "table" && !externalTable) {
+    ops = ops.filter(x => x !== Op.Like)
+  }
+
+  // Only allow equal/not equal for _id in SQL tables
+  if (field === "_id" && externalTable) {
+    ops = [Op.Equals, Op.NotEquals]
+  }
+
+  return ops
+}
+
+/**
+ * Operators which do not support empty strings as values
+ */
+export const NoEmptyFilterStrings = [
+  OperatorOptions.StartsWith.value,
+  OperatorOptions.Like.value,
+  OperatorOptions.Equals.value,
+  OperatorOptions.NotEquals.value,
+  OperatorOptions.Contains.value,
+  OperatorOptions.NotContains.value,
+]
+
+/**
+ * Removes any fields that contain empty strings that would cause inconsistent
+ * behaviour with how backend tables are filtered (no value means no filter).
+ */
+const cleanupQuery = query => {
+  if (!query) {
+    return query
+  }
+  for (let filterField of NoEmptyFilterStrings) {
+    if (!query[filterField]) {
+      continue
+    }
+    for (let [key, value] of Object.entries(query[filterField])) {
+      if (value == null || value === "") {
+        delete query[filterField][key]
+      }
+    }
+  }
+  return query
+}
+
+/**
+ * Removes a numeric prefix on field names designed to give fields uniqueness
+ */
+const removeKeyNumbering = key => {
+  if (typeof key === "string" && key.match(/\d[0-9]*:/g) != null) {
+    const parts = key.split(":")
+    parts.shift()
+    return parts.join(":")
+  } else {
+    return key
+  }
+}
+
+/**
+ * Builds a lucene JSON query from the filter structure generated in the builder
+ * @param filter the builder filter structure
+ */
+export const buildLuceneQuery = filter => {
+  let query = {
+    string: {},
+    fuzzy: {},
+    range: {},
+    equal: {},
+    notEqual: {},
+    empty: {},
+    notEmpty: {},
+    contains: {},
+    notContains: {},
+    oneOf: {},
+    containsAny: {},
+  }
+  if (Array.isArray(filter)) {
+    filter.forEach(expression => {
+      let { operator, field, type, value, externalType } = expression
+      const isHbs =
+        typeof value === "string" && value.match(HBS_REGEX)?.length > 0
+      // Parse all values into correct types
+      if (operator === "allOr") {
+        query.allOr = true
+        return
+      }
+      if (
+        type === "datetime" &&
+        !isHbs &&
+        operator !== "empty" &&
+        operator !== "notEmpty"
+      ) {
+        // Ensure date value is a valid date and parse into correct format
+        if (!value) {
+          return
+        }
+        try {
+          value = new Date(value).toISOString()
+        } catch (error) {
+          return
+        }
+      }
+      if (type === "number" && typeof value === "string") {
+        if (operator === "oneOf") {
+          value = value.split(",").map(item => parseFloat(item))
+        } else if (!isHbs) {
+          value = parseFloat(value)
+        }
+      }
+      if (type === "boolean") {
+        value = `${value}`?.toLowerCase() === "true"
+      }
+      if (
+        ["contains", "notContains", "containsAny"].includes(operator) &&
+        type === "array" &&
+        typeof value === "string"
+      ) {
+        value = value.split(",")
+      }
+      if (operator.startsWith("range")) {
+        const minint =
+          SqlNumberTypeRangeMap[externalType]?.min || Number.MIN_SAFE_INTEGER
+        const maxint =
+          SqlNumberTypeRangeMap[externalType]?.max || Number.MAX_SAFE_INTEGER
+        if (!query.range[field]) {
+          query.range[field] = {
+            low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
+            high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
+          }
+        }
+        if (operator === "rangeLow" && value != null && value !== "") {
+          query.range[field].low = value
+        } else if (operator === "rangeHigh" && value != null && value !== "") {
+          query.range[field].high = value
+        }
+      } else if (query[operator]) {
+        if (type === "boolean") {
+          // Transform boolean filters to cope with null.
+          // "equals false" needs to be "not equals true"
+          // "not equals false" needs to be "equals true"
+          if (operator === "equal" && value === false) {
+            query.notEqual[field] = true
+          } else if (operator === "notEqual" && value === false) {
+            query.equal[field] = true
+          } else {
+            query[operator][field] = value
+          }
+        } else {
+          query[operator][field] = value
+        }
+      }
+    })
+  }
+  return query
+}
+
+/**
+ * Performs a client-side lucene search on an array of data
+ * @param docs the data
+ * @param query the JSON lucene query
+ */
+export const runLuceneQuery = (docs, query) => {
+  if (!docs || !Array.isArray(docs)) {
+    return []
+  }
+  if (!query) {
+    return docs
+  }
+
+  // Make query consistent first
+  query = cleanupQuery(query)
+
+  // Iterates over a set of filters and evaluates a fail function against a doc
+  const match = (type, failFn) => doc => {
+    const filters = Object.entries(query[type] || {})
+    for (let i = 0; i < filters.length; i++) {
+      const [key, testValue] = filters[i]
+      const docValue = Helpers.deepGet(doc, removeKeyNumbering(key))
+      if (failFn(docValue, testValue)) {
+        return false
+      }
+    }
+    return true
+  }
+
+  // Process a string match (fails if the value does not start with the string)
+  const stringMatch = match("string", (docValue, testValue) => {
+    return (
+      !docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase())
+    )
+  })
+
+  // Process a fuzzy match (treat the same as starts with when running locally)
+  const fuzzyMatch = match("fuzzy", (docValue, testValue) => {
+    return (
+      !docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase())
+    )
+  })
+
+  // Process a range match
+  const rangeMatch = match("range", (docValue, testValue) => {
+    return (
+      docValue == null ||
+      docValue === "" ||
+      docValue < testValue.low ||
+      docValue > testValue.high
+    )
+  })
+
+  // Process an equal match (fails if the value is different)
+  const equalMatch = match("equal", (docValue, testValue) => {
+    return testValue != null && testValue !== "" && docValue !== testValue
+  })
+
+  // Process a not-equal match (fails if the value is the same)
+  const notEqualMatch = match("notEqual", (docValue, testValue) => {
+    return testValue != null && testValue !== "" && docValue === testValue
+  })
+
+  // Process an empty match (fails if the value is not empty)
+  const emptyMatch = match("empty", docValue => {
+    return docValue != null && docValue !== ""
+  })
+
+  // Process a not-empty match (fails is the value is empty)
+  const notEmptyMatch = match("notEmpty", docValue => {
+    return docValue == null || docValue === ""
+  })
+
+  // Process an includes match (fails if the value is not included)
+  const oneOf = match("oneOf", (docValue, testValue) => {
+    if (typeof testValue === "string") {
+      testValue = testValue.split(",")
+      if (typeof docValue === "number") {
+        testValue = testValue.map(item => parseFloat(item))
+      }
+    }
+    return !testValue?.includes(docValue)
+  })
+
+  const containsAny = match("containsAny", (docValue, testValue) => {
+    return !docValue?.includes(...testValue)
+  })
+
+  const contains = match("contains", (docValue, testValue) => {
+    return !testValue?.every(item => docValue?.includes(item))
+  })
+
+  const notContains = match("notContains", (docValue, testValue) => {
+    return testValue?.every(item => docValue?.includes(item))
+  })
+
+  // Match a document against all criteria
+  const docMatch = doc => {
+    return (
+      stringMatch(doc) &&
+      fuzzyMatch(doc) &&
+      rangeMatch(doc) &&
+      equalMatch(doc) &&
+      notEqualMatch(doc) &&
+      emptyMatch(doc) &&
+      notEmptyMatch(doc) &&
+      oneOf(doc) &&
+      contains(doc) &&
+      containsAny(doc) &&
+      notContains(doc)
+    )
+  }
+
+  // Process all docs
+  return docs.filter(docMatch)
+}
+
+/**
+ * Performs a client-side sort from the equivalent server-side lucene sort
+ * parameters.
+ * @param docs the data
+ * @param sort the sort column
+ * @param sortOrder the sort order ("ascending" or "descending")
+ * @param sortType the type of sort ("string" or "number")
+ */
+export const luceneSort = (docs, sort, sortOrder, sortType = "string") => {
+  if (!sort || !sortOrder || !sortType) {
+    return docs
+  }
+  const parse = sortType === "string" ? x => `${x}` : x => parseFloat(x)
+  return docs.slice().sort((a, b) => {
+    const colA = parse(a[sort])
+    const colB = parse(b[sort])
+    if (sortOrder === "Descending") {
+      return colA > colB ? -1 : 1
+    } else {
+      return colA > colB ? 1 : -1
+    }
+  })
+}
+
+/**
+ * Limits the specified docs to the specified number of rows from the equivalent
+ * server-side lucene limit parameters.
+ * @param docs the data
+ * @param limit the number of docs to limit to
+ */
+export const luceneLimit = (docs, limit) => {
+  const numLimit = parseFloat(limit)
+  if (isNaN(numLimit)) {
+    return docs
+  }
+  return docs.slice(0, numLimit)
+}