@budibase/shared-core 0.0.1

package/src/constants.ts

@@ -0,0 +1,69 @@
+export const OperatorOptions = {
+  Equals: {
+    value: "equal",
+    label: "Equals",
+  },
+  NotEquals: {
+    value: "notEqual",
+    label: "Not equals",
+  },
+  Empty: {
+    value: "empty",
+    label: "Is empty",
+  },
+  NotEmpty: {
+    value: "notEmpty",
+    label: "Is not empty",
+  },
+  StartsWith: {
+    value: "string",
+    label: "Starts with",
+  },
+  Like: {
+    value: "fuzzy",
+    label: "Like",
+  },
+  MoreThan: {
+    value: "rangeLow",
+    label: "More than or equal to",
+  },
+  LessThan: {
+    value: "rangeHigh",
+    label: "Less than or equal to",
+  },
+  Contains: {
+    value: "contains",
+    label: "Contains",
+  },
+  NotContains: {
+    value: "notContains",
+    label: "Does not contain",
+  },
+  In: {
+    value: "oneOf",
+    label: "Is in",
+  },
+  ContainsAny: {
+    value: "containsAny",
+    label: "Has any",
+  },
+}
+
+export const SqlNumberTypeRangeMap = {
+  integer: {
+    max: 2147483647,
+    min: -2147483648,
+  },
+  int: {
+    max: 2147483647,
+    min: -2147483648,
+  },
+  smallint: {
+    max: 32767,
+    min: -32768,
+  },
+  mediumint: {
+    max: 8388607,
+    min: -8388608,
+  },
+}

package/src/filters.ts

@@ -0,0 +1,462 @@
+import { Datasource, FieldType, SortDirection, SortType } from "@budibase/types"
+import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
+import { deepGet } from "./helpers"
+
+const HBS_REGEX = /{{([^{].*?)}}/g
+
+/**
+ * Returns the valid operator options for a certain data type
+ * @param type the data type
+ */
+export const getValidOperatorsForType = (
+  type: FieldType,
+  field: string,
+  datasource: Datasource & { tableId: any } // TODO: is this table id ever populated?
+) => {
+  const Op = OperatorOptions
+  const stringOps = [
+    Op.Equals,
+    Op.NotEquals,
+    Op.StartsWith,
+    Op.Like,
+    Op.Empty,
+    Op.NotEmpty,
+    Op.In,
+  ]
+  const numOps = [
+    Op.Equals,
+    Op.NotEquals,
+    Op.MoreThan,
+    Op.LessThan,
+    Op.Empty,
+    Op.NotEmpty,
+    Op.In,
+  ]
+  let ops: {
+    value: string
+    label: string
+  }[] = []
+  if (type === "string") {
+    ops = stringOps
+  } else if (type === "number") {
+    ops = numOps
+  } else if (type === "options") {
+    ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In]
+  } else if (type === "array") {
+    ops = [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty, Op.ContainsAny]
+  } else if (type === "boolean") {
+    ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
+  } else if (type === "longform") {
+    ops = stringOps
+  } else if (type === "datetime") {
+    ops = numOps
+  } else if (type === "formula") {
+    ops = stringOps.concat([Op.MoreThan, Op.LessThan])
+  }
+
+  // Filter out "like" for internal tables
+  const externalTable = datasource?.tableId?.includes("datasource_plus")
+  if (datasource?.type === "table" && !externalTable) {
+    ops = ops.filter(x => x !== Op.Like)
+  }
+
+  // Only allow equal/not equal for _id in SQL tables
+  if (field === "_id" && externalTable) {
+    ops = [Op.Equals, Op.NotEquals, Op.In]
+  }
+
+  return ops
+}
+
+/**
+ * Operators which do not support empty strings as values
+ */
+export const NoEmptyFilterStrings = [
+  OperatorOptions.StartsWith.value,
+  OperatorOptions.Like.value,
+  OperatorOptions.Equals.value,
+  OperatorOptions.NotEquals.value,
+  OperatorOptions.Contains.value,
+  OperatorOptions.NotContains.value,
+] as (keyof QueryFields)[]
+
+/**
+ * Removes any fields that contain empty strings that would cause inconsistent
+ * behaviour with how backend tables are filtered (no value means no filter).
+ */
+const cleanupQuery = (query: Query) => {
+  if (!query) {
+    return query
+  }
+  for (let filterField of NoEmptyFilterStrings) {
+    if (!query[filterField]) {
+      continue
+    }
+
+    for (let [key, value] of Object.entries(query[filterField]!)) {
+      if (value == null || value === "") {
+        delete query[filterField]![key]
+      }
+    }
+  }
+  return query
+}
+
+/**
+ * Removes a numeric prefix on field names designed to give fields uniqueness
+ */
+const removeKeyNumbering = (key: string) => {
+  if (typeof key === "string" && key.match(/\d[0-9]*:/g) != null) {
+    const parts = key.split(":")
+    parts.shift()
+    return parts.join(":")
+  } else {
+    return key
+  }
+}
+
+type Filter = {
+  operator: keyof Query
+  field: string
+  type: any
+  value: any
+  externalType: keyof typeof SqlNumberTypeRangeMap
+}
+
+type Query = QueryFields & QueryConfig
+type QueryFields = {
+  string?: {
+    [key: string]: string
+  }
+  fuzzy?: {
+    [key: string]: string
+  }
+  range?: {
+    [key: string]: {
+      high: number | string
+      low: number | string
+    }
+  }
+  equal?: {
+    [key: string]: any
+  }
+  notEqual?: {
+    [key: string]: any
+  }
+  empty?: {
+    [key: string]: any
+  }
+  notEmpty?: {
+    [key: string]: any
+  }
+  oneOf?: {
+    [key: string]: any[]
+  }
+  contains?: {
+    [key: string]: any[]
+  }
+  notContains?: {
+    [key: string]: any[]
+  }
+  containsAny?: {
+    [key: string]: any[]
+  }
+}
+
+type QueryConfig = {
+  allOr?: boolean
+}
+
+type QueryFieldsType = keyof QueryFields
+
+/**
+ * Builds a lucene JSON query from the filter structure generated in the builder
+ * @param filter the builder filter structure
+ */
+export const buildLuceneQuery = (filter: Filter[]) => {
+  let query: Query = {
+    string: {},
+    fuzzy: {},
+    range: {},
+    equal: {},
+    notEqual: {},
+    empty: {},
+    notEmpty: {},
+    contains: {},
+    notContains: {},
+    oneOf: {},
+    containsAny: {},
+  }
+  if (Array.isArray(filter)) {
+    filter.forEach(expression => {
+      let { operator, field, type, value, externalType } = expression
+      const isHbs =
+        typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
+      // Parse all values into correct types
+      if (operator === "allOr") {
+        query.allOr = true
+        return
+      }
+      if (
+        type === "datetime" &&
+        !isHbs &&
+        operator !== "empty" &&
+        operator !== "notEmpty"
+      ) {
+        // Ensure date value is a valid date and parse into correct format
+        if (!value) {
+          return
+        }
+        try {
+          value = new Date(value).toISOString()
+        } catch (error) {
+          return
+        }
+      }
+      if (type === "number" && typeof value === "string") {
+        if (operator === "oneOf") {
+          value = value.split(",").map(item => parseFloat(item))
+        } else if (!isHbs) {
+          value = parseFloat(value)
+        }
+      }
+      if (type === "boolean") {
+        value = `${value}`?.toLowerCase() === "true"
+      }
+      if (
+        ["contains", "notContains", "containsAny"].includes(operator) &&
+        type === "array" &&
+        typeof value === "string"
+      ) {
+        value = value.split(",")
+      }
+      if (operator.startsWith("range") && query.range) {
+        const minint =
+          SqlNumberTypeRangeMap[externalType]?.min || Number.MIN_SAFE_INTEGER
+        const maxint =
+          SqlNumberTypeRangeMap[externalType]?.max || Number.MAX_SAFE_INTEGER
+        if (!query.range[field]) {
+          query.range[field] = {
+            low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
+            high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
+          }
+        }
+        if ((operator as any) === "rangeLow" && value != null && value !== "") {
+          query.range[field].low = value
+        } else if (
+          (operator as any) === "rangeHigh" &&
+          value != null &&
+          value !== ""
+        ) {
+          query.range[field].high = value
+        }
+      } else if (query[operator]) {
+        if (type === "boolean") {
+          // Transform boolean filters to cope with null.
+          // "equals false" needs to be "not equals true"
+          // "not equals false" needs to be "equals true"
+          if (operator === "equal" && value === false) {
+            query.notEqual = query.notEqual || {}
+            query.notEqual[field] = true
+          } else if (operator === "notEqual" && value === false) {
+            query.equal = query.equal || {}
+            query.equal[field] = true
+          } else {
+            query[operator] = query[operator] || {}
+            query[operator]![field] = value
+          }
+        } else {
+          query[operator] = query[operator] || {}
+          query[operator]![field] = value
+        }
+      }
+    })
+  }
+  return query
+}
+
+/**
+ * Performs a client-side lucene search on an array of data
+ * @param docs the data
+ * @param query the JSON lucene query
+ */
+export const runLuceneQuery = (docs: any[], query?: Query) => {
+  if (!docs || !Array.isArray(docs)) {
+    return []
+  }
+  if (!query) {
+    return docs
+  }
+
+  // Make query consistent first
+  query = cleanupQuery(query)
+
+  // Iterates over a set of filters and evaluates a fail function against a doc
+  const match =
+    (
+      type: QueryFieldsType,
+      failFn: (docValue: any, testValue: any) => boolean
+    ) =>
+    (doc: any) => {
+      const filters = Object.entries(query![type] || {})
+      for (let i = 0; i < filters.length; i++) {
+        const [key, testValue] = filters[i]
+        const docValue = deepGet(doc, removeKeyNumbering(key))
+        if (failFn(docValue, testValue)) {
+          return false
+        }
+      }
+      return true
+    }
+
+  // Process a string match (fails if the value does not start with the string)
+  const stringMatch = match("string", (docValue: string, testValue: string) => {
+    return (
+      !docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase())
+    )
+  })
+
+  // Process a fuzzy match (treat the same as starts with when running locally)
+  const fuzzyMatch = match("fuzzy", (docValue: string, testValue: string) => {
+    return (
+      !docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase())
+    )
+  })
+
+  // Process a range match
+  const rangeMatch = match(
+    "range",
+    (
+      docValue: string | number | null,
+      testValue: { low: number; high: number }
+    ) => {
+      return (
+        docValue == null ||
+        docValue === "" ||
+        docValue < testValue.low ||
+        docValue > testValue.high
+      )
+    }
+  )
+
+  // Process an equal match (fails if the value is different)
+  const equalMatch = match(
+    "equal",
+    (docValue: any, testValue: string | null) => {
+      return testValue != null && testValue !== "" && docValue !== testValue
+    }
+  )
+
+  // Process a not-equal match (fails if the value is the same)
+  const notEqualMatch = match(
+    "notEqual",
+    (docValue: any, testValue: string | null) => {
+      return testValue != null && testValue !== "" && docValue === testValue
+    }
+  )
+
+  // Process an empty match (fails if the value is not empty)
+  const emptyMatch = match("empty", (docValue: string | null) => {
+    return docValue != null && docValue !== ""
+  })
+
+  // Process a not-empty match (fails is the value is empty)
+  const notEmptyMatch = match("notEmpty", (docValue: string | null) => {
+    return docValue == null || docValue === ""
+  })
+
+  // Process an includes match (fails if the value is not included)
+  const oneOf = match("oneOf", (docValue: any, testValue: any) => {
+    if (typeof testValue === "string") {
+      testValue = testValue.split(",")
+      if (typeof docValue === "number") {
+        testValue = testValue.map((item: string) => parseFloat(item))
+      }
+    }
+    return !testValue?.includes(docValue)
+  })
+
+  const containsAny = match("containsAny", (docValue: any, testValue: any) => {
+    return !docValue?.includes(...testValue)
+  })
+
+  const contains = match(
+    "contains",
+    (docValue: string | any[], testValue: any[]) => {
+      return !testValue?.every((item: any) => docValue?.includes(item))
+    }
+  )
+
+  const notContains = match(
+    "notContains",
+    (docValue: string | any[], testValue: any[]) => {
+      return testValue?.every((item: any) => docValue?.includes(item))
+    }
+  )
+
+  // Match a document against all criteria
+  const docMatch = (doc: any) => {
+    return (
+      stringMatch(doc) &&
+      fuzzyMatch(doc) &&
+      rangeMatch(doc) &&
+      equalMatch(doc) &&
+      notEqualMatch(doc) &&
+      emptyMatch(doc) &&
+      notEmptyMatch(doc) &&
+      oneOf(doc) &&
+      contains(doc) &&
+      containsAny(doc) &&
+      notContains(doc)
+    )
+  }
+
+  // Process all docs
+  return docs.filter(docMatch)
+}
+
+/**
+ * Performs a client-side sort from the equivalent server-side lucene sort
+ * parameters.
+ * @param docs the data
+ * @param sort the sort column
+ * @param sortOrder the sort order ("ascending" or "descending")
+ * @param sortType the type of sort ("string" or "number")
+ */
+export const luceneSort = (
+  docs: any[],
+  sort: string,
+  sortOrder: SortDirection,
+  sortType = SortType.STRING
+) => {
+  if (!sort || !sortOrder || !sortType) {
+    return docs
+  }
+  const parse =
+    sortType === "string" ? (x: any) => `${x}` : (x: string) => parseFloat(x)
+  return docs
+    .slice()
+    .sort((a: { [x: string]: any }, b: { [x: string]: any }) => {
+      const colA = parse(a[sort])
+      const colB = parse(b[sort])
+      if (sortOrder.toLowerCase() === "descending") {
+        return colA > colB ? -1 : 1
+      } else {
+        return colA > colB ? 1 : -1
+      }
+    })
+}
+
+/**
+ * Limits the specified docs to the specified number of rows from the equivalent
+ * server-side lucene limit parameters.
+ * @param docs the data
+ * @param limit the number of docs to limit to
+ */
+export const luceneLimit = (docs: any[], limit: string) => {
+  const numLimit = parseFloat(limit)
+  if (isNaN(numLimit)) {
+    return docs
+  }
+  return docs.slice(0, numLimit)
+}

package/src/helpers/helpers.ts

@@ -0,0 +1,23 @@
+/**
+ * Gets a key within an object. The key supports dot syntax for retrieving deep
+ * fields - e.g. "a.b.c".
+ * Exact matches of keys with dots in them take precedence over nested keys of
+ * the same path - e.g. getting "a.b" from { "a.b": "foo", a: { b: "bar" } }
+ * will return "foo" over "bar".
+ * @param obj the object
+ * @param key the key
+ * @return {*|null} the value or null if a value was not found for this key
+ */
+export const deepGet = (obj: { [x: string]: any }, key: string) => {
+  if (!obj || !key) {
+    return null
+  }
+  if (Object.prototype.hasOwnProperty.call(obj, key)) {
+    return obj[key]
+  }
+  const split = key.split(".")
+  for (let i = 0; i < split.length; i++) {
+    obj = obj?.[split[i]]
+  }
+  return obj
+}

package/src/helpers/index.ts

@@ -0,0 +1,2 @@
+export * from "./helpers"
+export * from "./integrations"

package/src/helpers/integrations.ts

@@ -0,0 +1,5 @@
+import { SourceName } from "@budibase/types"
+
+export function isGoogleSheets(type: SourceName) {
+  return type === SourceName.GOOGLE_SHEETS
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+export * from "./constants"
+export * as dataFilters from "./filters"
+export * as helpers from "./helpers"
+export * as utils from "./utils"

package/src/utils.ts

@@ -0,0 +1,6 @@
+export function unreachable(
+  value: never,
+  message = `No such case in exhaustive switch: ${value}`
+) {
+  throw new Error(message)
+}

package/tsconfig-base.build.json

@@ -0,0 +1,25 @@
+{
+  "compilerOptions": {
+    "target": "es6",
+    "moduleResolution": "node",
+    "lib": ["es2020"],
+    "strict": true,
+    "noImplicitAny": true,
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "incremental": true,
+    "sourceMap": true,
+    "declaration": true,
+    "types": ["node"],
+    "outDir": "dist",
+    "skipLibCheck": true
+  },
+  "include": ["**/*.js", "**/*.ts", "package.json"],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "**/*.spec.ts",
+    "**/*.spec.js",
+    "__mocks__"
+  ]
+}

package/tsconfig-cjs.build.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig-base.build.json",
+  "compilerOptions": {
+    "module": "commonjs",
+    "outDir": "dist/cjs",
+    "target": "es2015"
+  }
+}

package/tsconfig.build.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig-base.build.json",
+  "compilerOptions": {
+    "module": "esnext",
+    "outDir": "dist/mjs",
+    "target": "esnext"
+  }
+}

package/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "./tsconfig.build.json",
+  "compilerOptions": {
+    "composite": true,
+    "baseUrl": ".",
+    "paths": {
+      "@budibase/types": ["../types/src"]
+    }
+  }
+}