@budibase/shared-core 2.3.21-alpha.1

package/dist/src/index.js

@@ -0,0 +1,32 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.dataFilters = void 0;
+__exportStar(require("./constants"), exports);
+exports.dataFilters = __importStar(require("./filters"));
+//# sourceMappingURL=index.js.map

package/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "@budibase/shared-core",
+  "version": "2.3.21-alpha.1",
+  "description": "Shared data utils",
+  "main": "dist/src/index.js",
+  "types": "dist/src/index.d.ts",
+  "author": "Budibase",
+  "license": "GPL-3.0",
+  "scripts": {
+    "build": "tsc",
+    "dev:builder": "tsc --watch"
+  },
+  "devDependencies": {
+    "typescript": "4.7.3"
+  }
+}

package/src/constants.ts

@@ -0,0 +1,69 @@
+export const OperatorOptions = {
+  Equals: {
+    value: "equal",
+    label: "Equals",
+  },
+  NotEquals: {
+    value: "notEqual",
+    label: "Not equals",
+  },
+  Empty: {
+    value: "empty",
+    label: "Is empty",
+  },
+  NotEmpty: {
+    value: "notEmpty",
+    label: "Is not empty",
+  },
+  StartsWith: {
+    value: "string",
+    label: "Starts with",
+  },
+  Like: {
+    value: "fuzzy",
+    label: "Like",
+  },
+  MoreThan: {
+    value: "rangeLow",
+    label: "More than or equal to",
+  },
+  LessThan: {
+    value: "rangeHigh",
+    label: "Less than or equal to",
+  },
+  Contains: {
+    value: "contains",
+    label: "Contains",
+  },
+  NotContains: {
+    value: "notContains",
+    label: "Does not contain",
+  },
+  In: {
+    value: "oneOf",
+    label: "Is in",
+  },
+  ContainsAny: {
+    value: "containsAny",
+    label: "Has any",
+  },
+}
+
+export const SqlNumberTypeRangeMap = {
+  integer: {
+    max: 2147483647,
+    min: -2147483648,
+  },
+  int: {
+    max: 2147483647,
+    min: -2147483648,
+  },
+  smallint: {
+    max: 32767,
+    min: -32768,
+  },
+  mediumint: {
+    max: 8388607,
+    min: -8388608,
+  },
+}

package/src/filters.ts

@@ -0,0 +1,435 @@
+import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
+
+const HBS_REGEX = /{{([^{].*?)}}/g
+
+/**
+ * Returns the valid operator options for a certain data type
+ * @param type the data type
+ */
+export const getValidOperatorsForType = (
+  type: string,
+  field: string,
+  datasource: { tableId: string | string[]; type: string }
+) => {
+  const Op = OperatorOptions
+  const stringOps = [
+    Op.Equals,
+    Op.NotEquals,
+    Op.StartsWith,
+    Op.Like,
+    Op.Empty,
+    Op.NotEmpty,
+    Op.In,
+  ]
+  const numOps = [
+    Op.Equals,
+    Op.NotEquals,
+    Op.MoreThan,
+    Op.LessThan,
+    Op.Empty,
+    Op.NotEmpty,
+    Op.In,
+  ]
+  let ops: any[] = []
+  if (type === "string") {
+    ops = stringOps
+  } else if (type === "number") {
+    ops = numOps
+  } else if (type === "options") {
+    ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In]
+  } else if (type === "array") {
+    ops = [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty, Op.ContainsAny]
+  } else if (type === "boolean") {
+    ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
+  } else if (type === "longform") {
+    ops = stringOps
+  } else if (type === "datetime") {
+    ops = numOps
+  } else if (type === "formula") {
+    ops = stringOps.concat([Op.MoreThan, Op.LessThan])
+  }
+
+  // Filter out "like" for internal tables
+  const externalTable = datasource?.tableId?.includes("datasource_plus")
+  if (datasource?.type === "table" && !externalTable) {
+    ops = ops.filter(x => x !== Op.Like)
+  }
+
+  // Only allow equal/not equal for _id in SQL tables
+  if (field === "_id" && externalTable) {
+    ops = [Op.Equals, Op.NotEquals]
+  }
+
+  return ops
+}
+
+/**
+ * Operators which do not support empty strings as values
+ */
+export const NoEmptyFilterStrings = [
+  OperatorOptions.StartsWith.value,
+  OperatorOptions.Like.value,
+  OperatorOptions.Equals.value,
+  OperatorOptions.NotEquals.value,
+  OperatorOptions.Contains.value,
+  OperatorOptions.NotContains.value,
+]
+
+/**
+ * Removes any fields that contain empty strings that would cause inconsistent
+ * behaviour with how backend tables are filtered (no value means no filter).
+ */
+const cleanupQuery = (query: { [x: string]: { [x: string]: any } }) => {
+  if (!query) {
+    return query
+  }
+  for (let filterField of NoEmptyFilterStrings) {
+    if (!query[filterField]) {
+      continue
+    }
+    for (let [key, value] of Object.entries(query[filterField])) {
+      if (value == null || value === "") {
+        delete query[filterField][key]
+      }
+    }
+  }
+  return query
+}
+
+/**
+ * Removes a numeric prefix on field names designed to give fields uniqueness
+ */
+const removeKeyNumbering = (key: string) => {
+  if (typeof key === "string" && key.match(/\d[0-9]*:/g) != null) {
+    const parts = key.split(":")
+    parts.shift()
+    return parts.join(":")
+  } else {
+    return key
+  }
+}
+
+type Filter = {
+  operator: keyof Query
+  field: string
+  type: any
+  value: any
+  externalType: keyof typeof SqlNumberTypeRangeMap
+}
+
+type Query = {
+  string: Record<string, any>
+  fuzzy: Record<string, any>
+  range: Record<string, { low: string | number; high: string | number }>
+  equal: Record<string, true>
+  notEqual: Record<string, true>
+  empty: Record<string, any>
+  notEmpty: Record<string, any>
+  contains: Record<string, any>
+  notContains: Record<string, any>
+  oneOf: Record<string, any>
+  containsAny: Record<string, any>
+  allOr?: boolean
+}
+
+/**
+ * Builds a lucene JSON query from the filter structure generated in the builder
+ * @param filter the builder filter structure
+ */
+export const buildLuceneQuery = (filter: Filter[]) => {
+  let query: Query = {
+    string: {},
+    fuzzy: {},
+    range: {},
+    equal: {},
+    notEqual: {},
+    empty: {},
+    notEmpty: {},
+    contains: {},
+    notContains: {},
+    oneOf: {},
+    containsAny: {},
+  }
+  if (Array.isArray(filter)) {
+    filter.forEach(expression => {
+      let { operator, field, type, value, externalType } = expression
+      const isHbs =
+        typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
+      // Parse all values into correct types
+      if (operator === "allOr") {
+        query.allOr = true
+        return
+      }
+      if (
+        type === "datetime" &&
+        !isHbs &&
+        operator !== "empty" &&
+        operator !== "notEmpty"
+      ) {
+        // Ensure date value is a valid date and parse into correct format
+        if (!value) {
+          return
+        }
+        try {
+          value = new Date(value).toISOString()
+        } catch (error) {
+          return
+        }
+      }
+      if (type === "number" && typeof value === "string") {
+        if (operator === "oneOf") {
+          value = value.split(",").map(item => parseFloat(item))
+        } else if (!isHbs) {
+          value = parseFloat(value)
+        }
+      }
+      if (type === "boolean") {
+        value = `${value}`?.toLowerCase() === "true"
+      }
+      if (
+        ["contains", "notContains", "containsAny"].includes(operator) &&
+        type === "array" &&
+        typeof value === "string"
+      ) {
+        value = value.split(",")
+      }
+      if (operator.startsWith("range")) {
+        const minint =
+          SqlNumberTypeRangeMap[externalType]?.min || Number.MIN_SAFE_INTEGER
+        const maxint =
+          SqlNumberTypeRangeMap[externalType]?.max || Number.MAX_SAFE_INTEGER
+        if (!query.range[field]) {
+          query.range[field] = {
+            low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
+            high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
+          }
+        }
+        if ((operator as any) === "rangeLow" && value != null && value !== "") {
+          query.range[field].low = value
+        } else if (
+          (operator as any) === "rangeHigh" &&
+          value != null &&
+          value !== ""
+        ) {
+          query.range[field].high = value
+        }
+      } else if (query[operator]) {
+        if (type === "boolean") {
+          // Transform boolean filters to cope with null.
+          // "equals false" needs to be "not equals true"
+          // "not equals false" needs to be "equals true"
+          if (operator === "equal" && value === false) {
+            query.notEqual[field] = true
+          } else if (operator === "notEqual" && value === false) {
+            query.equal[field] = true
+          } else {
+            query[operator][field] = value
+          }
+        } else {
+          query[operator][field] = value
+        }
+      }
+    })
+  }
+  return query
+}
+
+const deepGet = (obj: { [x: string]: any }, key: string) => {
+  if (!obj || !key) {
+    return null
+  }
+  if (Object.prototype.hasOwnProperty.call(obj, key)) {
+    return obj[key]
+  }
+  const split = key.split(".")
+  for (let i = 0; i < split.length; i++) {
+    obj = obj?.[split[i]]
+  }
+  return obj
+}
+
+/**
+ * Performs a client-side lucene search on an array of data
+ * @param docs the data
+ * @param query the JSON lucene query
+ */
+export const runLuceneQuery = (
+  docs: any[],
+  query?: { [x: string]: any; sheet?: string }
+) => {
+  if (!docs || !Array.isArray(docs)) {
+    return []
+  }
+  if (!query) {
+    return docs
+  }
+
+  // Make query consistent first
+  query = cleanupQuery(query)
+
+  // Iterates over a set of filters and evaluates a fail function against a doc
+  const match =
+    (type: string, failFn: (docValue: any, testValue: any) => boolean) =>
+    (doc: any) => {
+      const filters = Object.entries(query![type] || {})
+      for (let i = 0; i < filters.length; i++) {
+        const [key, testValue] = filters[i]
+        const docValue = deepGet(doc, removeKeyNumbering(key))
+        if (failFn(docValue, testValue)) {
+          return false
+        }
+      }
+      return true
+    }
+
+  // Process a string match (fails if the value does not start with the string)
+  const stringMatch = match("string", (docValue: string, testValue: string) => {
+    return (
+      !docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase())
+    )
+  })
+
+  // Process a fuzzy match (treat the same as starts with when running locally)
+  const fuzzyMatch = match("fuzzy", (docValue: string, testValue: string) => {
+    return (
+      !docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase())
+    )
+  })
+
+  // Process a range match
+  const rangeMatch = match(
+    "range",
+    (
+      docValue: string | number | null,
+      testValue: { low: number; high: number }
+    ) => {
+      return (
+        docValue == null ||
+        docValue === "" ||
+        docValue < testValue.low ||
+        docValue > testValue.high
+      )
+    }
+  )
+
+  // Process an equal match (fails if the value is different)
+  const equalMatch = match(
+    "equal",
+    (docValue: any, testValue: string | null) => {
+      return testValue != null && testValue !== "" && docValue !== testValue
+    }
+  )
+
+  // Process a not-equal match (fails if the value is the same)
+  const notEqualMatch = match(
+    "notEqual",
+    (docValue: any, testValue: string | null) => {
+      return testValue != null && testValue !== "" && docValue === testValue
+    }
+  )
+
+  // Process an empty match (fails if the value is not empty)
+  const emptyMatch = match("empty", (docValue: string | null) => {
+    return docValue != null && docValue !== ""
+  })
+
+  // Process a not-empty match (fails is the value is empty)
+  const notEmptyMatch = match("notEmpty", (docValue: string | null) => {
+    return docValue == null || docValue === ""
+  })
+
+  // Process an includes match (fails if the value is not included)
+  const oneOf = match("oneOf", (docValue: any, testValue: any) => {
+    if (typeof testValue === "string") {
+      testValue = testValue.split(",")
+      if (typeof docValue === "number") {
+        testValue = testValue.map((item: string) => parseFloat(item))
+      }
+    }
+    return !testValue?.includes(docValue)
+  })
+
+  const containsAny = match("containsAny", (docValue: any, testValue: any) => {
+    return !docValue?.includes(...testValue)
+  })
+
+  const contains = match(
+    "contains",
+    (docValue: string | any[], testValue: any[]) => {
+      return !testValue?.every((item: any) => docValue?.includes(item))
+    }
+  )
+
+  const notContains = match(
+    "notContains",
+    (docValue: string | any[], testValue: any[]) => {
+      return testValue?.every((item: any) => docValue?.includes(item))
+    }
+  )
+
+  // Match a document against all criteria
+  const docMatch = (doc: any) => {
+    return (
+      stringMatch(doc) &&
+      fuzzyMatch(doc) &&
+      rangeMatch(doc) &&
+      equalMatch(doc) &&
+      notEqualMatch(doc) &&
+      emptyMatch(doc) &&
+      notEmptyMatch(doc) &&
+      oneOf(doc) &&
+      contains(doc) &&
+      containsAny(doc) &&
+      notContains(doc)
+    )
+  }
+
+  // Process all docs
+  return docs.filter(docMatch)
+}
+
+/**
+ * Performs a client-side sort from the equivalent server-side lucene sort
+ * parameters.
+ * @param docs the data
+ * @param sort the sort column
+ * @param sortOrder the sort order ("ascending" or "descending")
+ * @param sortType the type of sort ("string" or "number")
+ */
+export const luceneSort = (
+  docs: any[],
+  sort: string | number,
+  sortOrder: string,
+  sortType = "string"
+) => {
+  if (!sort || !sortOrder || !sortType) {
+    return docs
+  }
+  const parse =
+    sortType === "string" ? (x: any) => `${x}` : (x: string) => parseFloat(x)
+  return docs
+    .slice()
+    .sort((a: { [x: string]: any }, b: { [x: string]: any }) => {
+      const colA = parse(a[sort])
+      const colB = parse(b[sort])
+      if (sortOrder === "Descending") {
+        return colA > colB ? -1 : 1
+      } else {
+        return colA > colB ? 1 : -1
+      }
+    })
+}
+
+/**
+ * Limits the specified docs to the specified number of rows from the equivalent
+ * server-side lucene limit parameters.
+ * @param docs the data
+ * @param limit the number of docs to limit to
+ */
+export const luceneLimit = (docs: string | any[], limit: string) => {
+  const numLimit = parseFloat(limit)
+  if (isNaN(numLimit)) {
+    return docs
+  }
+  return docs.slice(0, numLimit)
+}

package/src/index.ts

@@ -0,0 +1,2 @@
+export * from "./constants"
+export * as dataFilters from "./filters"

package/tsconfig.build.json

@@ -0,0 +1,25 @@
+{
+  "compilerOptions": {
+    "target": "es6",
+    "module": "commonjs",
+    "lib": ["es2020"],
+    "strict": true,
+    "noImplicitAny": true,
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "incremental": true,
+    "sourceMap": true,
+    "declaration": true,
+    "types": ["node"],
+    "outDir": "dist",
+    "skipLibCheck": true
+  },
+  "include": ["**/*.js", "**/*.ts", "package.json"],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "**/*.spec.ts",
+    "**/*.spec.js",
+    "__mocks__"
+  ]
+}

package/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "./tsconfig.build.json",
+  "compilerOptions": {
+    "composite": true
+  }
+}