@budibase/backend-core 2.5.8 → 2.5.10-alpha.0

package/src/db/lucene.ts

@@ -1,12 +1,14 @@
 import fetch from "node-fetch"
 import { getCouchInfo } from "./couch"
 import { SearchFilters, Row } from "@budibase/types"
+import { createUserIndex } from "./searchIndexes/searchIndexes"
 
 const QUERY_START_REGEX = /\d[0-9]*:/g
 
 interface SearchResponse<T> {
   rows: T[] | any[]
-  bookmark: string
+  bookmark?: string
+  totalRows: number
 }
 
 interface PaginatedSearchResponse<T> extends SearchResponse<T> {
@@ -42,23 +44,26 @@ export function removeKeyNumbering(key: any): string {
  * Optionally takes a base lucene query object.
  */
 export class QueryBuilder<T> {
-  dbName: string
-  index: string
-  query: SearchFilters
-  limit: number
-  sort?: string
-  bookmark?: string
-  sortOrder: string
-  sortType: string
-  includeDocs: boolean
-  version?: string
-  indexBuilder?: () => Promise<any>
-  noEscaping = false
+  #dbName: string
+  #index: string
+  #query: SearchFilters
+  #limit: number
+  #sort?: string
+  #bookmark?: string
+  #sortOrder: string
+  #sortType: string
+  #includeDocs: boolean
+  #version?: string
+  #indexBuilder?: () => Promise<any>
+  #noEscaping = false
+  #skip?: number
+
+  static readonly maxLimit = 200
 
   constructor(dbName: string, index: string, base?: SearchFilters) {
-    this.dbName = dbName
-    this.index = index
-    this.query = {
+    this.#dbName = dbName
+    this.#index = index
+    this.#query = {
       allOr: false,
       string: {},
       fuzzy: {},
@@ -73,86 +78,96 @@ export class QueryBuilder<T> {
       containsAny: {},
       ...base,
     }
-    this.limit = 50
-    this.sortOrder = "ascending"
-    this.sortType = "string"
-    this.includeDocs = true
+    this.#limit = 50
+    this.#sortOrder = "ascending"
+    this.#sortType = "string"
+    this.#includeDocs = true
   }
 
   disableEscaping() {
-    this.noEscaping = true
+    this.#noEscaping = true
     return this
   }
 
   setIndexBuilder(builderFn: () => Promise<any>) {
-    this.indexBuilder = builderFn
+    this.#indexBuilder = builderFn
     return this
   }
 
   setVersion(version?: string) {
     if (version != null) {
-      this.version = version
+      this.#version = version
     }
     return this
   }
 
   setTable(tableId: string) {
-    this.query.equal!.tableId = tableId
+    this.#query.equal!.tableId = tableId
     return this
   }
 
   setLimit(limit?: number) {
     if (limit != null) {
-      this.limit = limit
+      this.#limit = limit
     }
     return this
   }
 
   setSort(sort?: string) {
     if (sort != null) {
-      this.sort = sort
+      this.#sort = sort
     }
     return this
   }
 
   setSortOrder(sortOrder?: string) {
     if (sortOrder != null) {
-      this.sortOrder = sortOrder
+      this.#sortOrder = sortOrder
     }
     return this
   }
 
   setSortType(sortType?: string) {
     if (sortType != null) {
-      this.sortType = sortType
+      this.#sortType = sortType
     }
     return this
   }
 
   setBookmark(bookmark?: string) {
     if (bookmark != null) {
-      this.bookmark = bookmark
+      this.#bookmark = bookmark
     }
     return this
   }
 
+  setSkip(skip: number | undefined) {
+    this.#skip = skip
+    return this
+  }
+
   excludeDocs() {
-    this.includeDocs = false
+    this.#includeDocs = false
+    return this
+  }
+
+  includeDocs() {
+    this.#includeDocs = true
     return this
   }
 
   addString(key: string, partial: string) {
-    this.query.string![key] = partial
+    this.#query.string![key] = partial
     return this
   }
 
   addFuzzy(key: string, fuzzy: string) {
-    this.query.fuzzy![key] = fuzzy
+    this.#query.fuzzy![key] = fuzzy
     return this
   }
 
   addRange(key: string, low: string | number, high: string | number) {
-    this.query.range![key] = {
+    this.#query.range![key] = {
       low,
       high,
     }
@@ -160,51 +175,51 @@ export class QueryBuilder<T> {
   }
 
   addEqual(key: string, value: any) {
-    this.query.equal![key] = value
+    this.#query.equal![key] = value
     return this
   }
 
   addNotEqual(key: string, value: any) {
-    this.query.notEqual![key] = value
+    this.#query.notEqual![key] = value
     return this
   }
 
   addEmpty(key: string, value: any) {
-    this.query.empty![key] = value
+    this.#query.empty![key] = value
     return this
   }
 
   addNotEmpty(key: string, value: any) {
-    this.query.notEmpty![key] = value
+    this.#query.notEmpty![key] = value
     return this
   }
 
   addOneOf(key: string, value: any) {
-    this.query.oneOf![key] = value
+    this.#query.oneOf![key] = value
     return this
   }
 
   addContains(key: string, value: any) {
-    this.query.contains![key] = value
+    this.#query.contains![key] = value
     return this
   }
 
   addNotContains(key: string, value: any) {
-    this.query.notContains![key] = value
+    this.#query.notContains![key] = value
     return this
   }
 
   addContainsAny(key: string, value: any) {
-    this.query.containsAny![key] = value
+    this.#query.containsAny![key] = value
     return this
   }
 
   setAllOr() {
-    this.query.allOr = true
+    this.#query.allOr = true
   }
 
   handleSpaces(input: string) {
-    if (this.noEscaping) {
+    if (this.#noEscaping) {
       return input
     } else {
       return input.replace(/ /g, "_")
@@ -219,7 +234,7 @@ export class QueryBuilder<T> {
    * @returns {string|*}
    */
   preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) {
-    const hasVersion = !!this.version
+    const hasVersion = !!this.#version
     // Determine if type needs wrapped
     const originalType = typeof value
     // Convert to lowercase
@@ -227,8 +242,8 @@ export class QueryBuilder<T> {
       value = value.toLowerCase ? value.toLowerCase() : value
     }
     // Escape characters
-    if (!this.noEscaping && escape && originalType === "string") {
-      value = `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&")
+    if (!this.#noEscaping && escape && originalType === "string") {
+      value = `${value}`.replace(/[ \/#+\-&|!(){}\]^"~*?:\\]/g, "\\$&")
     }
 
     // Wrap in quotes
@@ -242,7 +257,7 @@ export class QueryBuilder<T> {
 
   isMultiCondition() {
     let count = 0
-    for (let filters of Object.values(this.query)) {
+    for (let filters of Object.values(this.#query)) {
       // not contains is one massive filter in allOr mode
       if (typeof filters === "object") {
         count += Object.keys(filters).length
@@ -272,13 +287,13 @@ export class QueryBuilder<T> {
 
   buildSearchQuery() {
     const builder = this
-    let allOr = this.query && this.query.allOr
+    let allOr = this.#query && this.#query.allOr
     let query = allOr ? "" : "*:*"
     const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true }
     let tableId
-    if (this.query.equal!.tableId) {
-      tableId = this.query.equal!.tableId
-      delete this.query.equal!.tableId
+    if (this.#query.equal!.tableId) {
+      tableId = this.#query.equal!.tableId
+      delete this.#query.equal!.tableId
     }
 
     const equal = (key: string, value: any) => {
@@ -305,6 +320,18 @@ export class QueryBuilder<T> {
       return `${key}:(${statement})`
     }
 
+    const fuzzy = (key: string, value: any) => {
+      if (!value) {
+        return null
+      }
+      value = builder.preprocess(value, {
+        escape: true,
+        lowercase: true,
+        type: "fuzzy",
+      })
+      return `${key}:/.*${value}.*/`
+    }
+
     const notContains = (key: string, value: any) => {
       const allPrefix = allOr ? "*:* AND " : ""
       const mode = allOr ? "AND" : undefined
@@ -363,8 +390,8 @@ export class QueryBuilder<T> {
     }
 
     // Construct the actual lucene search query string from JSON structure
-    if (this.query.string) {
-      build(this.query.string, (key: string, value: any) => {
+    if (this.#query.string) {
+      build(this.#query.string, (key: string, value: any) => {
         if (!value) {
           return null
         }
@@ -376,8 +403,8 @@ export class QueryBuilder<T> {
         return `${key}:${value}*`
       })
     }
-    if (this.query.range) {
-      build(this.query.range, (key: string, value: any) => {
+    if (this.#query.range) {
+      build(this.#query.range, (key: string, value: any) => {
         if (!value) {
           return null
         }
@@ -392,47 +419,37 @@ export class QueryBuilder<T> {
         return `${key}:[${low} TO ${high}]`
       })
     }
-    if (this.query.fuzzy) {
-      build(this.query.fuzzy, (key: string, value: any) => {
-        if (!value) {
-          return null
-        }
-        value = builder.preprocess(value, {
-          escape: true,
-          lowercase: true,
-          type: "fuzzy",
-        })
-        return `${key}:${value}~`
-      })
+    if (this.#query.fuzzy) {
+      build(this.#query.fuzzy, fuzzy)
     }
-    if (this.query.equal) {
-      build(this.query.equal, equal)
+    if (this.#query.equal) {
+      build(this.#query.equal, equal)
     }
-    if (this.query.notEqual) {
-      build(this.query.notEqual, (key: string, value: any) => {
+    if (this.#query.notEqual) {
+      build(this.#query.notEqual, (key: string, value: any) => {
         if (!value) {
           return null
         }
         return `!${key}:${builder.preprocess(value, allPreProcessingOpts)}`
       })
     }
-    if (this.query.empty) {
-      build(this.query.empty, (key: string) => `!${key}:["" TO *]`)
+    if (this.#query.empty) {
+      build(this.#query.empty, (key: string) => `(*:* -${key}:["" TO *])`)
     }
-    if (this.query.notEmpty) {
-      build(this.query.notEmpty, (key: string) => `${key}:["" TO *]`)
+    if (this.#query.notEmpty) {
+      build(this.#query.notEmpty, (key: string) => `${key}:["" TO *]`)
     }
-    if (this.query.oneOf) {
-      build(this.query.oneOf, oneOf)
+    if (this.#query.oneOf) {
+      build(this.#query.oneOf, oneOf)
     }
-    if (this.query.contains) {
-      build(this.query.contains, contains)
+    if (this.#query.contains) {
+      build(this.#query.contains, contains)
     }
-    if (this.query.notContains) {
-      build(this.compressFilters(this.query.notContains), notContains)
+    if (this.#query.notContains) {
+      build(this.compressFilters(this.#query.notContains), notContains)
     }
-    if (this.query.containsAny) {
-      build(this.query.containsAny, containsAny)
+    if (this.#query.containsAny) {
+      build(this.#query.containsAny, containsAny)
     }
     // make sure table ID is always added as an AND
     if (tableId) {
@@ -446,29 +463,65 @@ export class QueryBuilder<T> {
   buildSearchBody() {
     let body: any = {
       q: this.buildSearchQuery(),
-      limit: Math.min(this.limit, 200),
-      include_docs: this.includeDocs,
+      limit: Math.min(this.#limit, QueryBuilder.maxLimit),
+      include_docs: this.#includeDocs,
     }
-    if (this.bookmark) {
-      body.bookmark = this.bookmark
+    if (this.#bookmark) {
+      body.bookmark = this.#bookmark
     }
-    if (this.sort) {
-      const order = this.sortOrder === "descending" ? "-" : ""
-      const type = `<${this.sortType}>`
-      body.sort = `${order}${this.handleSpaces(this.sort)}${type}`
+    if (this.#sort) {
+      const order = this.#sortOrder === "descending" ? "-" : ""
+      const type = `<${this.#sortType}>`
+      body.sort = `${order}${this.handleSpaces(this.#sort)}${type}`
     }
     return body
   }
 
   async run() {
+    if (this.#skip) {
+      await this.#skipItems(this.#skip)
+    }
+    return await this.#execute()
+  }
+
+  /**
+   * Lucene queries do not support pagination and use bookmarks instead.
+   * For the given builder, walk through pages using bookmarks until the desired
+   * page has been met.
+   */
+  async #skipItems(skip: number) {
+    // Lucene does not support pagination.
+    // Handle pagination by finding the right bookmark
+    const prevIncludeDocs = this.#includeDocs
+    const prevLimit = this.#limit
+
+    this.excludeDocs()
+    let skipRemaining = skip
+    let iterationFetched = 0
+    do {
+      const toSkip = Math.min(QueryBuilder.maxLimit, skipRemaining)
+      this.setLimit(toSkip)
+      const { bookmark, rows } = await this.#execute()
+      this.setBookmark(bookmark)
+      iterationFetched = rows.length
+      skipRemaining -= rows.length
+    } while (skipRemaining > 0 && iterationFetched > 0)
+
+    this.#includeDocs = prevIncludeDocs
+    this.#limit = prevLimit
+  }
+
+  async #execute() {
     const { url, cookie } = getCouchInfo()
-    const fullPath = `${url}/${this.dbName}/_design/database/_search/${this.index}`
+    const fullPath = `${url}/${this.#dbName}/_design/database/_search/${
+      this.#index
+    }`
     const body = this.buildSearchBody()
     try {
       return await runQuery<T>(fullPath, body, cookie)
     } catch (err: any) {
-      if (err.status === 404 && this.indexBuilder) {
-        await this.indexBuilder()
+      if (err.status === 404 && this.#indexBuilder) {
+        await this.#indexBuilder()
         return await runQuery<T>(fullPath, body, cookie)
       } else {
         throw err
@@ -502,8 +555,9 @@ async function runQuery<T>(
   }
   const json = await response.json()
 
-  let output: any = {
+  let output: SearchResponse<T> = {
     rows: [],
+    totalRows: 0,
   }
   if (json.rows != null && json.rows.length > 0) {
     output.rows = json.rows.map((row: any) => row.doc)
@@ -511,6 +565,9 @@ async function runQuery<T>(
   if (json.bookmark) {
     output.bookmark = json.bookmark
   }
+  if (json.total_rows) {
+    output.totalRows = json.total_rows
+  }
   return output
 }
 
@@ -543,8 +600,8 @@ async function recursiveSearch<T>(
   if (rows.length >= params.limit) {
     return rows
   }
-  let pageSize = 200
-  if (rows.length > params.limit - 200) {
+  let pageSize = QueryBuilder.maxLimit
+  if (rows.length > params.limit - QueryBuilder.maxLimit) {
     pageSize = params.limit - rows.length
   }
   const page = await new QueryBuilder<T>(dbName, index, query)
@@ -559,7 +616,7 @@ async function recursiveSearch<T>(
   if (!page.rows.length) {
     return rows
   }
-  if (page.rows.length < 200) {
+  if (page.rows.length < QueryBuilder.maxLimit) {
     return [...rows, ...page.rows]
   }
   const newParams = {
@@ -597,7 +654,7 @@ export async function paginatedSearch<T>(
   if (limit == null || isNaN(limit) || limit < 0) {
     limit = 50
   }
-  limit = Math.min(limit, 200)
+  limit = Math.min(limit, QueryBuilder.maxLimit)
   const search = new QueryBuilder<T>(dbName, index, query)
   if (params.version) {
     search.setVersion(params.version)

package/src/db/searchIndexes/index.ts

@@ -0,0 +1 @@
+export * from "./searchIndexes"

package/src/db/searchIndexes/searchIndexes.ts

@@ -0,0 +1,62 @@
+import { User, SearchIndex } from "@budibase/types"
+import { getGlobalDB } from "../../context"
+
+export async function createUserIndex() {
+  const db = getGlobalDB()
+  let designDoc
+  try {
+    designDoc = await db.get("_design/database")
+  } catch (err: any) {
+    if (err.status === 404) {
+      designDoc = { _id: "_design/database" }
+    }
+  }
+
+  const fn = function (user: User) {
+    if (user._id && !user._id.startsWith("us_")) {
+      return
+    }
+    const ignoredFields = [
+      "_id",
+      "_rev",
+      "password",
+      "account",
+      "license",
+      "budibaseAccess",
+      "accountPortalAccess",
+      "csrfToken",
+    ]
+
+    function idx(input: Record<string, any>, prev?: string) {
+      for (let key of Object.keys(input)) {
+        if (ignoredFields.includes(key)) {
+          continue
+        }
+        let idxKey = prev != null ? `${prev}.${key}` : key
+        if (typeof input[key] === "string") {
+          // eslint-disable-next-line no-undef
+          // @ts-ignore
+          index(idxKey, input[key].toLowerCase(), { facet: true })
+        } else if (typeof input[key] !== "object") {
+          // eslint-disable-next-line no-undef
+          // @ts-ignore
+          index(idxKey, input[key], { facet: true })
+        } else {
+          idx(input[key], idxKey)
+        }
+      }
+    }
+    idx(user)
+  }
+
+  designDoc.indexes = {
+    [SearchIndex.USER]: {
+      index: fn.toString(),
+      analyzer: {
+        default: "keyword",
+        name: "perfield",
+      },
+    },
+  }
+  await db.put(designDoc)
+}

package/src/db/tests/lucene.spec.ts

@@ -1,4 +1,4 @@
-import { newid } from "../../newid"
+import { newid } from "../../docIds/newid"
 import { getDB } from "../db"
 import { Database } from "@budibase/types"
 import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
@@ -136,6 +136,106 @@ describe("lucene", () => {
       const resp = await builder.run()
       expect(resp.rows.length).toBe(2)
     })
+
+    describe("skip", () => {
+      const skipDbName = `db-${newid()}`
+      let docs: {
+        _id: string
+        property: string
+        array: string[]
+      }[]
+
+      beforeAll(async () => {
+        const db = getDB(skipDbName)
+
+        docs = Array(QueryBuilder.maxLimit * 2.5)
+          .fill(0)
+          .map((_, i) => ({
+            _id: i.toString().padStart(3, "0"),
+            property: `value_${i.toString().padStart(3, "0")}`,
+            array: [],
+          }))
+        await db.bulkDocs(docs)
+
+        await db.put({
+          _id: "_design/database",
+          indexes: {
+            [INDEX_NAME]: {
+              index: index,
+              analyzer: "standard",
+            },
+          },
+        })
+      })
+
+      it("should be able to apply skip", async () => {
+        const builder = new QueryBuilder(skipDbName, INDEX_NAME)
+        const firstResponse = await builder.run()
+        builder.setSkip(40)
+        const secondResponse = await builder.run()
+
+        // Return the default limit
+        expect(firstResponse.rows.length).toBe(50)
+        expect(secondResponse.rows.length).toBe(50)
+
+        // Should have the expected overlap
+        expect(firstResponse.rows.slice(40)).toEqual(
+          secondResponse.rows.slice(0, 10)
+        )
+      })
+
+      it("should handle limits", async () => {
+        const builder = new QueryBuilder(skipDbName, INDEX_NAME)
+        builder.setLimit(10)
+        builder.setSkip(50)
+        builder.setSort("_id")
+
+        const resp = await builder.run()
+        expect(resp.rows.length).toBe(10)
+        expect(resp.rows).toEqual(
+          docs.slice(50, 60).map(expect.objectContaining)
+        )
+      })
+
+      it("should be able to skip searching through multiple responses", async () => {
+        const builder = new QueryBuilder(skipDbName, INDEX_NAME)
+        // Skipping 2 max limits plus a little bit more
+        const skip = QueryBuilder.maxLimit * 2 + 37
+        builder.setSkip(skip)
+        builder.setSort("_id")
+        const resp = await builder.run()
+
+        expect(resp.rows.length).toBe(50)
+        expect(resp.rows).toEqual(
+          docs.slice(skip, skip + resp.rows.length).map(expect.objectContaining)
+        )
+      })
+
+      it("should not return results if skipping all docs", async () => {
+        const builder = new QueryBuilder(skipDbName, INDEX_NAME)
+        // Skipping 2 max limits plus a little bit more
+        const skip = docs.length + 1
+        builder.setSkip(skip)
+
+        const resp = await builder.run()
+
+        expect(resp.rows.length).toBe(0)
+      })
+
+      it("skip should respect with filters", async () => {
+        const builder = new QueryBuilder(skipDbName, INDEX_NAME)
+        builder.setLimit(10)
+        builder.setSkip(50)
+        builder.addString("property", "value_1")
+        builder.setSort("property")
+
+        const resp = await builder.run()
+        expect(resp.rows.length).toBe(10)
+        expect(resp.rows).toEqual(
+          docs.slice(150, 160).map(expect.objectContaining)
+        )
+      })
+    })
   })
 
   describe("paginated search", () => {

package/src/db/tests/utils.spec.ts

@@ -3,7 +3,7 @@ import {
   getProdAppID,
   isDevAppID,
   isProdAppID,
-} from "../conversions"
+} from "../../docIds/conversions"
 import { generateAppID } from "../utils"
 
 describe("utils", () => {