@budibase/backend-core 2.22.17 → 2.23.0

package/src/db/lucene.ts

@@ -1,28 +1,16 @@
 import fetch from "node-fetch"
 import { getCouchInfo } from "./couch"
-import { SearchFilters, Row, EmptyFilterOption } from "@budibase/types"
+import {
+  SearchFilters,
+  Row,
+  EmptyFilterOption,
+  SearchResponse,
+  SearchParams,
+  WithRequired,
+} from "@budibase/types"
 
 const QUERY_START_REGEX = /\d[0-9]*:/g
 
-interface SearchResponse<T> {
-  rows: T[] | any[]
-  bookmark?: string
-  totalRows: number
-}
-
-export type SearchParams<T> = {
-  tableId?: string
-  sort?: string
-  sortOrder?: string
-  sortType?: string
-  limit?: number
-  bookmark?: string
-  version?: string
-  indexer?: () => Promise<any>
-  disableEscaping?: boolean
-  rows?: T | Row[]
-}
-
 export function removeKeyNumbering(key: any): string {
   if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) {
     const parts = key.split(":")
@@ -44,7 +32,7 @@ export class QueryBuilder<T> {
   #query: SearchFilters
   #limit: number
   #sort?: string
-  #bookmark?: string
+  #bookmark?: string | number
   #sortOrder: string
   #sortType: string
   #includeDocs: boolean
@@ -130,7 +118,7 @@ export class QueryBuilder<T> {
     return this
   }
 
-  setBookmark(bookmark?: string) {
+  setBookmark(bookmark?: string | number) {
     if (bookmark != null) {
       this.#bookmark = bookmark
     }
@@ -226,14 +214,20 @@ export class QueryBuilder<T> {
     }
   }
 
-  /**
-   * Preprocesses a value before going into a lucene search.
-   * Transforms strings to lowercase and wraps strings and bools in quotes.
-   * @param value The value to process
-   * @param options The preprocess options
-   * @returns {string|*}
-   */
-  preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) {
+  preprocess(
+    value: any,
+    {
+      escape,
+      lowercase,
+      wrap,
+      type,
+    }: {
+      escape?: boolean
+      lowercase?: boolean
+      wrap?: boolean
+      type?: string
+    } = {}
+  ): string | any {
     const hasVersion = !!this.#version
     // Determine if type needs wrapped
     const originalType = typeof value
@@ -561,7 +555,7 @@ async function runQuery<T>(
   url: string,
   body: any,
   cookie: string
-): Promise<SearchResponse<T>> {
+): Promise<WithRequired<SearchResponse<T>, "totalRows">> {
   const response = await fetch(url, {
     body: JSON.stringify(body),
     method: "POST",
@@ -575,7 +569,7 @@ async function runQuery<T>(
   }
   const json = await response.json()
 
-  let output: SearchResponse<T> = {
+  let output: WithRequired<SearchResponse<T>, "totalRows"> = {
     rows: [],
     totalRows: 0,
   }
@@ -613,63 +607,51 @@ async function recursiveSearch<T>(
   dbName: string,
   index: string,
   query: any,
-  params: any
+  params: SearchParams
 ): Promise<any> {
   const bookmark = params.bookmark
   const rows = params.rows || []
-  if (rows.length >= params.limit) {
+  if (params.limit && rows.length >= params.limit) {
     return rows
   }
   let pageSize = QueryBuilder.maxLimit
-  if (rows.length > params.limit - QueryBuilder.maxLimit) {
+  if (params.limit && rows.length > params.limit - QueryBuilder.maxLimit) {
     pageSize = params.limit - rows.length
   }
-  const page = await new QueryBuilder<T>(dbName, index, query)
+  const queryBuilder = new QueryBuilder<T>(dbName, index, query)
+  queryBuilder
     .setVersion(params.version)
-    .setTable(params.tableId)
     .setBookmark(bookmark)
     .setLimit(pageSize)
     .setSort(params.sort)
     .setSortOrder(params.sortOrder)
     .setSortType(params.sortType)
-    .run()
+
+  if (params.tableId) {
+    queryBuilder.setTable(params.tableId)
+  }
+
+  const page = await queryBuilder.run()
   if (!page.rows.length) {
     return rows
   }
   if (page.rows.length < QueryBuilder.maxLimit) {
     return [...rows, ...page.rows]
   }
-  const newParams = {
+  const newParams: SearchParams = {
     ...params,
     bookmark: page.bookmark,
-    rows: [...rows, ...page.rows],
+    rows: [...rows, ...page.rows] as Row[],
   }
   return await recursiveSearch(dbName, index, query, newParams)
 }
 
-/**
- * Performs a paginated search. A bookmark will be returned to allow the next
- * page to be fetched. There is a max limit off 200 results per page in a
- * paginated search.
- * @param dbName Which database to run a lucene query on
- * @param index Which search index to utilise
- * @param query The JSON query structure
- * @param params The search params including:
- *   tableId {string} The table ID to search
- *   sort {string} The sort column
- *   sortOrder {string} The sort order ("ascending" or "descending")
- *   sortType {string} Whether to treat sortable values as strings or
- *     numbers. ("string" or "number")
- *   limit {number} The desired page size
- *   bookmark {string} The bookmark to resume from
- * @returns {Promise<{hasNextPage: boolean, rows: *[]}>}
- */
 export async function paginatedSearch<T>(
   dbName: string,
   index: string,
   query: SearchFilters,
-  params: SearchParams<T>
-) {
+  params: SearchParams
+): Promise<SearchResponse<T>> {
   let limit = params.limit
   if (limit == null || isNaN(limit) || limit < 0) {
     limit = 50
@@ -713,29 +695,12 @@ export async function paginatedSearch<T>(
   }
 }
 
-/**
- * Performs a full search, fetching multiple pages if required to return the
- * desired amount of results. There is a limit of 1000 results to avoid
- * heavy performance hits, and to avoid client components breaking from
- * handling too much data.
- * @param dbName Which database to run a lucene query on
- * @param index Which search index to utilise
- * @param query The JSON query structure
- * @param params The search params including:
- *   tableId {string} The table ID to search
- *   sort {string} The sort column
- *   sortOrder {string} The sort order ("ascending" or "descending")
- *   sortType {string} Whether to treat sortable values as strings or
- *     numbers. ("string" or "number")
- *   limit {number} The desired number of results
- * @returns {Promise<{rows: *}>}
- */
 export async function fullSearch<T>(
   dbName: string,
   index: string,
   query: SearchFilters,
-  params: SearchParams<T>
-) {
+  params: SearchParams
+): Promise<{ rows: Row[] }> {
   let limit = params.limit
   if (limit == null || isNaN(limit) || limit < 0) {
     limit = 1000

package/src/db/tests/lucene.spec.ts

@@ -1,23 +1,39 @@
 import { newid } from "../../docIds/newid"
 import { getDB } from "../db"
-import { Database, EmptyFilterOption } from "@budibase/types"
-import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
+import {
+  Database,
+  EmptyFilterOption,
+  SortOrder,
+  SortType,
+  DocumentType,
+  SEPARATOR,
+} from "@budibase/types"
+import { fullSearch, paginatedSearch, QueryBuilder } from "../lucene"
 
 const INDEX_NAME = "main"
+const TABLE_ID = DocumentType.TABLE + SEPARATOR + newid()
 
 const index = `function(doc) {
-  let props = ["property", "number", "array"]
-  for (let key of props) {
-    if (Array.isArray(doc[key])) {
-      for (let val of doc[key]) {
+  if (!doc._id.startsWith("ro_")) {
+    return
+  }
+  let keys = Object.keys(doc).filter(key => !key.startsWith("_"))
+  for (let key of keys) {
+    const value = doc[key]
+    if (Array.isArray(value)) {
+      for (let val of value) {
         index(key, val)
       }
-    } else if (doc[key]) {
-      index(key, doc[key])
+    } else if (value) {
+      index(key, value)
     }
   }
 }`
 
+function rowId(id?: string) {
+  return DocumentType.ROW + SEPARATOR + (id || newid())
+}
+
 describe("lucene", () => {
   let db: Database, dbName: string
 
@@ -25,10 +41,21 @@ describe("lucene", () => {
     dbName = `db-${newid()}`
     // create the DB for testing
     db = getDB(dbName)
-    await db.put({ _id: newid(), property: "word", array: ["1", "4"] })
-    await db.put({ _id: newid(), property: "word2", array: ["3", "1"] })
     await db.put({
-      _id: newid(),
+      _id: rowId(),
+      tableId: TABLE_ID,
+      property: "word",
+      array: ["1", "4"],
+    })
+    await db.put({
+      _id: rowId(),
+      tableId: TABLE_ID,
+      property: "word2",
+      array: ["3", "1"],
+    })
+    await db.put({
+      _id: rowId(),
+      tableId: TABLE_ID,
       property: "word3",
       number: 1,
       array: ["1", "2"],
@@ -240,7 +267,8 @@ describe("lucene", () => {
         docs = Array(QueryBuilder.maxLimit * 2.5)
           .fill(0)
           .map((_, i) => ({
-            _id: i.toString().padStart(3, "0"),
+            _id: rowId(i.toString().padStart(3, "0")),
+            tableId: TABLE_ID,
             property: `value_${i.toString().padStart(3, "0")}`,
             array: [],
           }))
@@ -338,10 +366,11 @@ describe("lucene", () => {
           },
         },
         {
+          tableId: TABLE_ID,
           limit: 1,
           sort: "property",
-          sortType: "string",
-          sortOrder: "desc",
+          sortType: SortType.STRING,
+          sortOrder: SortOrder.DESCENDING,
         }
       )
       expect(page.rows.length).toBe(1)
@@ -360,7 +389,10 @@ describe("lucene", () => {
             property: "wo",
           },
         },
-        {}
+        {
+          tableId: TABLE_ID,
+          query: {},
+        }
       )
       expect(page.rows.length).toBe(3)
     })

package/src/index.ts

@@ -32,7 +32,6 @@ export { default as env } from "./environment"
 export * as blacklist from "./blacklist"
 export * as docUpdates from "./docUpdates"
 export * from "./utils/Duration"
-export { SearchParams } from "./db"
 export * as docIds from "./docIds"
 export * as security from "./security"
 // Add context to tenancy for backwards compatibility

package/tests/core/utilities/testContainerUtils.ts

@@ -77,9 +77,15 @@ export function setupEnv(...envs: any[]) {
     throw new Error("CouchDB port not found")
   }
 
+  const couchSqlPort = getExposedV4Port(couch, 4984)
+  if (!couchSqlPort) {
+    throw new Error("CouchDB SQL port not found")
+  }
+
   const configs = [
     { key: "COUCH_DB_PORT", value: `${couchPort}` },
     { key: "COUCH_DB_URL", value: `http://127.0.0.1:${couchPort}` },
+    { key: "COUCH_DB_SQL_URL", value: `http://127.0.0.1:${couchSqlPort}` },
   ]
 
   for (const config of configs.filter(x => !!x.value)) {