@teleporthq/teleport-plugin-next-data-source 0.44.12 → 0.44.13

package/src/fetchers/supabase.ts

@@ -5,7 +5,7 @@ import {
   generateSearchEscapeHelpersCode,
   generateFilterTreeHelpersCode,
 } from '../utils'
-import { generateSearchWordsCode } from '../search-relevance'
+import { generateSearchTermsCode } from '../search-relevance'
 
 export const validateSupabaseConfig = (
   config: Record<string, unknown>
@@ -132,7 +132,7 @@ ${generateSafeJSONParseCode()}
 ${generateFilterTreeHelpersCode()}
 
 ${generateSearchEscapeHelpersCode()}
-${generateSearchWordsCode()}
+${generateSearchTermsCode(false)}
 
 // Helper function to process filter values
 const processFilterValue = (value) => {
@@ -358,7 +358,7 @@ export default async function handler(req, res) {
         // raw % / _ in the user input still act as wildcards in this
         // backend; the canvas preview path escapes them explicitly. Each
         // typed word gets its own .or(), and PostgREST ANDs them.
-        searchWords(query).forEach((word) => {
+        searchTerms(query).forEach((word) => {
           const rawPattern = "%" + word + "%"
           const escapedForPostgrest = rawPattern
             .replace(/\\\\/g, "\\\\\\\\")
@@ -484,7 +484,7 @@ async function getCount(req, res) {
         // Mirror the fetch handler: sanitize identifiers, wrap the
         // pattern in double quotes for PostgREST .or() DSL, and escape
         // backslashes / quotes in the user's search term, one .or() per word.
-        searchWords(query).forEach((word) => {
+        searchTerms(query).forEach((word) => {
           const rawPattern = "%" + word + "%"
           const escapedForPostgrest = rawPattern
             .replace(/\\\\/g, "\\\\\\\\")

package/src/fetchers/teleport.ts

@@ -20,7 +20,7 @@ import { REQUEST_LOCALE_PARAM } from '../request-locale'
 import { generateReadGuardCall, generateTableAccessPreamble } from './utils/table-access-guard'
 import { generateRequestSqlGuardsCode } from './utils/request-sql-guards'
 import { generateArrayOverlapSqlCode } from '../array-overlap-sql'
-import { generateSearchSqlHelpersCode, PG_SEARCH_MATCH_SQL } from '../search-relevance'
+import { generateSearchSqlHelpersCode, PG_SEARCH_SQL } from '../search-relevance'
 import { PG_CLIENT_IMPORT } from './pg-client-import'
 
 interface TeleportDBConfig {
@@ -163,7 +163,7 @@ ${generateSafeJSONParseCode()}
 ${generateFilterTreeHelpersCode()}
 
 ${generateSearchEscapeHelpersCode()}
-${generateSearchSqlHelpersCode(PG_SEARCH_MATCH_SQL)}
+${generateSearchSqlHelpersCode(PG_SEARCH_SQL)}
 ${generateArrayOverlapSqlCode()}
 ${getTransformationCode(tableName, transformOptions)}
 ${getTransformWrapperCode(tableName, transformOptions)}
@@ -349,6 +349,7 @@ ${generateReadGuardCall(tableName, 'req.query && req.query.rawQuery')}
     const queryParams = []
     let paramIndex = 1
     let searchRank = ''
+    let searchFrom = ''
     
     if (query) {
       let columns = []
@@ -385,13 +386,15 @@ ${generateReadGuardCall(tableName, 'req.query && req.query.rawQuery')}
         queryParams.push(...search.params)
         paramIndex = search.nextIndex
         searchRank = search.rank
+        searchFrom = search.from
       }
     }
     
     paramIndex = processFilters(filters, conditions, queryParams, paramIndex)
     if (__rowPolicy) conditions.push('(' + __rowPolicy.predicate + ')')
     
-    let sql = \`SELECT * FROM ${tableName}\`
+    // A search's worked-out texts stay out of the rows: only the table's own columns.
+    let sql = \`SELECT \${searchFrom ? '${tableName}.*' : '*'} FROM ${tableName}\${searchFrom}\`
     
     if (conditions.length > 0) {
       sql += \` WHERE \${conditions.join(' AND ')}\`
@@ -542,6 +545,7 @@ ${generateReadGuardCall(tableName, null)}
     const conditions = []
     const queryParams = []
     let paramIndex = 1
+    let searchFrom = ''
 
     if (query) {
       let columns = []
@@ -577,13 +581,14 @@ ${generateReadGuardCall(tableName, null)}
         conditions.push(search.where)
         queryParams.push(...search.params)
         paramIndex = search.nextIndex
+        searchFrom = search.from
       }
     }
 
     paramIndex = processFilters(filters, conditions, queryParams, paramIndex)
     if (__rowPolicy) conditions.push('(' + __rowPolicy.predicate + ')')
 
-    let countSql = \`SELECT COUNT(*) FROM ${tableName}\`
+    let countSql = \`SELECT COUNT(*) FROM ${tableName}\${searchFrom}\`
     if (conditions.length > 0) {
       countSql += \` WHERE \${conditions.join(' AND ')}\`
     }

package/src/fetchers/turso.ts

@@ -5,7 +5,7 @@ import {
   generateFilterTreeHelpersCode,
   generateSearchEscapeHelpersCode,
 } from '../utils'
-import { generateSearchSqlHelpersCode, SQLITE_SEARCH_MATCH_SQL } from '../search-relevance'
+import { generateSearchSqlHelpersCode, SQLITE_SEARCH_SQL } from '../search-relevance'
 
 export const validateTursoConfig = (
   config: Record<string, unknown>
@@ -47,7 +47,7 @@ ${generateSafeJSONParseCode()}
 ${generateFilterTreeHelpersCode()}
 
 ${generateSearchEscapeHelpersCode()}
-${generateSearchSqlHelpersCode(SQLITE_SEARCH_MATCH_SQL)}
+${generateSearchSqlHelpersCode(SQLITE_SEARCH_SQL)}
 
 ${generateDateFormatterCode()}
 

package/src/search-relevance.ts

@@ -1,96 +1,230 @@
 /**
  * How a list's search box narrows and orders the rows of a generated route.
  *
- * Every typed word must appear in one of the searched columns, so "create
- * form" finds "How to create a form". The matches are then ordered by where
- * the words are: a row ranks by the first searched column, in the order the
- * list searches them, that holds every word (a title match before a summary
- * match before a body match), and a row whose words are spread over several
- * columns comes after all of those. The list's own sort orders the rows
- * within each of those groups.
+ * The typed text is split into terms at white space, letter case aside, and
+ * the punctuation around a term is dropped ("form?" looks for "form",
+ * "\"create" for "create"). Every term must appear in one of the searched
+ * columns, so "create form" finds "How to create a form".
+ *
+ * Where the stored text is read the same way as the typed text (the
+ * TeleportHQ database and other Postgres sources, and the routes that search
+ * rows in memory), punctuation inside a term counts as a space on both sides
+ * ("e-commerce" finds "E-Commerce", "what's" finds "what’s"), and HTML tags
+ * and entities count as spaces too, so a word that is only in a rich-text
+ * body's markup ("strong", "href", "amp") never matches.
+ *
+ * The matches are ordered column by column, in the order the list searches
+ * them: by how much of the typed text the first column (a title) holds, then
+ * the second (a summary), and so on. A term counts 3 as a whole word, 2 at
+ * the start of a word ("form" in "forms") and 1 inside one ("form" in
+ * "platform"). The list's own sort orders the rows that score alike.
+ */
+
+/**
+ * Characters a search reads as a space: punctuation, and line breaks and
+ * tabs, so a word at the end of a line is still a whole word.
  */
+export const SEARCH_PUNCTUATION =
+  '\t\n\v\f\r!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~\u00a0\u00a1\u00ab\u00b7\u00bb\u00bf' +
+  '\u2010\u2011\u2012\u2013\u2014\u2015\u2018\u2019\u201a\u201b\u201c\u201d\u201e\u2022\u2026\u2039\u203a'
+
+/** A JavaScript string literal that survives being pasted into any generated file. */
+const jsString = (value: string): string =>
+  JSON.stringify(value).replace(
+    /[\u007f-\uffff`$]/g,
+    (char) => '\\u' + ('0000' + char.charCodeAt(0).toString(16)).slice(-4)
+  )
 
-/** `searchWords(query)`: the typed words, lowercased, each once. */
-export const generateSearchWordsCode = (): string => `const searchWords = (query) => {
-  const words = []
-  String(query == null ? '' : query).toLowerCase().split(/\\s+/).forEach((word) => {
-    if (word && words.indexOf(word) === -1) words.push(word)
+/**
+ * One Postgres regular expression, as a string literal, for everything a
+ * normalised search reads as a space: a tag, an entity, or one of
+ * SEARCH_PUNCTUATION. The bracket expression puts `]` first and `-` last,
+ * escapes the backslash, and puts `[` before `-` so it never opens a class.
+ * One pass: `translate()` is several times slower over long multibyte text.
+ */
+const PG_SPACES_REGEX_SQL = `'${(
+  '<[^>]*>|&[a-z0-9#]+;|[]' +
+  SEARCH_PUNCTUATION.split('')
+    .filter((char) => !']-\\['.includes(char))
+    .join('') +
+  '\\\\[-]'
+).replace(/'/g, "''")}'`
+
+/**
+ * `searchTerms(query)`: the typed terms, lowercased, each once. `normalised`
+ * also reads the punctuation inside a term as a space, for the routes whose
+ * stored text is read the same way; otherwise only the punctuation around a
+ * term is dropped, so "e-commerce" still finds the raw text "e-commerce".
+ */
+export const generateSearchTermsCode = (
+  normalised: boolean
+): string => `const SEARCH_PUNCTUATION = ${jsString(SEARCH_PUNCTUATION)}
+const searchPunctuationToSpaces = (text) => {
+  let out = ''
+  for (const char of text) out += SEARCH_PUNCTUATION.indexOf(char) === -1 ? char : ' '
+  return out
+}
+const searchTerms = (query) => {
+  const terms = []
+  String(query == null ? '' : query).toLowerCase().split(/\\s+/).forEach((token) => {
+    const spaced = searchPunctuationToSpaces(token)
+${
+  normalised
+    ? `    const term = spaced.trim().replace(/\\s+/g, ' ')`
+    : `    const start = spaced.length - spaced.trimStart().length
+    const term = token.slice(start, start + spaced.trim().length)`
+}
+    if (term && terms.indexOf(term) === -1) terms.push(term)
   })
-  return words
+  return terms
 }`
 
+/**
+ * How one SQL dialect reads a column for a search. `textSql` is the source
+ * of `(column) => sql`, the column's text as the search reads it: lower case,
+ * with a space at each end so a whole word and a word start can be told
+ * apart. `likeSql` is the source of `(text, placeholder, before, after) =>
+ * sql`, testing `text LIKE before + term + after` with the term bound to the
+ * placeholder. `normalised` dialects also read tags, entities and punctuation
+ * in the stored text as spaces; `lateral` ones work each column's text out
+ * once per row, in a LATERAL subquery the route joins after its table.
+ */
+export interface SearchSqlDialect {
+  textSql: string
+  likeSql: string
+  normalised: boolean
+  lateral: boolean
+}
+
+const PG_LIKE_SQL = `(text, placeholder, before, after) => text + " LIKE ('" + before + "' || " + placeholder + "::text || '" + after + "') ESCAPE '|'"`
+
+export const PG_SEARCH_SQL: SearchSqlDialect = {
+  textSql: `(column) => "(' ' || regexp_replace(lower(\\"" + column + "\\"::text), " + SEARCH_SPACES_REGEX_SQL + ", ' ', 'g') || ' ')"`,
+  likeSql: PG_LIKE_SQL,
+  normalised: true,
+  lateral: true,
+}
+
+export const REDSHIFT_SEARCH_SQL: SearchSqlDialect = {
+  textSql: `(column) => "(' ' || lower(\\"" + column + "\\"::text) || ' ')"`,
+  likeSql: PG_LIKE_SQL,
+  normalised: false,
+  lateral: false,
+}
+
+const CONCAT_LIKE_SQL = `(text, placeholder, before, after) => text + " LIKE CONCAT('" + before + "', " + placeholder + ", '" + after + "') ESCAPE '|'"`
+
+export const MYSQL_SEARCH_SQL: SearchSqlDialect = {
+  textSql: `(column) => "CONCAT(' ', LOWER(CAST(" + mysql.escapeId(column) + " AS CHAR)), ' ')"`,
+  likeSql: CONCAT_LIKE_SQL,
+  normalised: false,
+  lateral: false,
+}
+
+export const MARIADB_SEARCH_SQL: SearchSqlDialect = {
+  textSql: `(column) => "CONCAT(' ', LOWER(CAST(\\\`" + column + "\\\` AS CHAR)), ' ')"`,
+  likeSql: CONCAT_LIKE_SQL,
+  normalised: false,
+  lateral: false,
+}
+
+export const SQLITE_SEARCH_SQL: SearchSqlDialect = {
+  textSql: `(column) => "(' ' || LOWER(CAST(\\"" + column + "\\" AS TEXT)) || ' ')"`,
+  likeSql: `(text, placeholder, before, after) => text + " LIKE ('" + before + "' || " + placeholder + " || '" + after + "') ESCAPE '|'"`,
+  normalised: false,
+  lateral: false,
+}
+
 /**
  * `buildSearchSql(query, columns, firstIndex)` for the SQL routes:
- * `{ where, params, rank, rankParams, nextIndex }`, or null when there is
- * nothing to search. `matchSql` is the dialect's case-insensitive substring
- * test, `(column, placeholder) => sql`. With `firstIndex` the placeholders are
- * numbered (`$n`) and each word's pattern is bound once, the rank reusing it;
- * without it they are positional (`?`), every occurrence binds its own copy,
- * and `rankParams` go after the parameters of the whole WHERE, as the ORDER BY
- * comes after it. `rank` is '' with fewer than two columns.
+ * `{ where, params, from, rank, rankParams, nextIndex }`, or null when there
+ * is nothing to search. `from` is the LATERAL join a `lateral` dialect's
+ * route appends after its table (selecting `table.*`, so the worked-out
+ * texts never reach a row), '' otherwise. `rank` is the ORDER BY keys, one
+ * per searched column, best first.
+ *
+ * With `firstIndex` the placeholders are numbered (`$n`) and each term is
+ * bound once, the rank reusing it; without it they are positional (`?`),
+ * every occurrence binds its own copy, and `rankParams` go after the
+ * parameters of the whole WHERE, as the ORDER BY comes after it.
  *
  * Uses `escapeLikePattern` and `sanitizeSearchIdentifier`
  * (`generateSearchEscapeHelpersCode`).
  */
 export const generateSearchSqlHelpersCode = (
-  matchSql: string
-): string => `${generateSearchWordsCode()}
-const searchMatchSql = ${matchSql}
+  dialect: SearchSqlDialect
+): string => `${generateSearchTermsCode(dialect.normalised)}
+${
+  dialect.normalised ? `const SEARCH_SPACES_REGEX_SQL = ${jsString(PG_SPACES_REGEX_SQL)}\n` : ''
+}const searchTextSql = ${dialect.textSql}
+const searchLikeSql = ${dialect.likeSql}
 const buildSearchSql = (query, columns, firstIndex) => {
-  const words = searchWords(query)
+  const terms = searchTerms(query)
   const searched = []
   ;(Array.isArray(columns) ? columns : []).forEach((column) => {
     const name = sanitizeSearchIdentifier(column)
     if (searched.indexOf(name) === -1) searched.push(name)
   })
-  if (words.length === 0 || searched.length === 0) return null
-  const patterns = words.map((word) => '%' + escapeLikePattern(word) + '%')
+  if (terms.length === 0 || searched.length === 0) return null
+  const patterns = terms.map((term) => escapeLikePattern(term))
   const numbered = typeof firstIndex === 'number'
   const params = numbered ? patterns.slice() : []
   const rankParams = []
-  const placeholder = (wordIndex, bound) => {
-    if (numbered) return '$' + (firstIndex + wordIndex)
-    bound.push(patterns[wordIndex])
+  const placeholder = (termIndex, bound) => {
+    if (numbered) return '$' + (firstIndex + termIndex)
+    bound.push(patterns[termIndex])
     return '?'
   }
-  const where = '(' + patterns.map((pattern, wordIndex) =>
-    '(' + searched.map((column) => searchMatchSql(column, placeholder(wordIndex, params))).join(' OR ') + ')'
+  const texts = searched.map((column, position) => ${
+    dialect.lateral ? `'"__tq_search"."__tq_search_' + position + '"'` : 'searchTextSql(column)'
+  })
+  const from = ${
+    dialect.lateral
+      ? `' CROSS JOIN LATERAL (SELECT ' + searched.map((column, position) => searchTextSql(column) + ' AS "__tq_search_' + position + '"').join(', ') + ' OFFSET 0) AS "__tq_search"'`
+      : `''`
+  }
+  const where = '(' + patterns.map((pattern, termIndex) =>
+    '(' + texts.map((text) => searchLikeSql(text, placeholder(termIndex, params), '%', '%')).join(' OR ') + ')'
   ).join(' AND ') + ')'
-  const rank = searched.length < 2 ? '' : 'CASE ' + searched.map((column, position) =>
-    'WHEN ' + patterns.map((pattern, wordIndex) => searchMatchSql(column, placeholder(wordIndex, rankParams))).join(' AND ') + ' THEN ' + position
-  ).join(' ') + ' ELSE ' + searched.length + ' END'
-  return { where, params, rank, rankParams, nextIndex: numbered ? firstIndex + patterns.length : firstIndex }
+  const rank = texts.map((text) => '(' + patterns.map((pattern, termIndex) =>
+    'CASE WHEN ' + searchLikeSql(text, placeholder(termIndex, rankParams), '% ', ' %') + ' THEN 3' +
+    ' WHEN ' + searchLikeSql(text, placeholder(termIndex, rankParams), '% ', '%') + ' THEN 2' +
+    ' WHEN ' + searchLikeSql(text, placeholder(termIndex, rankParams), '%', '%') + ' THEN 1 ELSE 0 END'
+  ).join(' + ') + ') DESC').join(', ')
+  return { where, params, from, rank, rankParams, nextIndex: numbered ? firstIndex + patterns.length : firstIndex }
 }`
 
 /**
- * `rankBySearch(rows, query, columns, valueOf)` for the routes that search
- * rows in memory: the same order as the SQL routes' rank, applied after the
- * list's sort (a stable sort keeps it within each group) and before the page
- * is cut.
+ * For the routes that search rows in memory: `matchesSearch(row, query,
+ * columns, valueOf)`, true when every term is in one of the columns, and
+ * `rankBySearch(rows, query, columns, valueOf)`, the same order as the SQL
+ * routes' rank, applied after the list's sort (a stable sort keeps it among
+ * rows that score alike) and before the page is cut. Stored text is read
+ * the normalised way: tags, entities and punctuation as spaces.
  */
-export const generateSearchRankHelperCode = (): string => `${generateSearchWordsCode()}
+export const generateSearchRankHelperCode = (): string => `${generateSearchTermsCode(true)}
+const searchText = (value) => {
+  const text = value === null || value === undefined ? '' : typeof value === 'object' ? JSON.stringify(value) : String(value)
+  return ' ' + searchPunctuationToSpaces(text.toLowerCase().replace(/<[^>]*>|&[a-z0-9#]+;/g, ' ')) + ' '
+}
+const searchScore = (text, terms) => terms.reduce((score, term) =>
+  score + (text.indexOf(' ' + term + ' ') !== -1 ? 3 : text.indexOf(' ' + term) !== -1 ? 2 : text.indexOf(term) !== -1 ? 1 : 0), 0)
+const matchesSearch = (row, query, columns, valueOf) => {
+  const terms = searchTerms(query)
+  if (terms.length === 0) return true
+  const texts = (Array.isArray(columns) ? columns : []).map((column) => searchText(valueOf(row, column)))
+  return terms.every((term) => texts.some((text) => text.indexOf(term) !== -1))
+}
 const rankBySearch = (rows, query, columns, valueOf) => {
-  const words = searchWords(query)
-  if (!Array.isArray(columns) || columns.length < 2 || words.length === 0) return rows
-  const tierOf = (row) => {
-    for (let position = 0; position < columns.length; position++) {
-      const value = valueOf(row, columns[position])
-      const text = value === null || value === undefined ? '' : String(value).toLowerCase()
-      if (words.every((word) => text.includes(word))) return position
-    }
-    return columns.length
-  }
+  const terms = searchTerms(query)
+  if (!Array.isArray(columns) || columns.length === 0 || terms.length === 0) return rows
   return rows
-    .map((row, index) => ({ row, index, tier: tierOf(row) }))
-    .sort((a, b) => a.tier - b.tier || a.index - b.index)
+    .map((row, index) => ({ row, index, scores: columns.map((column) => searchScore(searchText(valueOf(row, column)), terms)) }))
+    .sort((a, b) => {
+      for (let position = 0; position < a.scores.length; position++) {
+        if (a.scores[position] !== b.scores[position]) return b.scores[position] - a.scores[position]
+      }
+      return a.index - b.index
+    })
     .map((entry) => entry.row)
 }`
-
-/** Each dialect's case-insensitive substring match, for `generateSearchSqlHelpersCode`. */
-export const PG_SEARCH_MATCH_SQL = `(column, placeholder) => '"' + column + '"::text ILIKE ' + placeholder + " ESCAPE '|'"`
-
-export const MYSQL_SEARCH_MATCH_SQL = `(column, placeholder) => 'LOWER(CAST(' + mysql.escapeId(column) + ' AS CHAR)) LIKE LOWER(' + placeholder + ") ESCAPE '|'"`
-
-export const MARIADB_SEARCH_MATCH_SQL = `(column, placeholder) => 'LOWER(CAST(\`' + column + '\` AS CHAR)) LIKE LOWER(' + placeholder + ") ESCAPE '|'"`
-
-export const SQLITE_SEARCH_MATCH_SQL = `(column, placeholder) => 'LOWER(CAST("' + column + '" AS TEXT)) LIKE LOWER(' + placeholder + ") ESCAPE '|'"`