@teleporthq/teleport-plugin-next-data-source 0.44.12 → 0.44.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/__tests__/browser-row-policy.test.ts +6 -6
- package/__tests__/fetchers.test.ts +1 -1
- package/__tests__/items-path.test.ts +3 -2
- package/__tests__/search-relevance.test.ts +199 -52
- package/dist/cjs/fetchers/csv-file.d.ts.map +1 -1
- package/dist/cjs/fetchers/csv-file.js +1 -1
- package/dist/cjs/fetchers/csv-file.js.map +1 -1
- package/dist/cjs/fetchers/google-sheets.d.ts.map +1 -1
- package/dist/cjs/fetchers/google-sheets.js +1 -1
- package/dist/cjs/fetchers/google-sheets.js.map +1 -1
- package/dist/cjs/fetchers/javascript.d.ts.map +1 -1
- package/dist/cjs/fetchers/javascript.js +1 -1
- package/dist/cjs/fetchers/javascript.js.map +1 -1
- package/dist/cjs/fetchers/mariadb.js +1 -1
- package/dist/cjs/fetchers/mariadb.js.map +1 -1
- package/dist/cjs/fetchers/mysql.js +1 -1
- package/dist/cjs/fetchers/mysql.js.map +1 -1
- package/dist/cjs/fetchers/postgresql.d.ts.map +1 -1
- package/dist/cjs/fetchers/postgresql.js +3 -3
- package/dist/cjs/fetchers/postgresql.js.map +1 -1
- package/dist/cjs/fetchers/redshift.js +1 -1
- package/dist/cjs/fetchers/rest-api.d.ts.map +1 -1
- package/dist/cjs/fetchers/rest-api.js +1 -1
- package/dist/cjs/fetchers/rest-api.js.map +1 -1
- package/dist/cjs/fetchers/supabase.js +2 -2
- package/dist/cjs/fetchers/supabase.js.map +1 -1
- package/dist/cjs/fetchers/teleport.d.ts.map +1 -1
- package/dist/cjs/fetchers/teleport.js +4 -4
- package/dist/cjs/fetchers/teleport.js.map +1 -1
- package/dist/cjs/fetchers/turso.js +1 -1
- package/dist/cjs/fetchers/turso.js.map +1 -1
- package/dist/cjs/search-relevance.d.ts +67 -26
- package/dist/cjs/search-relevance.d.ts.map +1 -1
- package/dist/cjs/search-relevance.js +102 -29
- package/dist/cjs/search-relevance.js.map +1 -1
- package/dist/cjs/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/fetchers/csv-file.d.ts.map +1 -1
- package/dist/esm/fetchers/csv-file.js +1 -1
- package/dist/esm/fetchers/csv-file.js.map +1 -1
- package/dist/esm/fetchers/google-sheets.d.ts.map +1 -1
- package/dist/esm/fetchers/google-sheets.js +1 -1
- package/dist/esm/fetchers/google-sheets.js.map +1 -1
- package/dist/esm/fetchers/javascript.d.ts.map +1 -1
- package/dist/esm/fetchers/javascript.js +1 -1
- package/dist/esm/fetchers/javascript.js.map +1 -1
- package/dist/esm/fetchers/mariadb.js +2 -2
- package/dist/esm/fetchers/mariadb.js.map +1 -1
- package/dist/esm/fetchers/mysql.js +2 -2
- package/dist/esm/fetchers/mysql.js.map +1 -1
- package/dist/esm/fetchers/postgresql.d.ts.map +1 -1
- package/dist/esm/fetchers/postgresql.js +4 -4
- package/dist/esm/fetchers/postgresql.js.map +1 -1
- package/dist/esm/fetchers/redshift.js +2 -2
- package/dist/esm/fetchers/rest-api.d.ts.map +1 -1
- package/dist/esm/fetchers/rest-api.js +1 -1
- package/dist/esm/fetchers/rest-api.js.map +1 -1
- package/dist/esm/fetchers/supabase.js +3 -3
- package/dist/esm/fetchers/supabase.js.map +1 -1
- package/dist/esm/fetchers/teleport.d.ts.map +1 -1
- package/dist/esm/fetchers/teleport.js +5 -5
- package/dist/esm/fetchers/teleport.js.map +1 -1
- package/dist/esm/fetchers/turso.js +2 -2
- package/dist/esm/fetchers/turso.js.map +1 -1
- package/dist/esm/search-relevance.d.ts +67 -26
- package/dist/esm/search-relevance.d.ts.map +1 -1
- package/dist/esm/search-relevance.js +100 -27
- package/dist/esm/search-relevance.js.map +1 -1
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
- package/src/fetchers/csv-file.ts +6 -11
- package/src/fetchers/google-sheets.ts +6 -12
- package/src/fetchers/javascript.ts +5 -12
- package/src/fetchers/mariadb.ts +2 -2
- package/src/fetchers/mysql.ts +2 -2
- package/src/fetchers/postgresql.ts +9 -4
- package/src/fetchers/redshift.ts +2 -2
- package/src/fetchers/rest-api.ts +4 -11
- package/src/fetchers/supabase.ts +4 -4
- package/src/fetchers/teleport.ts +9 -4
- package/src/fetchers/turso.ts +2 -2
- package/src/search-relevance.ts +195 -61
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { replaceSecretReference, generateDateFormatterCode, generateSafeJSONParseCode, generateFilterTreeHelpersCode, generateSearchEscapeHelpersCode, } from '../utils';
|
|
2
|
-
import { generateSearchSqlHelpersCode,
|
|
2
|
+
import { generateSearchSqlHelpersCode, SQLITE_SEARCH_SQL } from '../search-relevance';
|
|
3
3
|
export var validateTursoConfig = function (config) {
|
|
4
4
|
if (!config || typeof config !== 'object') {
|
|
5
5
|
return { isValid: false, error: 'Config must be a valid object' };
|
|
@@ -16,6 +16,6 @@ export var generateTursoFetcher = function (config, tableName) {
|
|
|
16
16
|
var tursoConfig = config;
|
|
17
17
|
var databaseUrl = tursoConfig.databaseUrl;
|
|
18
18
|
var token = tursoConfig.token;
|
|
19
|
-
return "import { createClient } from '@libsql/client'\n\n".concat(generateSafeJSONParseCode(), "\n\n").concat(generateFilterTreeHelpersCode(), "\n\n").concat(generateSearchEscapeHelpersCode(), "\n").concat(generateSearchSqlHelpersCode(
|
|
19
|
+
return "import { createClient } from '@libsql/client'\n\n".concat(generateSafeJSONParseCode(), "\n\n").concat(generateFilterTreeHelpersCode(), "\n\n").concat(generateSearchEscapeHelpersCode(), "\n").concat(generateSearchSqlHelpersCode(SQLITE_SEARCH_SQL), "\n\n").concat(generateDateFormatterCode(), "\n\nexport default async function handler(req, res) {\n let client = null\n try {\n client = createClient({\n url: ").concat(JSON.stringify(databaseUrl), ",\n authToken: ").concat(replaceSecretReference(token), "\n })\n \n const { query, queryColumns, limit, page, perPage, sortBy, sortOrder, filters, sorts, offset } = req.query\n \n let sql = `SELECT * FROM ").concat(tableName, "`\n const whereClauses = []\n const queryParams = []\n let searchQueryColumns = null\n let search = null\n \n if (query) {\n if (queryColumns) {\n const parsed = safeJSONParse(queryColumns)\n const columns = Array.isArray(parsed) ? parsed : [parsed]\n // Cast columns to TEXT and LOWER both sides so the match is\n // case-insensitive regardless of SQLite collation.\n search = buildSearchSql(query, columns)\n if (search) {\n whereClauses.push(search.where)\n queryParams.push(...search.params)\n }\n } else {\n // Store query for post-filtering if columns not specified\n searchQueryColumns = query\n }\n }\n \n // Helper to sanitize identifier (prevent SQL injection in column names)\n const sanitizeIdentifier = (name) => {\n // Only allow alphanumeric and underscore\n if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {\n throw new Error(`Invalid identifier: ${name}`)\n }\n return `\"${name}\"`\n }\n \n if (filters) {\n const filterTree = normalizeFilterTree(safeJSONParse(filters))\n \n if (filterTree) {\n const buildCondition = (condition) => {\n const field = sanitizeIdentifier(condition.source)\n const normalizedCondition = normalizeInOperand(condition.operand, condition.destination)\n if (normalizedCondition === null) return null\n const value = normalizedCondition.value\n const operand = normalizedCondition.operand\n \n if (Array.isArray(value)) {\n if (value.length === 0) return null\n const placeholders = value.map(() => '?').join(', ')\n queryParams.push(...value)\n return operand === '!='\n ? `${field} NOT IN (${placeholders})`\n : `${field} IN (${placeholders})`\n }\n \n if (value === null) {\n if (operand === '=') return `${field} IS NULL`\n if (operand === '!=') return `${field} IS NOT NULL`\n return null\n }\n \n const validOps = ['=', '!=', '>', '<', '>=', '<=']\n const sqlOperator = validOps.includes(operand) ? operand : '='\n queryParams.push(value)\n return `${field} ${sqlOperator} ?`\n }\n \n const clause = buildFilterTreeClause(filterTree, buildCondition)\n if (clause) whereClauses.push(clause)\n }\n }\n \n if (whereClauses.length > 0) {\n sql += ` WHERE ${whereClauses.join(' AND ')}`\n }\n \n // Handle sorts - new array format. A search orders its matches by where\n // the words are first; the sort orders the rows within each of those groups.\n const orderClauses = search && search.rank ? [search.rank] : []\n if (sorts) {\n const parsedSorts = safeJSONParse(sorts)\n if (Array.isArray(parsedSorts) && parsedSorts.length > 0) {\n orderClauses.push(...parsedSorts.map((sort) => {\n if (!sort.field) return null\n const order = (sort.order || '').toUpperCase().startsWith('DESC') ? 'DESC' : 'ASC'\n return `${sanitizeIdentifier(sort.field)} ${order}`\n }).filter(Boolean))\n }\n } else if (sortBy) {\n const sortOrderValue = (sortOrder || '').toUpperCase().startsWith('DESC') ? 'DESC' : 'ASC'\n orderClauses.push(`${sanitizeIdentifier(sortBy)} ${sortOrderValue}`)\n }\n if (orderClauses.length > 0) {\n sql += ` ORDER BY ${orderClauses.join(', ')}`\n if (search && search.rank) queryParams.push(...search.rankParams)\n }\n \n const limitValue = limit || perPage\n const offsetValue = offset !== undefined ? parseInt(offset) : (page && perPage ? (parseInt(page) - 1) * parseInt(perPage) : undefined)\n \n // Only apply SQL pagination if we're not doing post-filtering\n if (!searchQueryColumns) {\n if (limitValue) {\n sql += ` LIMIT ?`\n queryParams.push(parseInt(limitValue))\n }\n \n if (offsetValue !== undefined) {\n sql += ` OFFSET ?`\n queryParams.push(offsetValue)\n }\n }\n \n const result = await client.execute({\n sql,\n args: queryParams\n })\n \n let data = result.rows.map((row) => {\n const obj = {}\n result.columns.forEach((col, idx) => {\n obj[col] = row[col]\n })\n return obj\n })\n \n // Apply post-filtering for search without queryColumns\n if (searchQueryColumns) {\n const searchTerms = searchQueryColumns.toLowerCase().split(' ').filter(Boolean)\n data = data.filter((item) => {\n try {\n const stringified = JSON.stringify(item).toLowerCase()\n return searchTerms.every((term) => stringified.includes(term))\n } catch {\n return false\n }\n })\n \n // Apply pagination after filtering\n if (limitValue) {\n const start = offsetValue || 0\n data = data.slice(start, start + parseInt(limitValue))\n } else if (offsetValue) {\n data = data.slice(offsetValue)\n }\n }\n \n const safeData = JSON.parse(JSON.stringify(data, dateReplacer))\n \n return res.status(200).json({\n success: true,\n data: safeData,\n timestamp: Date.now()\n })\n } catch (error) {\n console.error('Turso fetch error:', error)\n return res.status(500).json({\n success: false,\n error: error.message || 'Failed to fetch data',\n timestamp: Date.now()\n })\n } finally {\n if (client) {\n try {\n await client.close()\n } catch (error) {\n console.error('Error closing Turso client:', error)\n }\n }\n }\n}\n");
|
|
20
20
|
};
|
|
21
21
|
//# sourceMappingURL=turso.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"turso.js","sourceRoot":"","sources":["../../../src/fetchers/turso.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,sBAAsB,EACtB,yBAAyB,EACzB,yBAAyB,EACzB,6BAA6B,EAC7B,+BAA+B,GAChC,MAAM,UAAU,CAAA;AACjB,OAAO,EAAE,4BAA4B,EAAE,
|
|
1
|
+
{"version":3,"file":"turso.js","sourceRoot":"","sources":["../../../src/fetchers/turso.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,sBAAsB,EACtB,yBAAyB,EACzB,yBAAyB,EACzB,6BAA6B,EAC7B,+BAA+B,GAChC,MAAM,UAAU,CAAA;AACjB,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AAErF,MAAM,CAAC,IAAM,mBAAmB,GAAG,UACjC,MAA+B;IAE/B,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE;QACzC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,+BAA+B,EAAE,CAAA;KAClE;IAED,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ,EAAE;QACjE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAA;KACnE;IAED,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE;QACrD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,wCAAwC,EAAE,CAAA;KAC3E;IAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;AAC1B,CAAC,CAAA;AASD,MAAM,CAAC,IAAM,oBAAoB,GAAG,UAClC,MAA+B,EAC/B,SAAiB;IAEjB,IAAM,WAAW,GAAG,MAAqB,CAAA;IACzC,IAAM,WAAW,GAAG,WAAW,CAAC,WAAW,CAAA;IAC3C,IAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAA;IAE/B,OAAO,2DAEP,yBAAyB,EAAE,iBAE3B,6BAA6B,EAAE,iBAE/B,+BAA+B,EAAE,eACjC,4BAA4B,CAAC,iBAAiB,CAAC,iBAE/C,yBAAyB,EAAE,0IAMhB,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,iCACrB,sBAAsB,CAAC,KAAK,CAAC,gLAKhB,SAAS,spLAwKxC,CAAA;AACD,CAAC,CAAA"}
|
|
@@ -1,40 +1,81 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* How a list's search box narrows and orders the rows of a generated route.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
4
|
+
* The typed text is split into terms at white space, letter case aside, and
|
|
5
|
+
* the punctuation around a term is dropped ("form?" looks for "form",
|
|
6
|
+
* "\"create" for "create"). Every term must appear in one of the searched
|
|
7
|
+
* columns, so "create form" finds "How to create a form".
|
|
8
|
+
*
|
|
9
|
+
* Where the stored text is read the same way as the typed text (the
|
|
10
|
+
* TeleportHQ database and other Postgres sources, and the routes that search
|
|
11
|
+
* rows in memory), punctuation inside a term counts as a space on both sides
|
|
12
|
+
* ("e-commerce" finds "E-Commerce", "what's" finds "what’s"), and HTML tags
|
|
13
|
+
* and entities count as spaces too, so a word that is only in a rich-text
|
|
14
|
+
* body's markup ("strong", "href", "amp") never matches.
|
|
15
|
+
*
|
|
16
|
+
* The matches are ordered column by column, in the order the list searches
|
|
17
|
+
* them: by how much of the typed text the first column (a title) holds, then
|
|
18
|
+
* the second (a summary), and so on. A term counts 3 as a whole word, 2 at
|
|
19
|
+
* the start of a word ("form" in "forms") and 1 inside one ("form" in
|
|
20
|
+
* "platform"). The list's own sort orders the rows that score alike.
|
|
11
21
|
*/
|
|
12
|
-
/**
|
|
13
|
-
|
|
22
|
+
/**
|
|
23
|
+
* Characters a search reads as a space: punctuation, and line breaks and
|
|
24
|
+
* tabs, so a word at the end of a line is still a whole word.
|
|
25
|
+
*/
|
|
26
|
+
export declare const SEARCH_PUNCTUATION: string;
|
|
27
|
+
/**
|
|
28
|
+
* `searchTerms(query)`: the typed terms, lowercased, each once. `normalised`
|
|
29
|
+
* also reads the punctuation inside a term as a space, for the routes whose
|
|
30
|
+
* stored text is read the same way; otherwise only the punctuation around a
|
|
31
|
+
* term is dropped, so "e-commerce" still finds the raw text "e-commerce".
|
|
32
|
+
*/
|
|
33
|
+
export declare const generateSearchTermsCode: (normalised: boolean) => string;
|
|
34
|
+
/**
|
|
35
|
+
* How one SQL dialect reads a column for a search. `textSql` is the source
|
|
36
|
+
* of `(column) => sql`, the column's text as the search reads it: lower case,
|
|
37
|
+
* with a space at each end so a whole word and a word start can be told
|
|
38
|
+
* apart. `likeSql` is the source of `(text, placeholder, before, after) =>
|
|
39
|
+
* sql`, testing `text LIKE before + term + after` with the term bound to the
|
|
40
|
+
* placeholder. `normalised` dialects also read tags, entities and punctuation
|
|
41
|
+
* in the stored text as spaces; `lateral` ones work each column's text out
|
|
42
|
+
* once per row, in a LATERAL subquery the route joins after its table.
|
|
43
|
+
*/
|
|
44
|
+
export interface SearchSqlDialect {
|
|
45
|
+
textSql: string;
|
|
46
|
+
likeSql: string;
|
|
47
|
+
normalised: boolean;
|
|
48
|
+
lateral: boolean;
|
|
49
|
+
}
|
|
50
|
+
export declare const PG_SEARCH_SQL: SearchSqlDialect;
|
|
51
|
+
export declare const REDSHIFT_SEARCH_SQL: SearchSqlDialect;
|
|
52
|
+
export declare const MYSQL_SEARCH_SQL: SearchSqlDialect;
|
|
53
|
+
export declare const MARIADB_SEARCH_SQL: SearchSqlDialect;
|
|
54
|
+
export declare const SQLITE_SEARCH_SQL: SearchSqlDialect;
|
|
14
55
|
/**
|
|
15
56
|
* `buildSearchSql(query, columns, firstIndex)` for the SQL routes:
|
|
16
|
-
* `{ where, params, rank, rankParams, nextIndex }`, or null when there
|
|
17
|
-
* nothing to search. `
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
57
|
+
* `{ where, params, from, rank, rankParams, nextIndex }`, or null when there
|
|
58
|
+
* is nothing to search. `from` is the LATERAL join a `lateral` dialect's
|
|
59
|
+
* route appends after its table (selecting `table.*`, so the worked-out
|
|
60
|
+
* texts never reach a row), '' otherwise. `rank` is the ORDER BY keys, one
|
|
61
|
+
* per searched column, best first.
|
|
62
|
+
*
|
|
63
|
+
* With `firstIndex` the placeholders are numbered (`$n`) and each term is
|
|
64
|
+
* bound once, the rank reusing it; without it they are positional (`?`),
|
|
65
|
+
* every occurrence binds its own copy, and `rankParams` go after the
|
|
66
|
+
* parameters of the whole WHERE, as the ORDER BY comes after it.
|
|
23
67
|
*
|
|
24
68
|
* Uses `escapeLikePattern` and `sanitizeSearchIdentifier`
|
|
25
69
|
* (`generateSearchEscapeHelpersCode`).
|
|
26
70
|
*/
|
|
27
|
-
export declare const generateSearchSqlHelpersCode: (
|
|
71
|
+
export declare const generateSearchSqlHelpersCode: (dialect: SearchSqlDialect) => string;
|
|
28
72
|
/**
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
73
|
+
* For the routes that search rows in memory: `matchesSearch(row, query,
|
|
74
|
+
* columns, valueOf)`, true when every term is in one of the columns, and
|
|
75
|
+
* `rankBySearch(rows, query, columns, valueOf)`, the same order as the SQL
|
|
76
|
+
* routes' rank, applied after the list's sort (a stable sort keeps it among
|
|
77
|
+
* rows that score alike) and before the page is cut. Stored text is read
|
|
78
|
+
* the normalised way: tags, entities and punctuation as spaces.
|
|
33
79
|
*/
|
|
34
80
|
export declare const generateSearchRankHelperCode: () => string;
|
|
35
|
-
/** Each dialect's case-insensitive substring match, for `generateSearchSqlHelpersCode`. */
|
|
36
|
-
export declare const PG_SEARCH_MATCH_SQL = "(column, placeholder) => '\"' + column + '\"::text ILIKE ' + placeholder + \" ESCAPE '|'\"";
|
|
37
|
-
export declare const MYSQL_SEARCH_MATCH_SQL = "(column, placeholder) => 'LOWER(CAST(' + mysql.escapeId(column) + ' AS CHAR)) LIKE LOWER(' + placeholder + \") ESCAPE '|'\"";
|
|
38
|
-
export declare const MARIADB_SEARCH_MATCH_SQL = "(column, placeholder) => 'LOWER(CAST(`' + column + '` AS CHAR)) LIKE LOWER(' + placeholder + \") ESCAPE '|'\"";
|
|
39
|
-
export declare const SQLITE_SEARCH_MATCH_SQL = "(column, placeholder) => 'LOWER(CAST(\"' + column + '\" AS TEXT)) LIKE LOWER(' + placeholder + \") ESCAPE '|'\"";
|
|
40
81
|
//# sourceMappingURL=search-relevance.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search-relevance.d.ts","sourceRoot":"","sources":["../../src/search-relevance.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"search-relevance.d.ts","sourceRoot":"","sources":["../../src/search-relevance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;GAGG;AACH,eAAO,MAAM,kBAAkB,QAE2E,CAAA;AAwB1G;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,eACtB,OAAO,KAClB,MAmBD,CAAA;AAEF;;;;;;;;;GASG;AACH,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,OAAO,CAAA;IACnB,OAAO,EAAE,OAAO,CAAA;CACjB;AAID,eAAO,MAAM,aAAa,EAAE,gBAK3B,CAAA;AAED,eAAO,MAAM,mBAAmB,EAAE,gBAKjC,CAAA;AAID,eAAO,MAAM,gBAAgB,EAAE,gBAK9B,CAAA;AAED,eAAO,MAAM,kBAAkB,EAAE,gBAKhC,CAAA;AAED,eAAO,MAAM,iBAAiB,EAAE,gBAK/B,CAAA;AAED;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,4BAA4B,YAC9B,gBAAgB,KACxB,MAuCD,CAAA;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,4BAA4B,QAAO,MAyB9C,CAAA"}
|
|
@@ -1,40 +1,113 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* How a list's search box narrows and orders the rows of a generated route.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
4
|
+
* The typed text is split into terms at white space, letter case aside, and
|
|
5
|
+
* the punctuation around a term is dropped ("form?" looks for "form",
|
|
6
|
+
* "\"create" for "create"). Every term must appear in one of the searched
|
|
7
|
+
* columns, so "create form" finds "How to create a form".
|
|
8
|
+
*
|
|
9
|
+
* Where the stored text is read the same way as the typed text (the
|
|
10
|
+
* TeleportHQ database and other Postgres sources, and the routes that search
|
|
11
|
+
* rows in memory), punctuation inside a term counts as a space on both sides
|
|
12
|
+
* ("e-commerce" finds "E-Commerce", "what's" finds "what’s"), and HTML tags
|
|
13
|
+
* and entities count as spaces too, so a word that is only in a rich-text
|
|
14
|
+
* body's markup ("strong", "href", "amp") never matches.
|
|
15
|
+
*
|
|
16
|
+
* The matches are ordered column by column, in the order the list searches
|
|
17
|
+
* them: by how much of the typed text the first column (a title) holds, then
|
|
18
|
+
* the second (a summary), and so on. A term counts 3 as a whole word, 2 at
|
|
19
|
+
* the start of a word ("form" in "forms") and 1 inside one ("form" in
|
|
20
|
+
* "platform"). The list's own sort orders the rows that score alike.
|
|
11
21
|
*/
|
|
12
|
-
/**
|
|
13
|
-
|
|
22
|
+
/**
|
|
23
|
+
* Characters a search reads as a space: punctuation, and line breaks and
|
|
24
|
+
* tabs, so a word at the end of a line is still a whole word.
|
|
25
|
+
*/
|
|
26
|
+
export var SEARCH_PUNCTUATION = '\t\n\v\f\r!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~\u00a0\u00a1\u00ab\u00b7\u00bb\u00bf' +
|
|
27
|
+
'\u2010\u2011\u2012\u2013\u2014\u2015\u2018\u2019\u201a\u201b\u201c\u201d\u201e\u2022\u2026\u2039\u203a';
|
|
28
|
+
/** A JavaScript string literal that survives being pasted into any generated file. */
|
|
29
|
+
var jsString = function (value) {
|
|
30
|
+
return JSON.stringify(value).replace(/[\u007f-\uffff`$]/g, function (char) { return '\\u' + ('0000' + char.charCodeAt(0).toString(16)).slice(-4); });
|
|
31
|
+
};
|
|
32
|
+
/**
|
|
33
|
+
* One Postgres regular expression, as a string literal, for everything a
|
|
34
|
+
* normalised search reads as a space: a tag, an entity, or one of
|
|
35
|
+
* SEARCH_PUNCTUATION. The bracket expression puts `]` first and `-` last,
|
|
36
|
+
* escapes the backslash, and puts `[` before `-` so it never opens a class.
|
|
37
|
+
* One pass: `translate()` is several times slower over long multibyte text.
|
|
38
|
+
*/
|
|
39
|
+
var PG_SPACES_REGEX_SQL = "'".concat(('<[^>]*>|&[a-z0-9#]+;|[]' +
|
|
40
|
+
SEARCH_PUNCTUATION.split('')
|
|
41
|
+
.filter(function (char) { return !']-\\['.includes(char); })
|
|
42
|
+
.join('') +
|
|
43
|
+
'\\\\[-]').replace(/'/g, "''"), "'");
|
|
44
|
+
/**
|
|
45
|
+
* `searchTerms(query)`: the typed terms, lowercased, each once. `normalised`
|
|
46
|
+
* also reads the punctuation inside a term as a space, for the routes whose
|
|
47
|
+
* stored text is read the same way; otherwise only the punctuation around a
|
|
48
|
+
* term is dropped, so "e-commerce" still finds the raw text "e-commerce".
|
|
49
|
+
*/
|
|
50
|
+
export var generateSearchTermsCode = function (normalised) { return "const SEARCH_PUNCTUATION = ".concat(jsString(SEARCH_PUNCTUATION), "\nconst searchPunctuationToSpaces = (text) => {\n let out = ''\n for (const char of text) out += SEARCH_PUNCTUATION.indexOf(char) === -1 ? char : ' '\n return out\n}\nconst searchTerms = (query) => {\n const terms = []\n String(query == null ? '' : query).toLowerCase().split(/\\s+/).forEach((token) => {\n const spaced = searchPunctuationToSpaces(token)\n").concat(normalised
|
|
51
|
+
? " const term = spaced.trim().replace(/\\s+/g, ' ')"
|
|
52
|
+
: " const start = spaced.length - spaced.trimStart().length\n const term = token.slice(start, start + spaced.trim().length)", "\n if (term && terms.indexOf(term) === -1) terms.push(term)\n })\n return terms\n}"); };
|
|
53
|
+
var PG_LIKE_SQL = "(text, placeholder, before, after) => text + \" LIKE ('\" + before + \"' || \" + placeholder + \"::text || '\" + after + \"') ESCAPE '|'\"";
|
|
54
|
+
export var PG_SEARCH_SQL = {
|
|
55
|
+
textSql: "(column) => \"(' ' || regexp_replace(lower(\\\"\" + column + \"\\\"::text), \" + SEARCH_SPACES_REGEX_SQL + \", ' ', 'g') || ' ')\"",
|
|
56
|
+
likeSql: PG_LIKE_SQL,
|
|
57
|
+
normalised: true,
|
|
58
|
+
lateral: true,
|
|
59
|
+
};
|
|
60
|
+
export var REDSHIFT_SEARCH_SQL = {
|
|
61
|
+
textSql: "(column) => \"(' ' || lower(\\\"\" + column + \"\\\"::text) || ' ')\"",
|
|
62
|
+
likeSql: PG_LIKE_SQL,
|
|
63
|
+
normalised: false,
|
|
64
|
+
lateral: false,
|
|
65
|
+
};
|
|
66
|
+
var CONCAT_LIKE_SQL = "(text, placeholder, before, after) => text + \" LIKE CONCAT('\" + before + \"', \" + placeholder + \", '\" + after + \"') ESCAPE '|'\"";
|
|
67
|
+
export var MYSQL_SEARCH_SQL = {
|
|
68
|
+
textSql: "(column) => \"CONCAT(' ', LOWER(CAST(\" + mysql.escapeId(column) + \" AS CHAR)), ' ')\"",
|
|
69
|
+
likeSql: CONCAT_LIKE_SQL,
|
|
70
|
+
normalised: false,
|
|
71
|
+
lateral: false,
|
|
72
|
+
};
|
|
73
|
+
export var MARIADB_SEARCH_SQL = {
|
|
74
|
+
textSql: "(column) => \"CONCAT(' ', LOWER(CAST(\\`\" + column + \"\\` AS CHAR)), ' ')\"",
|
|
75
|
+
likeSql: CONCAT_LIKE_SQL,
|
|
76
|
+
normalised: false,
|
|
77
|
+
lateral: false,
|
|
78
|
+
};
|
|
79
|
+
export var SQLITE_SEARCH_SQL = {
|
|
80
|
+
textSql: "(column) => \"(' ' || LOWER(CAST(\\\"\" + column + \"\\\" AS TEXT)) || ' ')\"",
|
|
81
|
+
likeSql: "(text, placeholder, before, after) => text + \" LIKE ('\" + before + \"' || \" + placeholder + \" || '\" + after + \"') ESCAPE '|'\"",
|
|
82
|
+
normalised: false,
|
|
83
|
+
lateral: false,
|
|
84
|
+
};
|
|
14
85
|
/**
|
|
15
86
|
* `buildSearchSql(query, columns, firstIndex)` for the SQL routes:
|
|
16
|
-
* `{ where, params, rank, rankParams, nextIndex }`, or null when there
|
|
17
|
-
* nothing to search. `
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
87
|
+
* `{ where, params, from, rank, rankParams, nextIndex }`, or null when there
|
|
88
|
+
* is nothing to search. `from` is the LATERAL join a `lateral` dialect's
|
|
89
|
+
* route appends after its table (selecting `table.*`, so the worked-out
|
|
90
|
+
* texts never reach a row), '' otherwise. `rank` is the ORDER BY keys, one
|
|
91
|
+
* per searched column, best first.
|
|
92
|
+
*
|
|
93
|
+
* With `firstIndex` the placeholders are numbered (`$n`) and each term is
|
|
94
|
+
* bound once, the rank reusing it; without it they are positional (`?`),
|
|
95
|
+
* every occurrence binds its own copy, and `rankParams` go after the
|
|
96
|
+
* parameters of the whole WHERE, as the ORDER BY comes after it.
|
|
23
97
|
*
|
|
24
98
|
* Uses `escapeLikePattern` and `sanitizeSearchIdentifier`
|
|
25
99
|
* (`generateSearchEscapeHelpersCode`).
|
|
26
100
|
*/
|
|
27
|
-
export var generateSearchSqlHelpersCode = function (
|
|
101
|
+
export var generateSearchSqlHelpersCode = function (dialect) { return "".concat(generateSearchTermsCode(dialect.normalised), "\n").concat(dialect.normalised ? "const SEARCH_SPACES_REGEX_SQL = ".concat(jsString(PG_SPACES_REGEX_SQL), "\n") : '', "const searchTextSql = ").concat(dialect.textSql, "\nconst searchLikeSql = ").concat(dialect.likeSql, "\nconst buildSearchSql = (query, columns, firstIndex) => {\n const terms = searchTerms(query)\n const searched = []\n ;(Array.isArray(columns) ? columns : []).forEach((column) => {\n const name = sanitizeSearchIdentifier(column)\n if (searched.indexOf(name) === -1) searched.push(name)\n })\n if (terms.length === 0 || searched.length === 0) return null\n const patterns = terms.map((term) => escapeLikePattern(term))\n const numbered = typeof firstIndex === 'number'\n const params = numbered ? patterns.slice() : []\n const rankParams = []\n const placeholder = (termIndex, bound) => {\n if (numbered) return '$' + (firstIndex + termIndex)\n bound.push(patterns[termIndex])\n return '?'\n }\n const texts = searched.map((column, position) => ").concat(dialect.lateral ? "'\"__tq_search\".\"__tq_search_' + position + '\"'" : 'searchTextSql(column)', ")\n const from = ").concat(dialect.lateral
|
|
102
|
+
? "' CROSS JOIN LATERAL (SELECT ' + searched.map((column, position) => searchTextSql(column) + ' AS \"__tq_search_' + position + '\"').join(', ') + ' OFFSET 0) AS \"__tq_search\"'"
|
|
103
|
+
: "''", "\n const where = '(' + patterns.map((pattern, termIndex) =>\n '(' + texts.map((text) => searchLikeSql(text, placeholder(termIndex, params), '%', '%')).join(' OR ') + ')'\n ).join(' AND ') + ')'\n const rank = texts.map((text) => '(' + patterns.map((pattern, termIndex) =>\n 'CASE WHEN ' + searchLikeSql(text, placeholder(termIndex, rankParams), '% ', ' %') + ' THEN 3' +\n ' WHEN ' + searchLikeSql(text, placeholder(termIndex, rankParams), '% ', '%') + ' THEN 2' +\n ' WHEN ' + searchLikeSql(text, placeholder(termIndex, rankParams), '%', '%') + ' THEN 1 ELSE 0 END'\n ).join(' + ') + ') DESC').join(', ')\n return { where, params, from, rank, rankParams, nextIndex: numbered ? firstIndex + patterns.length : firstIndex }\n}"); };
|
|
28
104
|
/**
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
105
|
+
* For the routes that search rows in memory: `matchesSearch(row, query,
|
|
106
|
+
* columns, valueOf)`, true when every term is in one of the columns, and
|
|
107
|
+
* `rankBySearch(rows, query, columns, valueOf)`, the same order as the SQL
|
|
108
|
+
* routes' rank, applied after the list's sort (a stable sort keeps it among
|
|
109
|
+
* rows that score alike) and before the page is cut. Stored text is read
|
|
110
|
+
* the normalised way: tags, entities and punctuation as spaces.
|
|
33
111
|
*/
|
|
34
|
-
export var generateSearchRankHelperCode = function () { return "".concat(
|
|
35
|
-
/** Each dialect's case-insensitive substring match, for `generateSearchSqlHelpersCode`. */
|
|
36
|
-
export var PG_SEARCH_MATCH_SQL = "(column, placeholder) => '\"' + column + '\"::text ILIKE ' + placeholder + \" ESCAPE '|'\"";
|
|
37
|
-
export var MYSQL_SEARCH_MATCH_SQL = "(column, placeholder) => 'LOWER(CAST(' + mysql.escapeId(column) + ' AS CHAR)) LIKE LOWER(' + placeholder + \") ESCAPE '|'\"";
|
|
38
|
-
export var MARIADB_SEARCH_MATCH_SQL = "(column, placeholder) => 'LOWER(CAST(`' + column + '` AS CHAR)) LIKE LOWER(' + placeholder + \") ESCAPE '|'\"";
|
|
39
|
-
export var SQLITE_SEARCH_MATCH_SQL = "(column, placeholder) => 'LOWER(CAST(\"' + column + '\" AS TEXT)) LIKE LOWER(' + placeholder + \") ESCAPE '|'\"";
|
|
112
|
+
export var generateSearchRankHelperCode = function () { return "".concat(generateSearchTermsCode(true), "\nconst searchText = (value) => {\n const text = value === null || value === undefined ? '' : typeof value === 'object' ? JSON.stringify(value) : String(value)\n return ' ' + searchPunctuationToSpaces(text.toLowerCase().replace(/<[^>]*>|&[a-z0-9#]+;/g, ' ')) + ' '\n}\nconst searchScore = (text, terms) => terms.reduce((score, term) =>\n score + (text.indexOf(' ' + term + ' ') !== -1 ? 3 : text.indexOf(' ' + term) !== -1 ? 2 : text.indexOf(term) !== -1 ? 1 : 0), 0)\nconst matchesSearch = (row, query, columns, valueOf) => {\n const terms = searchTerms(query)\n if (terms.length === 0) return true\n const texts = (Array.isArray(columns) ? columns : []).map((column) => searchText(valueOf(row, column)))\n return terms.every((term) => texts.some((text) => text.indexOf(term) !== -1))\n}\nconst rankBySearch = (rows, query, columns, valueOf) => {\n const terms = searchTerms(query)\n if (!Array.isArray(columns) || columns.length === 0 || terms.length === 0) return rows\n return rows\n .map((row, index) => ({ row, index, scores: columns.map((column) => searchScore(searchText(valueOf(row, column)), terms)) }))\n .sort((a, b) => {\n for (let position = 0; position < a.scores.length; position++) {\n if (a.scores[position] !== b.scores[position]) return b.scores[position] - a.scores[position]\n }\n return a.index - b.index\n })\n .map((entry) => entry.row)\n}"); };
|
|
40
113
|
//# sourceMappingURL=search-relevance.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search-relevance.js","sourceRoot":"","sources":["../../src/search-relevance.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"search-relevance.js","sourceRoot":"","sources":["../../src/search-relevance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;GAGG;AACH,MAAM,CAAC,IAAM,kBAAkB,GAC7B,kFAAkF;IAClF,wGAAwG,CAAA;AAE1G,sFAAsF;AACtF,IAAM,QAAQ,GAAG,UAAC,KAAa;IAC7B,OAAA,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAC3B,oBAAoB,EACpB,UAAC,IAAI,IAAK,OAAA,KAAK,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAA5D,CAA4D,CACvE;AAHD,CAGC,CAAA;AAEH;;;;;;GAMG;AACH,IAAM,mBAAmB,GAAG,WAAI,CAC9B,yBAAyB;IACzB,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;SACzB,MAAM,CAAC,UAAC,IAAI,IAAK,OAAA,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAvB,CAAuB,CAAC;SACzC,IAAI,CAAC,EAAE,CAAC;IACX,SAAS,CACV,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,MAAG,CAAA;AAExB;;;;;GAKG;AACH,MAAM,CAAC,IAAM,uBAAuB,GAAG,UACrC,UAAmB,IACR,OAAA,qCAA8B,QAAQ,CAAC,kBAAkB,CAAC,yXAWrE,UAAU;IACR,CAAC,CAAC,sDAAsD;IACxD,CAAC,CAAC,gIAC4D,4FAKhE,EAnBW,CAmBX,CAAA;AAmBF,IAAM,WAAW,GAAG,4IAAoI,CAAA;AAExJ,MAAM,CAAC,IAAM,aAAa,GAAqB;IAC7C,OAAO,EAAE,oIAA4H;IACrI,OAAO,EAAE,WAAW;IACpB,UAAU,EAAE,IAAI;IAChB,OAAO,EAAE,IAAI;CACd,CAAA;AAED,MAAM,CAAC,IAAM,mBAAmB,GAAqB;IACnD,OAAO,EAAE,uEAAiE;IAC1E,OAAO,EAAE,WAAW;IACpB,UAAU,EAAE,KAAK;IACjB,OAAO,EAAE,KAAK;CACf,CAAA;AAED,IAAM,eAAe,GAAG,wIAAgI,CAAA;AAExJ,MAAM,CAAC,IAAM,gBAAgB,GAAqB;IAChD,OAAO,EAAE,yFAAqF;IAC9F,OAAO,EAAE,eAAe;IACxB,UAAU,EAAE,KAAK;IACjB,OAAO,EAAE,KAAK;CACf,CAAA;AAED,MAAM,CAAC,IAAM,kBAAkB,GAAqB;IAClD,OAAO,EAAE,+EAA6E;IACtF,OAAO,EAAE,eAAe;IACxB,UAAU,EAAE,KAAK;IACjB,OAAO,EAAE,KAAK;CACf,CAAA;AAED,MAAM,CAAC,IAAM,iBAAiB,GAAqB;IACjD,OAAO,EAAE,+EAAyE;IAClF,OAAO,EAAE,sIAA8H;IACvI,UAAU,EAAE,KAAK;IACjB,OAAO,EAAE,KAAK;CACf,CAAA;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,IAAM,4BAA4B,GAAG,UAC1C,OAAyB,IACd,OAAA,UAAG,uBAAuB,CAAC,OAAO,CAAC,UAAU,CAAC,eAEzD,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,0CAAmC,QAAQ,CAAC,mBAAmB,CAAC,OAAI,CAAC,CAAC,CAAC,EAAE,mCACvE,OAAO,CAAC,OAAO,qCAChB,OAAO,CAAC,OAAO,gxBAmBnC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,oDAAgD,CAAC,CAAC,CAAC,uBAAuB,+BAG5F,OAAO,CAAC,OAAO;IACb,CAAC,CAAC,kLAA8K;IAChL,CAAC,CAAC,IAAI,0uBAWV,EAvCW,CAuCX,CAAA;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,IAAM,4BAA4B,GAAG,cAAc,OAAA,UAAG,uBAAuB,CAAC,IAAI,CAAC,q4CAyBxF,EAzBwD,CAyBxD,CAAA"}
|