@vibeorm/extensions 2.2.2 → 2.3.1

package/dist/search/index.js

@@ -85,10 +85,198 @@ async function pgExtensionCheck(params) {
 
 // packages/extensions/src/search/index.ts
 import { ParamCollector, getSqlDialect, renderExpr } from "@vibeorm/sql";
-var VERSION = "2.0.0-alpha.1";
+var VERSION = "2.0.0-alpha.2";
 var SCORE_ALIAS = "_vibe_score";
 var HIGHLIGHT_ALIAS = "_vibe_highlight";
 var WEIGHT_LETTERS = ["A", "B", "C", "D"];
+var BM25_TOKENIZERS = Object.freeze([
+  "unicode_words",
+  "simple",
+  "whitespace",
+  "ngram",
+  "edge_ngram",
+  "icu",
+  "literal_normalized",
+  "source_code",
+  "chinese_compatible",
+  "jieba"
+]);
+var STEMMER_LANGUAGES = Object.freeze([
+  "arabic",
+  "czech",
+  "danish",
+  "dutch",
+  "english",
+  "finnish",
+  "french",
+  "german",
+  "greek",
+  "hungarian",
+  "italian",
+  "norwegian",
+  "polish",
+  "portuguese",
+  "romanian",
+  "russian",
+  "spanish",
+  "swedish",
+  "tamil",
+  "turkish"
+]);
+var STOPWORD_LANGUAGES = Object.freeze([
+  "czech",
+  "danish",
+  "dutch",
+  "english",
+  "finnish",
+  "french",
+  "german",
+  "hungarian",
+  "italian",
+  "norwegian",
+  "polish",
+  "portuguese",
+  "russian",
+  "spanish",
+  "swedish"
+]);
+var TOKENIZER_KEYS = ["tokenizer", "minGram", "maxGram", "stemmer", "asciiFolding", "lowercase", "stopwordsLanguage"];
+function isPositiveInteger(value) {
+  return typeof value === "number" && Number.isSafeInteger(value) && value >= 1;
+}
+function readBm25Tokenizer(declaration) {
+  if (TOKENIZER_KEYS.every((key) => declaration[key] === undefined))
+    return { tokenizer: undefined };
+  const name = declaration["tokenizer"] ?? "unicode_words";
+  if (typeof name !== "string" || !BM25_TOKENIZERS.includes(name)) {
+    return { error: `tokenizer must be one of ${BM25_TOKENIZERS.join(", ")}` };
+  }
+  const grams = name === "ngram" || name === "edge_ngram";
+  const minGram = declaration["minGram"];
+  const maxGram = declaration["maxGram"];
+  if (grams) {
+    if (!isPositiveInteger(minGram) || !isPositiveInteger(maxGram) || minGram > maxGram) {
+      return { error: `${name} needs integer minGram and maxGram with 1 <= minGram <= maxGram` };
+    }
+  } else if (minGram !== undefined || maxGram !== undefined) {
+    return { error: "minGram/maxGram apply only to the ngram and edge_ngram tokenizers" };
+  }
+  const filters = [];
+  const lowercase = declaration["lowercase"];
+  if (lowercase !== undefined && typeof lowercase !== "boolean")
+    return { error: "lowercase must be a boolean" };
+  if (lowercase === false)
+    filters.push("lowercase=false");
+  const folding = declaration["asciiFolding"];
+  if (folding !== undefined && typeof folding !== "boolean")
+    return { error: "asciiFolding must be a boolean" };
+  if (folding === true)
+    filters.push("ascii_folding=true");
+  const stemmer = declaration["stemmer"];
+  if (stemmer !== undefined) {
+    if (typeof stemmer !== "string" || !STEMMER_LANGUAGES.includes(stemmer.toLowerCase())) {
+      return { error: `stemmer must be one of ${STEMMER_LANGUAGES.join(", ")}` };
+    }
+    filters.push(`stemmer=${stemmer.toLowerCase()}`);
+  }
+  const stopwords = declaration["stopwordsLanguage"];
+  if (stopwords !== undefined) {
+    if (typeof stopwords !== "string" || !STOPWORD_LANGUAGES.includes(stopwords.toLowerCase())) {
+      return { error: `stopwordsLanguage must be one of ${STOPWORD_LANGUAGES.join(", ")}` };
+    }
+    filters.push(`stopwords_language=${stopwords.toLowerCase()}`);
+  }
+  return {
+    tokenizer: {
+      name,
+      ...grams ? { minGram, maxGram } : {},
+      filters
+    }
+  };
+}
+function bm25TokenizerArguments(params) {
+  const { dialect, tokenizer } = params;
+  const args = [
+    ...tokenizer.minGram === undefined ? [] : [String(tokenizer.minGram), String(tokenizer.maxGram)],
+    ...tokenizer.filters.map((filter) => dialect.quoteStringLiteral({ value: filter }))
+  ];
+  return args.length === 0 ? "" : `(${args.join(", ")})`;
+}
+function bm25TokenizedColumn(params) {
+  const { dialect, column, tokenizer } = params;
+  const quoted = dialect.quoteIdent({ name: column });
+  if (tokenizer === undefined)
+    return quoted;
+  return `(${quoted}::pdb.${tokenizer.name}${bm25TokenizerArguments({ dialect, tokenizer })})`;
+}
+function bm25SearchTokenizer(params) {
+  const { tokenizer } = params;
+  if (tokenizer === undefined || tokenizer.name !== "edge_ngram")
+    return;
+  return tokenizer.filters.length === 0 ? "unicode_words" : `unicode_words(${tokenizer.filters.join(",")})`;
+}
+var BM25_FILTER_SCALARS = Object.freeze(["Boolean", "Int", "BigInt", "Float", "Decimal", "DateTime"]);
+var BM25_BOOST_LIMIT = 2048;
+function bm25StringStorage(field) {
+  if (field.isUnsupported === true || field.isList || field.type !== "String")
+    return null;
+  const native = (field.nativeType ?? "").toLowerCase();
+  if (native === "" || native === "text" || native.startsWith("varchar"))
+    return "text";
+  if (native === "uuid")
+    return "uuid";
+  return null;
+}
+function describeField(field) {
+  const scalar = typeof field.type === "string" ? field.type : `enum ${field.type.enum}`;
+  return `${scalar}${field.isList ? "[]" : ""}${field.nativeType === undefined ? "" : ` @db.${field.nativeType}`}`;
+}
+function bm25IndexColumns(params) {
+  const { model, decl } = params;
+  const pkField = primaryKeyFieldOf({ model });
+  if (pkField === null) {
+    throw new VibeError2({
+      code: "VIBE_SCHEMA_INVALID",
+      message: `@@search with engine "bm25" needs a single-column primary key on "${model.name}" (key_field)`,
+      meta: { model: model.name }
+    });
+  }
+  const searched = decl.fields.map((name) => {
+    const field = model.fields.find((candidate) => candidate.name === name);
+    if (field === undefined)
+      return columnNameOf({ model, field: name });
+    if (bm25StringStorage(field) !== "text") {
+      throw unsupportedCapability({
+        dialect: "postgres",
+        feature: `@@search field "${name}" (${describeField(field)}) on the bm25 index`,
+        hint: "ParadeDB tokenizes text and varchar columns; search another column or store this one as text"
+      });
+    }
+    return field.dbName ?? field.name;
+  });
+  const filters = (decl.filterFields ?? []).map((name) => {
+    const field = model.fields.find((candidate) => candidate.name === name);
+    if (field === undefined || name === pkField || decl.fields.includes(name)) {
+      throw new VibeError2({
+        code: "VIBE_SCHEMA_INVALID",
+        message: `@@search filterFields on "${model.name}": "${name}" must be a scalar field that is neither the primary key nor a searched field`,
+        meta: { model: model.name, field: name }
+      });
+    }
+    const column = field.dbName ?? field.name;
+    const storage = bm25StringStorage(field);
+    if (storage !== null)
+      return { column, literal: storage === "text" };
+    if (!field.isList && typeof field.type === "string" && BM25_FILTER_SCALARS.includes(field.type))
+      return { column, literal: false };
+    throw unsupportedCapability({
+      dialect: "postgres",
+      feature: `@@search filterFields entry "${name}" (${describeField(field)}) on the bm25 index`,
+      hint: "ParadeDB pushes filters into the index for text/varchar (exact match / IN), uuid, Boolean, Int, BigInt, Float, Decimal and DateTime columns \u2014 filter other columns outside the index"
+    });
+  });
+  return { pkColumn: columnNameOf({ model, field: pkField }), searched, filters };
+}
 function resolveSearchEngine(params) {
   const { dialect, provider, requested } = params;
   const native = dialect === "postgres" ? "tsvector" : dialect === "sqlite" ? "fts5" : "fulltext";
@@ -123,6 +311,10 @@ function readDeclaration(params) {
   if (params.declaration["weights"] !== undefined && (!Array.isArray(params.declaration["weights"]) || params.declaration["weights"].length !== fields.length || params.declaration["weights"].some((weight) => typeof weight !== "number" || !Number.isFinite(weight)))) {
     throw new VibeError2({ code: "VIBE_SCHEMA_INVALID", message: "weights must be a finite number list matching the field list", meta: { model: params.model.name } });
   }
+  const tokenizer = readBm25Tokenizer(params.declaration);
+  if ("error" in tokenizer) {
+    throw new VibeError2({ code: "VIBE_SCHEMA_INVALID", message: `@@search on "${params.model.name}": ${tokenizer.error}`, meta: { model: params.model.name } });
+  }
   return {
     fields,
     weights: declNumberArray(params.declaration, "weights"),
@@ -130,7 +322,9 @@ function readDeclaration(params) {
     language: declString(params.declaration, "language"),
     columnName: declString(params.declaration, "columnName"),
     indexName: declString(params.declaration, "indexName"),
-    rank: declRank(params.declaration)
+    rank: declRank(params.declaration),
+    tokenizer: tokenizer.tokenizer,
+    filterFields: declStringArray(params.declaration, "filterFields")
   };
 }
 function weightLetters(params) {
@@ -168,11 +362,18 @@ function resolveArtifactNames(params) {
 }
 function resolveRank(params) {
   const { decl, options, engine, dialect } = params;
-  if (decl.weights !== undefined && engine !== "tsvector") {
+  if (decl.weights !== undefined && engine !== "tsvector" && engine !== "bm25") {
     throw unsupportedCapability({
       dialect,
       feature: `@@search weights on the ${engine} engine`,
-      hint: "weights are supported only by tsvector; omit weights for this engine"
+      hint: "weights are supported by tsvector (setweight classes) and bm25 (pdb.boost factors); omit weights for this engine"
+    });
+  }
+  if (decl.weights !== undefined && engine === "bm25" && decl.weights.some((weight) => Math.abs(weight) > BM25_BOOST_LIMIT)) {
+    throw new VibeError2({
+      code: "VIBE_SCHEMA_INVALID",
+      message: `@@search weights on the bm25 engine are pdb.boost factors and must stay within \xB1${BM25_BOOST_LIMIT}`,
+      meta: { weights: decl.weights }
     });
   }
   if (decl.rank !== undefined && engine !== "tsvector") {
@@ -187,6 +388,25 @@ function resolveRank(params) {
   const fallback = options["rank"];
   return decl.rank ?? (fallback === "ts_rank" || fallback === "ts_rank_cd" ? fallback : undefined);
 }
+function assertBm25Declarations(params) {
+  const { decl, engine, dialect } = params;
+  if (engine === "bm25")
+    return;
+  if (decl.tokenizer !== undefined) {
+    throw unsupportedCapability({
+      dialect,
+      feature: `@@search tokenizer options on the ${engine} engine`,
+      hint: 'tokenizer, minGram, maxGram, stemmer, asciiFolding, lowercase and stopwordsLanguage configure a ParadeDB bm25 index \u2014 drop them, or opt in with search({ engine: "bm25" })'
+    });
+  }
+  if (decl.filterFields !== undefined) {
+    throw unsupportedCapability({
+      dialect,
+      feature: `@@search filterFields on the ${engine} engine`,
+      hint: 'filterFields add columns to a ParadeDB bm25 index for filter pushdown \u2014 drop them, or opt in with search({ engine: "bm25" })'
+    });
+  }
+}
 function tsvectorExpression(params) {
   const { dialect, model, decl, language } = params;
   const quote = (name) => dialect.quoteIdent({ name });
@@ -207,7 +427,7 @@ function searchArtifacts(ctx) {
     artifacts.push({
       id: "search:extension:pg_search",
       owns: { kind: "postgresExtension", name: "pg_search" },
-      create: ["CREATE EXTENSION IF NOT EXISTS pg_search;"],
+      create: ["CREATE EXTENSION IF NOT EXISTS pg_search CASCADE;"],
       drop: ["DROP EXTENSION IF EXISTS pg_search;"]
     });
   }
@@ -221,6 +441,7 @@ function searchArtifacts(ctx) {
     const table = tableNameOf({ model });
     const columns = decl.fields.map((field) => columnNameOf({ model, field }));
     resolveRank({ decl, options: ctx.options, engine, dialect: ctx.dialect });
+    assertBm25Declarations({ decl, engine, dialect: ctx.dialect });
     if (engine === "tsvector") {
       const language = decl.language ?? ctx.options["language"] ?? "english";
       const { tsvColumn, indexName } = resolveArtifactNames({ decl, engine, dialect: ctx.dialect });
@@ -243,21 +464,22 @@ function searchArtifacts(ctx) {
       continue;
     }
     if (engine === "bm25") {
-      const pkField = primaryKeyFieldOf({ model });
-      if (pkField === null) {
-        throw new VibeError2({
-          code: "VIBE_SCHEMA_INVALID",
-          message: `@@search with engine "bm25" needs a single-column primary key on "${modelName}" (key_field)`,
-          meta: { model: modelName }
-        });
-      }
-      const pkColumn = columnNameOf({ model, field: pkField });
+      const { pkColumn, searched, filters } = bm25IndexColumns({ model, decl });
       const bm25Names = resolveArtifactNames({ decl, engine, dialect: ctx.dialect });
+      const indexed = [
+        quote(pkColumn),
+        ...searched.map((column) => bm25TokenizedColumn({ dialect, column, tokenizer: decl.tokenizer })),
+        ...filters.map((filter) => filter.literal ? `(${quote(filter.column)}::pdb.literal)` : quote(filter.column))
+      ];
+      const withOptions = [`key_field=${dialect.quoteStringLiteral({ value: pkColumn })}`];
+      const searchTokenizer = bm25SearchTokenizer({ tokenizer: decl.tokenizer });
+      if (searchTokenizer !== undefined)
+        withOptions.push(`search_tokenizer=${dialect.quoteStringLiteral({ value: searchTokenizer })}`);
       artifacts.push({
         id: `search:${modelName}:index`,
         owns: { kind: "index", table, name: bm25Names.indexName },
         create: [
-          `CREATE INDEX IF NOT EXISTS ${quote(bm25Names.indexName)} ON ${quote(table)} USING bm25 (${[pkColumn, ...columns].map((column) => quote(column)).join(", ")}) WITH (key_field=${dialect.quoteStringLiteral({ value: pkColumn })});`
+          `CREATE INDEX IF NOT EXISTS ${quote(bm25Names.indexName)} ON ${quote(table)} USING bm25 (${indexed.join(", ")}) WITH (${withOptions.join(", ")});`
         ],
         drop: [`DROP INDEX IF EXISTS ${quote(bm25Names.indexName)};`]
       });
@@ -372,11 +594,26 @@ function createSearchDelegate(config) {
       if (args.minScore !== undefined)
         finiteNumber({ value: args.minScore, name: "minScore" });
       const prefix = args.prefix === true;
-      if (prefix && config.engine === "bm25") {
+      const fuzzy = args.fuzzy;
+      if (fuzzy !== undefined && fuzzy !== 1 && fuzzy !== 2) {
+        throw new VibeError2({
+          code: "VIBE_VALIDATION",
+          message: "fuzzy must be 1 or 2 (ParadeDB caps the edit distance at 2)",
+          meta: { argument: "fuzzy" }
+        });
+      }
+      if (fuzzy !== undefined && config.engine !== "bm25") {
+        throw unsupportedCapability({
+          dialect: dialect.name,
+          feature: `search fuzzy matching on the "${config.engine}" engine`,
+          hint: "only the bm25 engine (ParadeDB pdb.fuzzy) matches within an edit distance \u2014 on postgres/pglite declare @@trgm and filter with similarTo"
+        });
+      }
+      if (prefix && config.engine === "bm25" && config.prefixIndex !== true && fuzzy === undefined) {
         throw unsupportedCapability({
           dialect: "postgres",
-          feature: 'search prefix mode on the "bm25" engine',
-          hint: "ParadeDB prefix behavior is decided by the index tokenizer (ngram/edge-ngram) \u2014 configure the index instead, or use the tsvector engine"
+          feature: 'search prefix mode on the "bm25" engine without an edge_ngram index',
+          hint: 'declare @@search(\u2026, tokenizer: "edge_ngram", minGram: 1, maxGram: 12) so every token is indexed as a prefix, or pass fuzzy: 1 | 2 for fuzzy prefix matching (pdb.fuzzy(n, t))'
         });
       }
       let matchText = args.query;
@@ -384,7 +621,7 @@ function createSearchDelegate(config) {
         const tokens = args.query.match(/[\p{L}\p{N}]+/gu) ?? [];
         if (tokens.length === 0)
           return [];
-        matchText = config.engine === "tsvector" ? tokens.map((token) => `${token}:*`).join(" & ") : config.engine === "fts5" ? tokens.map((token) => `"${token}"*`).join(" ") : tokens.map((token) => `+${token}*`).join(" ");
+        matchText = config.engine === "tsvector" ? tokens.map((token) => `${token}:*`).join(" & ") : config.engine === "fts5" ? tokens.map((token) => `"${token}"*`).join(" ") : config.engine === "bm25" ? args.query : tokens.map((token) => `+${token}*`).join(" ");
       }
       const hlColumn = highlightColumn({ config, args });
       const idOnly = args.select === "id";
@@ -455,31 +692,35 @@ function createSearchDelegate(config) {
         text += ` ORDER BY ${quote(SCORE_ALIAS)} DESC`;
       } else {
         const pk = `${t}.${quote(config.pkColumn ?? "")}`;
+        const operator = prefix || fuzzy !== undefined ? "&&&" : "@@@";
+        const fuzzyCast = fuzzy === undefined ? "" : `::pdb.fuzzy(${fuzzy}${prefix ? ", t" : ""})`;
         text = `SELECT ${projection}, paradedb.score(${pk}) AS ${quote(SCORE_ALIAS)}`;
         if (hlColumn !== null) {
           text += `, paradedb.snippet(${t}.${quote(hlColumn)}) AS ${quote(HIGHLIGHT_ALIAS)}`;
         }
-        text += ` FROM ${quote(config.table)} AS ${t} WHERE (${config.fields.map((field) => `${t}.${quote(field.column)} @@@ ${collector.add(args.query)}`).join(" OR ")})`;
+        text += ` FROM ${quote(config.table)} AS ${t} WHERE (${config.fields.map((field, index) => {
+          const boost = config.boosts?.[index];
+          const boostCast = boost === undefined || boost === 1 ? "" : `::pdb.boost(${boost})`;
+          return `${t}.${quote(field.column)} ${operator} ${collector.add(matchText)}${fuzzyCast}${boostCast}`;
+        }).join(" OR ")})`;
         const where = userWhere("t");
         if (where !== null)
           text += ` AND (${where})`;
+        if (args.minScore !== undefined) {
+          text += ` AND paradedb.score(${pk}) >= ${collector.add(args.minScore)}`;
+        }
         text += ` ORDER BY ${quote(SCORE_ALIAS)} DESC`;
       }
       text += ` LIMIT ${take}`;
       if (skip !== null)
         text += ` OFFSET ${skip}`;
       const rows = await ctx.executeRows({ text, values: collector.values, method: "search" });
-      let hits = shapeHits({
+      return shapeHits({
         rows,
         strip,
         highlightsRequested: args.highlight !== undefined,
         engineHasHighlights
       });
-      if (config.engine === "bm25" && args.minScore !== undefined) {
-        const min = args.minScore;
-        hits = hits.filter((hit) => hit.score >= min);
-      }
-      return hits;
     };
   };
 }
@@ -526,6 +767,19 @@ function search(options = {}) {
             if (rank !== undefined && rank !== "ts_rank" && rank !== "ts_rank_cd") {
               return { error: `rank must be "ts_rank" or "ts_rank_cd"` };
             }
+            const tokenizer = readBm25Tokenizer(args.named);
+            if ("error" in tokenizer)
+              return { error: tokenizer.error };
+            const filterFields = args.named["filterFields"];
+            if (filterFields !== undefined) {
+              if (!Array.isArray(filterFields) || filterFields.length === 0 || filterFields.some((name) => typeof name !== "string")) {
+                return { error: "filterFields must be a non-empty field list, e.g. filterFields: [municipalityNumber, active]" };
+              }
+              const invalid = filterFields.find((name) => !fieldNames.includes(String(name)) || fields.includes(String(name)));
+              if (invalid !== undefined) {
+                return { error: `filterFields entry "${String(invalid)}" must be a field on the model that is not also searched` };
+              }
+            }
             return { declaration: { fields, ...args.named } };
           }
         }
@@ -550,6 +804,9 @@ function search(options = {}) {
         }
         const pkField = primaryKeyFieldOf({ model });
         const rank = resolveRank({ decl, options: extOptions, engine, dialect });
+        assertBm25Declarations({ decl, engine, dialect });
+        if (engine === "bm25")
+          bm25IndexColumns({ model, decl });
         const names = resolveArtifactNames({ decl, engine, dialect });
         const config = {
           engine,
@@ -560,7 +817,9 @@ function search(options = {}) {
           })),
           ...engine === "tsvector" ? { tsvColumn: names.tsvColumn, language, ...rank === undefined ? {} : { rank } } : {},
           ...engine === "fts5" ? { ftsTable: `${decl.base}_fts` } : {},
-          ...pkField !== null ? { pkColumn: columnNameOf({ model, field: pkField }) } : {}
+          ...pkField !== null ? { pkColumn: columnNameOf({ model, field: pkField }) } : {},
+          ...engine === "bm25" && decl.weights !== undefined ? { boosts: decl.weights } : {},
+          ...engine === "bm25" && decl.tokenizer?.name === "edge_ngram" ? { prefixIndex: true } : {}
         };
         const fieldUnion = decl.fields.map((field) => JSON.stringify(field)).join(" | ");
         const argLines = [
@@ -570,7 +829,8 @@ function search(options = {}) {
           `  readonly skip?: number;`,
           `  readonly minScore?: number;`,
           `  readonly highlight?: { readonly field: ${fieldUnion} };`,
-          `  readonly prefix?: boolean;`
+          `  readonly prefix?: boolean;`,
+          ...engine === "bm25" ? [`  readonly fuzzy?: 1 | 2;`] : []
         ];
         const dts = pkField === null ? [
           `readonly search: (args: {`,
@@ -638,4 +898,4 @@ export {
   search
 };
 
-//# debugId=02F6D4E257D8B7B464756E2164756E21
+//# debugId=B339B57B135FF5BD64756E2164756E21