@njinlabs/njin 0.10.0 → 0.10.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@njinlabs/njin",
-  "version": "0.10.0",
+  "version": "0.10.2",
   "description": "A modern framework for building company profiles, landing pages, and content-driven websites.",
   "type": "module",
   "keywords": ["bun", "elysia", "surrealdb", "edgejs", "cms", "framework"],

package/src/core/model/index.ts

@@ -4,6 +4,10 @@ import { RecordId, Table, type Values } from "surrealdb";
 import { z } from "zod";
 import { runAfterHooks, runBeforeDestroyHooks, runBeforeHooks } from "./hooks";
 
+// See the containment-boost comment in read()'s relevanceSelect below for why this exists and
+// why the value just needs to clear typical BM25 scores (~0-10), not be precisely tuned.
+const CONTAINMENT_BOOST = 100;
+
 export type FormMeta = {
   label: string;
   unique?: boolean;
@@ -262,25 +266,55 @@ export const makeModel = <Rules extends z.ZodObject>(
     // user-defined schema shape (they're injected in create()/update()) — allow sorting by them too.
     const sortableFields = new Set([...Object.keys(config.schema.shape), "id", "createdAt", "updatedAt"]);
     const hasExplicitSort = Boolean(sort && sortableFields.has(sort));
-    // An explicit sort always wins; otherwise, when searching, rank by BM25 relevance
-    // (summed across every matched flat search field) instead of leaving result order
-    // unspecified. Nested (relation) entries are excluded from this sum — a match found via
-    // the IN/CONTAINSANY subquery above has no per-record score in this query's context, since
-    // it happened on a different table entirely. If a model's searchFields are all nested,
-    // there's no score to rank by, so relevance ordering is skipped (same as no searchFields).
-    // `ORDER BY` only accepts a bare identifier here, not a function call — so relevance is
-    // projected as an aliased field below (SELECT ... AS __relevance) and stripped back out
-    // of each returned record afterwards, since it isn't part of the model's schema.
-    const useRelevance = !hasExplicitSort && Boolean(search && searchPlan.some((e) => e.kind === "flat"));
+    // An explicit sort always wins; otherwise, when searching, rank by relevance instead of
+    // leaving result order unspecified. `ORDER BY` only accepts a bare identifier here, not a
+    // function call — so relevance is projected as an aliased field below (SELECT ... AS
+    // __relevance) and stripped back out of each returned record afterwards, since it isn't
+    // part of the model's schema.
+    //
+    // Every field — flat or nested — contributes raw BM25 (search::score(N)) plus a containment
+    // boost. search::score(N) only works against a FULLTEXT index on the table actually being
+    // queried, and a nested match happened on a different table entirely — so its score is
+    // pulled back via a correlated subquery using SurrealDB's $parent (the current outer row),
+    // re-running the same `targetField @N@ $search` match scoped to just that one linked record
+    // (`id = $parent.<local>`, or `id IN $parent.<local>` for a multi relation) and summing the
+    // result with math::sum (0 rows -> 0, exactly what an unmatched/absent relation should
+    // contribute). Nested entries used to be excluded from __relevance altogether, which meant a
+    // row matched *only* through its relation (e.g. brand.name === "Red Wing", normally the most
+    // precise signal available) scored exactly 0 — tied with "didn't match" and ranked below any
+    // row that merely shared a few ngrams with the query on a flat field.
+    //
+    // The containment check itself: the ngram analyzer (see SEARCH_ANALYZER_DEFINITION in
+    // ../../modules/surreal) matches on shared n-grams, which means a document can match without
+    // ever containing the search string as a whole — two unrelated titles can share enough short
+    // n-grams to both "match", and short/generic field values are disproportionately likely to
+    // do so. A field that truly contains the (lowercased) search string is a much stronger
+    // signal than raw BM25 alone, so it's boosted well above the normal BM25 range (empirically
+    // small, well under 10) to consistently outrank n-gram-only matches, without removing those
+    // matches from the result set (fuzzy/typo recall from ngram is unaffected — this only changes
+    // ordering). Both sides also have their spaces stripped before comparing, since real product
+    // data routinely writes a multi-word term as one run-together token (e.g. "REDWING 2415..."
+    // for "Red Wing") — a plain substring check against "red wing" (with the space) would miss
+    // that despite it being a stronger match than most ngram overlaps.
+    const useRelevance = !hasExplicitSort && Boolean(search && searchPlan.length);
     const orderBy = hasExplicitSort
       ? `ORDER BY ${sort} ${order === "desc" ? "DESC" : "ASC"}`
       : useRelevance
         ? "ORDER BY __relevance DESC"
         : "";
+    const containmentCheck = (field: string) =>
+      `string::contains(string::replace(string::lowercase(${field}), " ", ""), string::replace(string::lowercase($search), " ", ""))`;
     const relevanceSelect = useRelevance
       ? `, (${searchPlan
-          .map((e, i) => (e.kind === "flat" ? `search::score(${i + 1})` : null))
-          .filter((s): s is string => s !== null)
+          .map((e, i) => {
+            const n = i + 1;
+            const boost = (field: string) => `(IF ${containmentCheck(field)} THEN ${CONTAINMENT_BOOST} ELSE 0 END)`;
+            if (e.kind === "flat") {
+              return `(search::score(${n}) + ${boost(e.field)})`;
+            }
+            const idFilter = e.multi ? `id IN $parent.${e.local}` : `id = $parent.${e.local}`;
+            return `math::sum((SELECT VALUE (search::score(${n}) + ${boost(e.targetField)}) FROM ${e.targetPrefix} WHERE ${idFilter} AND ${e.targetField} @${n}@ $search))`;
+          })
           .join(" + ")}) AS __relevance`
       : "";
 

package/src/modules/surreal.ts

@@ -15,10 +15,20 @@ export const isRemotePath = (path: string) => REMOTE_SCHEMES.some((scheme) => pa
 // tokenizer splits on whitespace only (unlike `class`, which also splits on punctuation:
 // "Next.js" would become "next" / "." / "js", and a lone "." can't form any 2-char ngram,
 // so a query for "next.js" — tokenized the same way — would never match). The `ngram`
-// filter then indexes overlapping 2-10 char slices of each whitespace-delimited token so
+// filter then indexes overlapping 3-10 char slices of each whitespace-delimited token so
 // the `@N@` match operator can find a term anywhere inside a field (not just a whole-field
-// match) and still tolerate minor typos, similar to trigram search.
+// match) and still tolerate minor typos, similar to trigram search. Minimum is 3, not 2 —
+// 2-char slices ("si", "ze", digit pairs, ...) are common enough across unrelated documents
+// that they matched all over the place on real-sized catalogs (a 2-char generic word turns
+// into near-universal noise); 3+ cuts that collision rate drastically. core/model/index.ts's
+// relevanceSelect adds a containment-based score boost on top of this, so a document that
+// truly contains the query substring still outranks one that only shares a few ngrams with it.
 const SEARCH_ANALYZER = "njin_search";
+// Single source of truth for the analyzer body — reused both in the DEFINE below and in
+// schemaHash, so any future change here (e.g. a different ngram range) automatically
+// invalidates the stored hash and re-runs the DEFINE on next boot instead of silently
+// leaving an already-migrated DB on the old analyzer definition.
+const SEARCH_ANALYZER_DEFINITION = "TOKENIZERS blank FILTERS lowercase,ngram(3,10)";
 
 // Records the hash of the last schema this DB was migrated to, so a worker booting against
 // an already-migrated DB (idle-evict/crash respawn — every DEFINE below is idempotent but
@@ -70,7 +80,13 @@ const ensureTables = async (db: Surreal) => {
   }
 
   const schemaHash = createHash("sha256")
-    .update(JSON.stringify({ prefixes: [...prefixes].sort(), searchIndexes: [...searchIndexes.keys()].sort() }))
+    .update(
+      JSON.stringify({
+        prefixes: [...prefixes].sort(),
+        searchIndexes: [...searchIndexes.keys()].sort(),
+        analyzer: SEARCH_ANALYZER_DEFINITION,
+      }),
+    )
     .digest("hex");
 
   // Must be DEFINE'd before the SELECT below can even run — unlike a table that exists but
@@ -87,13 +103,16 @@ const ensureTables = async (db: Surreal) => {
     await db.query(`DEFINE TABLE IF NOT EXISTS ${prefix} SCHEMALESS;`);
   }
 
-  await db.query(`DEFINE ANALYZER IF NOT EXISTS ${SEARCH_ANALYZER} TOKENIZERS blank FILTERS lowercase,ngram(2,10);`);
+  // OVERWRITE, not IF NOT EXISTS — this block only runs when schemaHash just changed (see the
+  // early return above), so an analyzer/index that already exists under this name here means its
+  // definition is stale and needs replacing, not skipping. FULLTEXT, not SEARCH — this SurrealDB
+  // version renamed the index-type keyword; SEARCH ANALYZER ... is a parse error here even though
+  // older docs/examples use it.
+  await db.query(`DEFINE ANALYZER OVERWRITE ${SEARCH_ANALYZER} ${SEARCH_ANALYZER_DEFINITION};`);
 
   for (const { prefix: targetPrefix, field } of searchIndexes.values()) {
-    // FULLTEXT, not SEARCH — this SurrealDB version renamed the index-type keyword;
-    // SEARCH ANALYZER ... is a parse error here even though older docs/examples use it.
     await db.query(
-      `DEFINE INDEX IF NOT EXISTS idx_search_${targetPrefix}_${field} ON TABLE ${targetPrefix} FIELDS ${field} FULLTEXT ANALYZER ${SEARCH_ANALYZER} BM25 HIGHLIGHTS;`,
+      `DEFINE INDEX OVERWRITE idx_search_${targetPrefix}_${field} ON TABLE ${targetPrefix} FIELDS ${field} FULLTEXT ANALYZER ${SEARCH_ANALYZER} BM25 HIGHLIGHTS;`,
     );
   }