@njinlabs/njin 0.9.0 → 0.10.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@njinlabs/njin",
-  "version": "0.9.0",
+  "version": "0.10.1",
   "description": "A modern framework for building company profiles, landing pages, and content-driven websites.",
   "type": "module",
   "keywords": ["bun", "elysia", "surrealdb", "edgejs", "cms", "framework"],

package/src/core/model/index.ts

@@ -4,6 +4,10 @@ import { RecordId, Table, type Values } from "surrealdb";
 import { z } from "zod";
 import { runAfterHooks, runBeforeDestroyHooks, runBeforeHooks } from "./hooks";
 
+// See the containment-boost comment in read()'s relevanceSelect below for why this exists and
+// why the value just needs to clear typical BM25 scores (~0-10), not be precisely tuned.
+const CONTAINMENT_BOOST = 100;
+
 export type FormMeta = {
   label: string;
   unique?: boolean;
@@ -271,6 +275,15 @@ export const makeModel = <Rules extends z.ZodObject>(
     // `ORDER BY` only accepts a bare identifier here, not a function call — so relevance is
     // projected as an aliased field below (SELECT ... AS __relevance) and stripped back out
     // of each returned record afterwards, since it isn't part of the model's schema.
+    //
+    // Each flat field's score also gets a containment boost: the ngram analyzer (see
+    // SEARCH_ANALYZER_DEFINITION in ../../modules/surreal) matches on shared n-grams, which
+    // means a document can match without ever containing the search string as a whole — two
+    // unrelated titles can share enough short n-grams to both "match". A document whose field
+    // literally contains the (lowercased) search string is a much stronger relevance signal than
+    // raw BM25 alone, so it's boosted well above the normal BM25 range (empirically small, well
+    // under 10) to consistently outrank n-gram-only matches, while leaving those matches in the
+    // result set (fuzzy/typo recall from ngram is unaffected — this only changes ordering).
     const useRelevance = !hasExplicitSort && Boolean(search && searchPlan.some((e) => e.kind === "flat"));
     const orderBy = hasExplicitSort
       ? `ORDER BY ${sort} ${order === "desc" ? "DESC" : "ASC"}`
@@ -279,7 +292,11 @@ export const makeModel = <Rules extends z.ZodObject>(
         : "";
     const relevanceSelect = useRelevance
       ? `, (${searchPlan
-          .map((e, i) => (e.kind === "flat" ? `search::score(${i + 1})` : null))
+          .map((e, i) =>
+            e.kind === "flat"
+              ? `(search::score(${i + 1}) + (IF string::contains(string::lowercase(${e.field}), string::lowercase($search)) THEN ${CONTAINMENT_BOOST} ELSE 0 END))`
+              : null,
+          )
           .filter((s): s is string => s !== null)
           .join(" + ")}) AS __relevance`
       : "";

package/src/modules/surreal.ts

@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
 import { getConfig } from "../core/config";
 import { resolveSearchPlan } from "../core/model";
 import { makeModule } from "../core/module";
@@ -14,10 +15,27 @@ export const isRemotePath = (path: string) => REMOTE_SCHEMES.some((scheme) => pa
 // tokenizer splits on whitespace only (unlike `class`, which also splits on punctuation:
 // "Next.js" would become "next" / "." / "js", and a lone "." can't form any 2-char ngram,
 // so a query for "next.js" — tokenized the same way — would never match). The `ngram`
-// filter then indexes overlapping 2-10 char slices of each whitespace-delimited token so
+// filter then indexes overlapping 3-10 char slices of each whitespace-delimited token so
 // the `@N@` match operator can find a term anywhere inside a field (not just a whole-field
-// match) and still tolerate minor typos, similar to trigram search.
+// match) and still tolerate minor typos, similar to trigram search. Minimum is 3, not 2 —
+// 2-char slices ("si", "ze", digit pairs, ...) are common enough across unrelated documents
+// that they matched all over the place on real-sized catalogs (a 2-char generic word turns
+// into near-universal noise); 3+ cuts that collision rate drastically. core/model/index.ts's
+// relevanceSelect adds a containment-based score boost on top of this, so a document that
+// truly contains the query substring still outranks one that only shares a few ngrams with it.
 const SEARCH_ANALYZER = "njin_search";
+// Single source of truth for the analyzer body — reused both in the DEFINE below and in
+// schemaHash, so any future change here (e.g. a different ngram range) automatically
+// invalidates the stored hash and re-runs the DEFINE on next boot instead of silently
+// leaving an already-migrated DB on the old analyzer definition.
+const SEARCH_ANALYZER_DEFINITION = "TOKENIZERS blank FILTERS lowercase,ngram(3,10)";
+
+// Records the hash of the last schema this DB was migrated to, so a worker booting against
+// an already-migrated DB (idle-evict/crash respawn — every DEFINE below is idempotent but
+// still a full network round trip per statement) can skip straight past ensureTables(). Keyed
+// off the DB itself rather than the process/build: a worker pointed at a fresh or different DB
+// (e.g. a per-client env change) finds no matching record here and still runs the DEFINEs.
+const SCHEMA_META_TABLE = "njin_schema_meta";
 
 // SurrealDB's `GROUP ALL` aggregate (used for count queries) throws NotFoundError
 // on a table that has never had a row created, unlike plain SELECT. Defining every
@@ -36,25 +54,11 @@ const ensureTables = async (db: Surreal) => {
   const prefixes = new Set<string>(models.map((model) => model.prefix));
   prefixes.add("vars");
 
-  for (const prefix of prefixes) {
-    await db.query(`DEFINE TABLE IF NOT EXISTS ${prefix} SCHEMALESS;`);
-  }
-
-  await db.query(`DEFINE ANALYZER IF NOT EXISTS ${SEARCH_ANALYZER} TOKENIZERS blank FILTERS lowercase,ngram(2,10);`);
-
-  const defineSearchIndex = async (targetPrefix: string, field: string) => {
-    const key = `${targetPrefix}.${field}`;
-    if (definedIndexes.has(key)) return;
-    definedIndexes.add(key);
-
-    // FULLTEXT, not SEARCH — this SurrealDB version renamed the index-type keyword;
-    // SEARCH ANALYZER ... is a parse error here even though older docs/examples use it.
-    await db.query(
-      `DEFINE INDEX IF NOT EXISTS idx_search_${targetPrefix}_${field} ON TABLE ${targetPrefix} FIELDS ${field} FULLTEXT ANALYZER ${SEARCH_ANALYZER} BM25 HIGHLIGHTS;`,
-    );
-  };
-
-  const definedIndexes = new Set<string>(); // dedupe prefix+field in case two factories share a prefix, or a nested reference targets an already-indexed field
+  // Resolve every search index's target table+field up front (also dedupes prefix+field in
+  // case two factories share a prefix, or a nested reference targets an already-indexed
+  // field) — needed both to compute the schema hash below and to drive the DEFINE INDEX loop
+  // further down.
+  const searchIndexes = new Map<string, { prefix: string; field: string }>();
   for (const model of models) {
     // A nested searchFields entry (e.g. "author.name") needs its index defined on the
     // *target* table/field instead — the local relation field holds a record link, not
@@ -67,13 +71,52 @@ const ensureTables = async (db: Surreal) => {
       : (model.searchFields ?? []).map((field) => ({ kind: "flat" as const, field }));
 
     for (const entry of plan) {
-      if (entry.kind === "flat") {
-        await defineSearchIndex(model.prefix, entry.field);
-      } else {
-        await defineSearchIndex(entry.targetPrefix, entry.targetField);
-      }
+      const target =
+        entry.kind === "flat"
+          ? { prefix: model.prefix, field: entry.field }
+          : { prefix: entry.targetPrefix, field: entry.targetField };
+      searchIndexes.set(`${target.prefix}.${target.field}`, target);
     }
   }
+
+  const schemaHash = createHash("sha256")
+    .update(
+      JSON.stringify({
+        prefixes: [...prefixes].sort(),
+        searchIndexes: [...searchIndexes.keys()].sort(),
+        analyzer: SEARCH_ANALYZER_DEFINITION,
+      }),
+    )
+    .digest("hex");
+
+  // Must be DEFINE'd before the SELECT below can even run — unlike a table that exists but
+  // has no rows (see the GROUP ALL note above), a table SurrealDB has never seen DEFINE'd at
+  // all makes ANY query against a specific record id in it throw NotFoundError, plain SELECT
+  // included. One extra round trip on every boot (skip path too), still far cheaper than the
+  // N DEFINEs it's gating.
+  await db.query(`DEFINE TABLE IF NOT EXISTS ${SCHEMA_META_TABLE} SCHEMALESS;`);
+
+  const [rows] = await db.query<[{ hash: string }[]]>(`SELECT hash FROM ${SCHEMA_META_TABLE}:current;`);
+  if (rows?.[0]?.hash === schemaHash) return;
+
+  for (const prefix of prefixes) {
+    await db.query(`DEFINE TABLE IF NOT EXISTS ${prefix} SCHEMALESS;`);
+  }
+
+  // OVERWRITE, not IF NOT EXISTS — this block only runs when schemaHash just changed (see the
+  // early return above), so an analyzer/index that already exists under this name here means its
+  // definition is stale and needs replacing, not skipping. FULLTEXT, not SEARCH — this SurrealDB
+  // version renamed the index-type keyword; SEARCH ANALYZER ... is a parse error here even though
+  // older docs/examples use it.
+  await db.query(`DEFINE ANALYZER OVERWRITE ${SEARCH_ANALYZER} ${SEARCH_ANALYZER_DEFINITION};`);
+
+  for (const { prefix: targetPrefix, field } of searchIndexes.values()) {
+    await db.query(
+      `DEFINE INDEX OVERWRITE idx_search_${targetPrefix}_${field} ON TABLE ${targetPrefix} FIELDS ${field} FULLTEXT ANALYZER ${SEARCH_ANALYZER} BM25 HIGHLIGHTS;`,
+    );
+  }
+
+  await db.query(`UPSERT ${SCHEMA_META_TABLE}:current SET hash = '${schemaHash}';`);
 };
 
 const surreal = makeModule(() => {