@aspan-corporation/ac-shared 1.2.36 → 1.2.37

package/lib/utils/diary.d.ts

@@ -49,6 +49,14 @@ export declare const parseDiaryKeyDate: (key: string) => {
  * words. Strips markdown syntax (code fences, image/link targets, formatting),
  * drops stop-words and tokens shorter than 2 chars, and caps the result at
  * `MAX_TEXT_TOKENS` to bound the meta-item size. No stemming (exact-word match).
+ *
+ * Word boundaries and lowercasing are Unicode-aware: tokens are split on any
+ * run of characters that are neither a Unicode letter (`\p{L}`) nor number
+ * (`\p{N}`), so non-Latin scripts (e.g. Cyrillic) survive as searchable words
+ * rather than being discarded. `toLowerCase()` is likewise Unicode-aware; we
+ * deliberately avoid `toLocaleLowerCase()` so tokenisation is deterministic and
+ * independent of the host locale (the indexer lambda and the browser that
+ * builds search queries must produce identical tokens).
  */
 export declare const tokenizeText: (markdown: string) => string[];
 /** Plain-text preview: strip markdown, collapse whitespace, truncate. */

package/lib/utils/diary.js

@@ -91,6 +91,14 @@ const STOP_WORDS = new Set([
  * words. Strips markdown syntax (code fences, image/link targets, formatting),
  * drops stop-words and tokens shorter than 2 chars, and caps the result at
  * `MAX_TEXT_TOKENS` to bound the meta-item size. No stemming (exact-word match).
+ *
+ * Word boundaries and lowercasing are Unicode-aware: tokens are split on any
+ * run of characters that are neither a Unicode letter (`\p{L}`) nor number
+ * (`\p{N}`), so non-Latin scripts (e.g. Cyrillic) survive as searchable words
+ * rather than being discarded. `toLowerCase()` is likewise Unicode-aware; we
+ * deliberately avoid `toLocaleLowerCase()` so tokenisation is deterministic and
+ * independent of the host locale (the indexer lambda and the browser that
+ * builds search queries must produce identical tokens).
  */
 export const tokenizeText = (markdown) => {
     if (!markdown)
@@ -109,11 +117,12 @@ export const tokenizeText = (markdown) => {
         // leftover markdown punctuation
         .replace(/[#>*_~\-]+/g, " ");
     const seen = new Set();
-    for (const raw of stripped.toLowerCase().split(/[^a-z0-9]+/)) {
+    for (const raw of stripped.toLowerCase().split(/[^\p{L}\p{N}]+/u)) {
         if (raw.length < 2)
             continue;
-        // Drop pure numbers (dates/counts add noise); keep alphanumerics like "v2".
-        if (/^\d+$/.test(raw))
+        // Drop pure-number tokens (dates/counts add noise) in any script; keep
+        // alphanumerics like "v2".
+        if (/^\p{N}+$/u.test(raw))
             continue;
         if (STOP_WORDS.has(raw))
             continue;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aspan-corporation/ac-shared",
-  "version": "1.2.36",
+  "version": "1.2.37",
   "description": "",
   "keywords": [],
   "exports": {