@dzhechkov/memory 0.2.21 → 0.2.22

package/.dz-manifest.json

@@ -9,7 +9,7 @@
       },
       {
         "path": "README.md",
-        "sha256": "6ae2da1d09bbe9ce6e3742079d659690f5c505a39eb187806c5f5be4c2d3a7d4"
+        "sha256": "4560bc1052213616df71c0eca90758f976eaf9683cfda760cfa67d701e57604b"
       },
       {
         "path": "dist/backend.d.ts",
@@ -77,19 +77,19 @@
       },
       {
         "path": "dist/index.d.ts",
-        "sha256": "c4e8be344baf02933c8e453bdba9ec3f95062d28672c8d0dabdd8035ef39e58e"
+        "sha256": "e73b2a1fc8f83272323e24cc117943712c929c6e2daddaf407c2ed71ab7ca1b8"
       },
       {
         "path": "dist/index.d.ts.map",
-        "sha256": "9576e873ac4003f6b217e0eff04f2cd5f8633397e004164abe7f10f9d7886511"
+        "sha256": "b84d8f5995bd3a1285a69361a638af35eba6c64c9b4945268da942894d48aff6"
       },
       {
         "path": "dist/index.js",
-        "sha256": "b6f72385ca8f3fbde2a5c11cf0deb98f58ea9836d216377e6585dc22b4b629b5"
+        "sha256": "be8ab9da014b2c9bf7472f342cebe2e9ae81d0867b3efbfb35f006ff811973c3"
       },
       {
         "path": "dist/index.js.map",
-        "sha256": "92e221dc762befc999d234a173b09b2e5c09c7e927518fccdd8bbebe15923712"
+        "sha256": "1c78cb041d2f482319f7a691330b313ed0235441835ec9ae78f8165a27eb3e5d"
       },
       {
         "path": "dist/json-backend.d.ts",
@@ -97,15 +97,15 @@
       },
       {
         "path": "dist/json-backend.d.ts.map",
-        "sha256": "95adb5287a060d417ed83d2364d8f63b973790e9885dc29ea7182132f4231b7c"
+        "sha256": "158819665930ed2cd3f86f4fa257b8b7bfa69d7638a69cfcc6d947be2c7d014c"
       },
       {
         "path": "dist/json-backend.js",
-        "sha256": "d53fcda53b7a1d24ccff4c3e1d98794985a9de01c97f6c357eec91ad64c2a341"
+        "sha256": "eb621bac0d3447892064fefb8c9e0f118cef1bf3ca6da538177afe4614320bb9"
       },
       {
         "path": "dist/json-backend.js.map",
-        "sha256": "7e0a073bfc25ae6fea53f29ad518319c31cdac070a1215bfb8a7584cb7ce9c33"
+        "sha256": "662b463dd30f390aaa60527bca2df3b42894bb70de3162a55c24aee1718efb65"
       },
       {
         "path": "dist/reflexion.d.ts",
@@ -129,15 +129,15 @@
       },
       {
         "path": "dist/sqlite-backend.d.ts.map",
-        "sha256": "0df025e4853d8420efc87d2b7aa8ea9f36e5b9a0a0be4351a7eb7614caf62b4c"
+        "sha256": "62b60365ba990ae1be666b5190e4f01d4a60989ddc62cea687e3823a4f0bb2cc"
       },
       {
         "path": "dist/sqlite-backend.js",
-        "sha256": "4f2a5c61fc27a8aa39f1adfb06215af80bbde0236f1f2a7f73ab65d3ab3f3652"
+        "sha256": "664eb414c5438dda67b5e6b2e9c3281a20c38c4b85a1e9ee48b004a1e7c08d24"
       },
       {
         "path": "dist/sqlite-backend.js.map",
-        "sha256": "4106d2a7cfa2ad50ea5497dab8ad18d2102019f05e5a3aca2469ab2ec325ce48"
+        "sha256": "12ba0851550864589cb8fad02950cfc7ff468bdf0e87b429ba4bfc6a7294e28d"
       },
       {
         "path": "dist/sqlite-probe.d.ts",
@@ -171,9 +171,25 @@
         "path": "dist/sqlite-readonly.js.map",
         "sha256": "03ddeec35c1389f249b0595f2612b8c3f86e5069b4c3f1be1e30eb6f6949bf1c"
       },
+      {
+        "path": "dist/tokenize.d.ts",
+        "sha256": "5c1fcc80c8d6836c3b80dba6de50e98b06c70bc2d15c46b1392bb305af6e5e57"
+      },
+      {
+        "path": "dist/tokenize.d.ts.map",
+        "sha256": "ec47c8a2687a0f598c0e81a5871e9a011cf686880c8a34915af31a09701faddc"
+      },
+      {
+        "path": "dist/tokenize.js",
+        "sha256": "bc1b9368d3e375bed2293afead3cc5d23409f8f83fae2e8f9eb790542ab02635"
+      },
+      {
+        "path": "dist/tokenize.js.map",
+        "sha256": "8fd9e4236d32873ee9b9ea4b3d44f9e4dfffa09071dc3831e26956f88f662fcf"
+      },
       {
         "path": "package.json",
-        "sha256": "cd706c6ae1def0fd6e4bdb9c84e0d95239a24bda69eb8f70b47377189c26fc14"
+        "sha256": "ea0e5da14d9fdddbb6a5a6278610f0bcda3b98f1cae6287def6e5565f01a3ab6"
       },
       {
         "path": "src/backend.ts",
@@ -193,11 +209,11 @@
       },
       {
         "path": "src/index.ts",
-        "sha256": "9152ce2677f8b209bb26a90cf15b1f97dccd7e5fcdcc95e7cf417a698798fc56"
+        "sha256": "4ac9affbf4fbe220a7a23c08ce48a1d9ffc02c1ddc4774fd41abbeccd33d8993"
       },
       {
         "path": "src/json-backend.ts",
-        "sha256": "ba881b55f48dabe7d95ce87ee1c840180850899f551173cd5940e54a8d6c6cf6"
+        "sha256": "37b2e031e8dc9437b55b3b57ad83e2ebc962335d16c496834a7c4aed9a1770cc"
       },
       {
         "path": "src/reflexion.ts",
@@ -205,7 +221,7 @@
       },
       {
         "path": "src/sqlite-backend.ts",
-        "sha256": "08b618bf7fd351120aac5c4140ae84c2d207c45d708dc42a98f175a99938a11c"
+        "sha256": "d91cff21f894ae44483b06807fdbbe4958bee00292c0cd85469e2c5f3374c055"
       },
       {
         "path": "src/sqlite-probe.ts",
@@ -214,8 +230,12 @@
       {
         "path": "src/sqlite-readonly.ts",
         "sha256": "c973b0eeae5567af4c519c178ccb48d496fa1cf2ad226363b26ecc2e7d11dc45"
+      },
+      {
+        "path": "src/tokenize.ts",
+        "sha256": "16b59783d4cf4023d810584564063b195e1cae83a4f4c4fd9d11751ad3d1e7be"
       }
     ]
   },
-  "signature": "HZbuFkEr2m7sCfqEpoQvK8RH++xeSYo2l+EYQ5O5ibJfg9WsL8gdFFSLJMu6iDgBcU4LkfJ8LXYxL7aPr+qbDQ=="
+  "signature": "gjCK4WP4Jqgmf4A4E33RyaxjTuEzDxR4UYJGIKskmig3Dpvb1v4nldWiPsfBhvtDPzyIrjxN1Do7FzYT4OHIAQ=="
 }

package/README.md

@@ -21,18 +21,69 @@ with the whole store, reordered. MEASURED on two records (`hello world`, `anothe
 `zebrafish` returned both, and so did `hello`, which matches exactly one. With no matches the sort
 degenerates into confidence order — which is how it looked like "ranking by confidence".
 
-Now the two situations are separated:
+Now the three situations are separated:
 
 | The query | What comes back |
 |---|---|
 | has usable terms and matches nothing | nothing |
 | has usable terms and matches some records | only those |
-| has NO usable terms (punctuation, single characters, no text) | the whole store, ranked by confidence — unchanged, because no filter was expressible |
+| was PROVIDED but tokenizes to NO usable terms (pure punctuation/whitespace/empty) | nothing — a named `no-searchable-terms` reason, not a silent full-store dump (`recall-short-terms`, 2026-09-15; see below) |
+| has NO text field at all (`{}` — a browse, not a search) | the whole store, ranked by confidence — unchanged, because nothing was ever expressible to filter on |
 
 The filter is `overlap > 0`, never a tuned threshold: a weak match is still a match, and zero overlap
 is not a weak match. Both backends take the same rule, so a store's answers never depend on which one
 is installed. Note that "matched" is measured over the record's TEXT and its `skillId`.
 
+## Short terms — `x`, `C`, `Go`, `ID`, `db` are now searchable (`recall-short-terms`, 2026-09-15)
+
+Until 2026-09-15 `tokenize` dropped every token of length <= 1 code point (`memory/src/tokenize.ts`,
+shared by both backends since this feature — FR-1). Single- and two-character alphanumeric entities
+are real in this domain (`C`, `Go`, `R`, `ID`, `db`, ADR letter variants), so a record whose entire
+text is `x` was never found by querying `x`: the query tokenized to zero terms and fell into the
+"nothing to search by" branch, which used to dump the whole store instead of the specific match
+(backlog 529c31ab, MEASURED by Codex 2026-08-22).
+
+The floor is gone: any non-empty run of `\p{L}\p{N}` characters is a token now, matched **only by
+exact equality** — never a prefix or stem (`stemOf`, in each backend, already refuses anything under
+5 code points, so this was already the rule for 1-2 char terms; recall-short-terms only widens which
+terms REACH that rule). `Go concurrency` finds a record about Go, not one about "going".
+
+`tokenize.ts` also exports `hasSearchableTerms(text)` — the single place that decides whether a query
+falls into the "no-searchable-terms" branch above — and `noSearchableTermsReason(text)`, which returns
+the literal `'no-searchable-terms'` string or `undefined`. Both backends call `hasSearchableTerms`
+directly for the branch decision now (fix-round 1, Codex HIGH-1) rather than reimplementing it as
+`terms.length === 0`; `dz recall`'s CLI printer calls `noSearchableTermsReason` (re-exported from
+`@dzhechkov/harness-core`) so the reason it prints — `no searchable terms in "<query>" (only
+punctuation/whitespace) — reason: no-searchable-terms` — can never drift from what the backends
+actually decided.
+
+**No stop-list, by design.** Every alphanumeric token is searchable, including one-letter ones —
+`a`, `i`, `и`, `в` all count. A one-letter query can legitimately match every record that contains
+that letter as a standalone token; the caller's own `limit` bounds the flood, this package does not
+maintain a stop-list to pre-filter it (measured: 50 records each containing token `a`, query `a`,
+`limit: 5` → exactly 5 hits, in both backends — `test/short-terms.test.ts`).
+
+**Shipping FTS5 (fix-round 1, Codex HIGH-2/MEDIUM-5).** `test/short-terms.test.ts`'s twins corpus
+now runs BOTH with FTS5 forced off (the keyword-overlap CONTROL — both backends run the identical
+algorithm, so any divergence can only be the shared tokenizer) and with FTS5 left ON, the shipping
+default. The FTS5-on comparison checks the MATCHED SET, not ranking order: FTS5's bm25 and the
+keyword path's hit-count relevance are different algorithms and may legitimately tie-break ties
+differently even when they agree on which records qualify. Separately, single-character Cyrillic,
+CJK, and astral-plane terms are each planted and then FOUND via shipping FTS5 — proving they are
+reachable, not merely that an absent one returns empty (which `tokenize-unicode.test.ts`'s A7
+already covered).
+
+`tokenize()` also `.normalize('NFC')`s before splitting (fix-round 1, HIGH-2, MEASURED via a
+`better-sqlite3` probe against an in-memory FTS5 table): precomposed `café` (`é` = U+00E9) and
+decomposed `café` (`e` + combining acute U+0301) are the same word to a reader and to shipping
+FTS5's own tokenizer, which already matched both forms (and even bare `cafe`) to each other. Without
+NFC, this tokenizer disagreed with itself across representations — the combining mark falls in the
+SEPARATOR class, so decomposed `café` tokenized to `cafe` (accent silently dropped) while
+precomposed `café` tokenized to the distinct string `café`. NFC-normalizing first fixes that
+representation bug: both forms now tokenize to the identical `café`.
+
+**Diacritics are FOLDED, in both backends (Codex r2, lead fix).** Shipping FTS5's default `unicode61` tokenizer strips diacritics (`remove_diacritics=1`): a MATCH for plain `cafe` finds a row stored as `café`. The keyword-overlap path (JSON backend, or sqlite with FTS5 off) used to keep `café` and `cafe` as two tokens — an exact-parity violation between the two backends of one store. `tokenize()` therefore folds diacritics the way `unicode61` does — for LATIN script only (NFD, drop combining marks after a Latin base letter, NFC back), so both paths answer alike (tested in both backends: `cafe`/`café`/decomposed `café` and `ano`/`año` fold; Cyrillic `й`≠`и`, `ё`≠`е` do NOT — MEASURED against FTS5, which keeps them distinct; and a Latin letter carrying TWO marks — `ộ`, `ǘ` — is NOT folded either, exactly as `remove_diacritics=1` leaves it, measured: `MATCH 'o'` does not return the `ộ` row). **Named limit, inherited from the shipping engine:** words that differ only by an accent (`año`/`ano`, `côté`/`cote`) conflate — exactly as FTS5 already conflated them; parity was chosen over a narrower keyword rule.
+
 ## Backend strategy
 
 The default `JsonFileBackend` is **pure JavaScript with zero dependencies** — no
@@ -83,9 +134,10 @@ missing, the writer creates/rebuilds FTS5 on open while the read-only store only
 the two on that specific old/incomplete schema; not fixed here, a named limit.
 
 **Реестр мутаций:** этот пакет несёт собственный `test/mutation-registry.json` (ADR-001 C-1…C-3 —
-опенер чтения) — `dz mutation-gate --package packages/@dzhechkov/memory --only <id>` мутирует
+опенер чтения; `short-terms-searchable` — AC-4, `recall-short-terms`, 2026-09-15, единый токенизатор
+в `src/tokenize.ts`) — `dz mutation-gate --package packages/@dzhechkov/memory --only <id>` мутирует
 названное свойство в scratch-копии и требует, чтобы `npx vitest run test/sqlite-readonly.test.ts
-test/sqlite-backend.test.ts` покраснел; форму реестра проверяет
+test/sqlite-backend.test.ts test/short-terms.test.ts` покраснел; форму реестра проверяет
 `test/mutation-registry-shape.test.ts`.
 
 ## Status
@@ -94,7 +146,9 @@ test/sqlite-backend.test.ts` покраснел; форму реестра пр
 
 ## Status
 
-`0.2.21` — ships `openSqliteReadOnly` (read-only opener ladder: in-place → tmp copy → honest error; feature store-readonly-reads, 2026-09-12) — the export `@dzhechkov/harness-core` ≥ 0.8.31 imports; `0.2.19` was a signature-only republish; `0.2.11` shipped with a stale manifest.
+`0.2.22` — ONE shared `tokenize()` (`src/tokenize.ts`) for both backends: no length floor (one-character terms are searchable), Latin-only diacritic folding measured against FTS5 `unicode61 remove_diacritics=1` (one combining mark folds; two marks, Cyrillic `й`/`ё` do not), `hasSearchableTerms` / `noSearchableTermsReason` exported, a query that tokenizes to nothing returns `[]` (feature recall-short-terms, 2026-09-15; `@dzhechkov/harness-core` ≥ 0.8.35 re-exports the reason helper).
+
+`0.2.22` — ships `openSqliteReadOnly` (read-only opener ladder: in-place → tmp copy → honest error; feature store-readonly-reads, 2026-09-12) — the export `@dzhechkov/harness-core` ≥ 0.8.31 imports; `0.2.19` was a signature-only republish; `0.2.11` shipped with a stale manifest.
 
 `0.2.11` — a query that matched nothing returns nothing (see above). `0.2.10` — the lexical tokenizer is Unicode-aware. It split on `[^a-z0-9]+`, so every non-Latin letter
 was a separator and a Cyrillic query produced **zero terms**: the FTS5 branch was skipped, relevance
@@ -106,6 +160,9 @@ floor counts code points rather than UTF-16 units.
 No migration is needed: FTS5's own tokenizer always indexed the text correctly — only the query was
 being stripped of its terms on the way out.
 
-**Unchanged on purpose:** a query that yields no terms still returns the store. That behaviour is
-pinned by four existing tests which comment it as intended, so it is a contract to be changed by
-decision, not folded into an alphabet fix.
+**Left unchanged HERE, on purpose:** at the time of this fix, a query that yielded no terms still
+returned the store — a contract deliberately NOT folded into this alphabet-only fix, pinned by four
+existing tests that commented it as intended. That contract was narrowed by a LATER, SEPARATE
+decision — `recall-short-terms` (2026-09-15, see above): the one-character floor was removed, and a
+PROVIDED query that still yields zero terms (pure punctuation) now returns empty, not the store. A
+query with no text field at all still returns the store — that half of the old contract survives.

package/dist/index.d.ts

@@ -6,6 +6,7 @@
 /** Package version. Kept in sync with `package.json`. */
 export declare const MEMORY_VERSION: string;
 export type { MemoryBackend, MemoryQuery, MemoryRecord } from './backend.js';
+export { tokenize, hasSearchableTerms, noSearchableTermsReason } from './tokenize.js';
 export { JsonFileBackend } from './json-backend.js';
 export type { JsonFileBackendOptions } from './json-backend.js';
 export { selectBackend } from './cascade.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,yDAAyD;AACzD,eAAO,MAAM,cAAc,EAAE,MACuD,CAAC;AAErF,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,YAAY,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,YAAY,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC/E,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC/F,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,YAAY,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACjE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC5G,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,yDAAyD;AACzD,eAAO,MAAM,cAAc,EAAE,MACuD,CAAC;AAErF,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC7E,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AACtF,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,YAAY,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,YAAY,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC/E,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC/F,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,YAAY,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACjE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC5G,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC"}

package/dist/index.js

@@ -6,6 +6,7 @@
 import { createRequire } from 'node:module';
 /** Package version. Kept in sync with `package.json`. */
 export const MEMORY_VERSION = createRequire(import.meta.url)('../package.json').version;
+export { tokenize, hasSearchableTerms, noSearchableTermsReason } from './tokenize.js';
 export { JsonFileBackend } from './json-backend.js';
 export { selectBackend } from './cascade.js';
 export { SqliteBackend } from './sqlite-backend.js';

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,yDAAyD;AACzD,MAAM,CAAC,MAAM,cAAc,GACxB,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAyB,CAAC,OAAO,CAAC;AAGrF,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE/E,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,OAAO,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,yDAAyD;AACzD,MAAM,CAAC,MAAM,cAAc,GACxB,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAyB,CAAC,OAAO,CAAC;AAGrF,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AACtF,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE/E,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,OAAO,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC"}

package/dist/json-backend.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"json-backend.d.ts","sourceRoot":"","sources":["../src/json-backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAKH,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAmE7E,2CAA2C;AAC3C,MAAM,WAAW,sBAAsB;IACrC,uEAAuE;IACvE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,sFAAsF;AACtF,qBAAa,eAAgB,YAAW,aAAa;IACnD,QAAQ,CAAC,IAAI,eAAe;IAE5B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;gBAElC,OAAO,GAAE,sBAA2B;IAIhD,6EAA6E;WAChE,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAM7D;;;;OAIG;IACH,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,eAAe;IAMlD,GAAG,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAIlD,4EAA4E;IAC5E,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,YAAY,EAAE;IAwB7C,GAAG,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;IAI9B,+CAA+C;IAC/C,OAAO,IAAI,YAAY,EAAE;IAIzB,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKjC,gFAAgF;IAChF,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAI5B,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAIxB;;;;;;;OAOG;IACH,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAerB,qFAAqF;IACrF,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAKrB,gDAAgD;IAChD,QAAQ,IAAI,IAAI;CAMjB"}
+{"version":3,"file":"json-backend.d.ts","sourceRoot":"","sources":["../src/json-backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAKH,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AA4C7E,2CAA2C;AAC3C,MAAM,WAAW,sBAAsB;IACrC,uEAAuE;IACvE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,sFAAsF;AACtF,qBAAa,eAAgB,YAAW,aAAa;IACnD,QAAQ,CAAC,IAAI,eAAe;IAE5B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;gBAElC,OAAO,GAAE,sBAA2B;IAIhD,6EAA6E;WAChE,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAM7D;;;;OAIG;IACH,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,eAAe;IAMlD,GAAG,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAIlD,4EAA4E;IAC5E,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,YAAY,EAAE;IAwC7C,GAAG,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;IAI9B,+CAA+C;IAC/C,OAAO,IAAI,YAAY,EAAE;IAIzB,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKjC,gFAAgF;IAChF,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAI5B,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAIxB;;;;;;;OAOG;IACH,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAerB,qFAAqF;IACrF,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAKrB,gDAAgD;IAChD,QAAQ,IAAI,IAAI;CAMjB"}

package/dist/json-backend.js

@@ -9,30 +9,8 @@
  */
 import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
 import { dirname } from 'node:path';
+import { tokenize, hasSearchableTerms } from './tokenize.js';
 const DEFAULT_LIMIT = 20;
-/**
- * Split text into lowercase word tokens of length > 1.
- *
- * The class is `\p{L}\p{N}`, not `a-z0-9`. Until 2026-08-21 it was ASCII-only, so every non-Latin
- * letter was a SEPARATOR and a Cyrillic query produced ZERO tokens — the FTS5 branch was then skipped
- * entirely, `relevanceOf` returned 0 for every record, and the sort collapsed onto its confidence
- * tie-break. MEASURED on a 267-record clone of the real brain: RU top-1 0/10 against EN 10/10, while
- * 63% of real recall traffic is Cyrillic. The INDEX was never wrong — FTS5's own tokenizer handles
- * Cyrillic — so nothing on disk needed migrating; only the query was being stripped of its terms.
- *
- * `\p{L}` admits letters and `\p{N}` digits; it does NOT admit `"`, `*`, `(` or any other FTS5
- * operator, which is what keeps the joined terms safe to interpolate into a MATCH expression.
- */
-function tokenize(text) {
-    return text
-        .toLowerCase()
-        .split(/[^\p{L}\p{N}]+/u)
-        // Count CODE POINTS, not UTF-16 units. `token.length` counts units, so a single astral letter
-        // (`𐐀`, one character, two units) would slip past a floor meant to reject one-character words —
-        // an accidental threshold change smuggled in by the alphabet change (cross-family review,
-        // 2026-08-21). The promise was "the alphabet, not the thresholds"; this keeps it.
-        .filter((token) => [...token].length > 1);
-}
 /**
  * Crude prefix-stem for morphology-bearing languages — feature recall-ru-morphology.
  *
@@ -116,7 +94,22 @@ export class JsonFileBackend {
     /** Synchronous {@link JsonFileBackend.query} — same ranking, no Promise. */
     querySync(query) {
         const limit = query.limit ?? DEFAULT_LIMIT;
-        const terms = query.text !== undefined ? tokenize(query.text) : [];
+        const noText = query.text === undefined;
+        const terms = noText ? [] : tokenize(query.text);
+        // FR-3 (recall-short-terms): text was SUPPLIED but tokenized to nothing (pure
+        // punctuation/whitespace/empty). Until 2026-09-15 this silently returned the WHOLE STORE,
+        // indistinguishable from "here is everything you asked for". Honest behavior for an
+        // unsatisfiable search is EMPTY, same as any other failed search (ADR-001's own
+        // "no-match-means-no-results"). A query with NO text field at all is a different intent —
+        // "browse everything", never attempted as a search — untouched below.
+        //
+        // Fix-round 1 (Codex HIGH-1): this used to reimplement the decision as `terms.length === 0` —
+        // a hand-rolled copy of `hasSearchableTerms`'s own logic, forkable exactly the way the old
+        // per-backend `tokenize()` was. Calling the shared helper directly closes that gap; `terms`
+        // is still computed above for `relevanceOf` below, but the BRANCH decision is never re-derived.
+        if (!noText && !hasSearchableTerms(query.text)) {
+            return []; // reason: 'no-searchable-terms' — see tokenize.ts `noSearchableTermsReason`
+        }
         let candidates = [...this.records.values()];
         if (query.skillId !== undefined) {
             candidates = candidates.filter((record) => record.skillId === query.skillId);
@@ -128,9 +121,10 @@ export class JsonFileBackend {
             b.record.timestamp.localeCompare(a.record.timestamp));
         // ASKING and finding nothing returns nothing. Without this the keyword path RANKS by overlap and
         // never EXCLUDES, so every query returned the whole store reordered — MEASURED on two records:
-        // `zebrafish` (matches neither) AND `hello` (matches one) both returned both. With no usable
-        // terms there was nothing to match on, and returning the store ranked by confidence stays the
-        // right answer; that distinction is the whole decision (ADR-001).
+        // `zebrafish` (matches neither) AND `hello` (matches one) both returned both. `noText` is the
+        // ONLY way `terms.length === 0` reaches this point now (the provided-but-unsearchable case
+        // returned above) — nothing was ever expressible to filter on, so the store still comes back
+        // ranked by confidence; that distinction is the whole decision (ADR-001).
         const filtered = terms.length > 0 ? ranked.filter((entry) => entry.relevance > 0) : ranked;
         return filtered.slice(0, limit).map((entry) => entry.record);
     }

package/dist/json-backend.js.map

@@ -1 +1 @@
-{"version":3,"file":"json-backend.js","sourceRoot":"","sources":["../src/json-backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACjG,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB;;;;;;;;;;;;GAYG;AACH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI;SACR,WAAW,EAAE;SACb,KAAK,CAAC,iBAAiB,CAAC;QACzB,8FAA8F;QAC9F,iGAAiG;QACjG,0FAA0F;QAC1F,kFAAkF;SACjF,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,MAAM,CAAC,KAAa;IAC3B,MAAM,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACvB,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtD,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;mGACmG;AACnG,SAAS,WAAW,CAAC,MAAoB,EAAE,KAAwB;IACjE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,IAAI,IAAI,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QAChD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,IAAI,KAAK,GAAG,KAAK,CAAC;YAClB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBAAC,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;oBAAC,KAAK,GAAG,IAAI,CAAC;oBAAC,MAAM;gBAAC,CAAC;YAAC,CAAC;YAC5E,IAAI,KAAK;gBAAE,IAAI,IAAI,GAAG,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAQD,sFAAsF;AACtF,MAAM,OAAO,eAAe;IACjB,IAAI,GAAG,WAAW,CAAC;IAEX,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC1C,QAAQ,CAAqB;IAE9C,YAAY,UAAkC,EAAE;QAC9C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IACnC,CAAC;IAED,6EAA6E;IAC7E,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAgB;QAChC,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClD,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QACrB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,QAAQ,CAAC,QAAgB;QAC9B,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,QAAQ,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,GAAG,CAAC,MAAoB;QACtB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACpC,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,KAAkB;QACtB,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,4EAA4E;IAC5E,SAAS,CAAC,KAAkB;QAC1B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,aAAa,CAAC;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACnE,IAAI,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5C,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAChC,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/E,CAAC;QACD,MAAM,MAAM,GAAG,UAAU;aACtB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;aACpE,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS;YACzB,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK;YAC/B,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CACvD,CAAC;QACJ,iGAAiG;QACjG,+FAA+F;QAC/F,6FAA6F;QAC7F,8FAA8F;QAC9F,kEAAkE;QAClE,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC3F,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/D,CAAC;IAED,GAAG;QACD,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,+CAA+C;IAC/C,OAAO;QACL,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,CAAC,EAAU;QACf,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACpB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,gFAAgF;IAChF,UAAU,CAAC,EAAU;QACnB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK;QACH,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;OAOG;IACH,IAAI;QACF,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YAChC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACvD,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,QAAQ,QAAQ,OAAO,CAAC,GAAG,EAAE,CAAC;YAClD,IAAI,CAAC;gBACH,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;gBACxE,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,sCAAsC;gBACpE,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,qFAAqF;IACrF,IAAI;QACF,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,gDAAgD;IAChD,QAAQ;QACN,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAmB,CAAC;YAChF,KAAK,MAAM,MAAM,IAAI,IAAI;gBAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"json-backend.js","sourceRoot":"","sources":["../src/json-backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACjG,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAE7D,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB;;;;;;;;;;;;GAYG;AACH,SAAS,MAAM,CAAC,KAAa;IAC3B,MAAM,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACvB,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtD,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;mGACmG;AACnG,SAAS,WAAW,CAAC,MAAoB,EAAE,KAAwB;IACjE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,IAAI,IAAI,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QAChD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,IAAI,KAAK,GAAG,KAAK,CAAC;YAClB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBAAC,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;oBAAC,KAAK,GAAG,IAAI,CAAC;oBAAC,MAAM;gBAAC,CAAC;YAAC,CAAC;YAC5E,IAAI,KAAK;gBAAE,IAAI,IAAI,GAAG,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAQD,sFAAsF;AACtF,MAAM,OAAO,eAAe;IACjB,IAAI,GAAG,WAAW,CAAC;IAEX,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC1C,QAAQ,CAAqB;IAE9C,YAAY,UAAkC,EAAE;QAC9C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IACnC,CAAC;IAED,6EAA6E;IAC7E,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAgB;QAChC,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClD,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QACrB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,QAAQ,CAAC,QAAgB;QAC9B,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,QAAQ,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,GAAG,CAAC,MAAoB;QACtB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACpC,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,KAAkB;QACtB,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,4EAA4E;IAC5E,SAAS,CAAC,KAAkB;QAC1B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,aAAa,CAAC;QAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC;QACxC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAK,CAAC,CAAC;QAClD,8EAA8E;QAC9E,0FAA0F;QAC1F,oFAAoF;QACpF,gFAAgF;QAChF,0FAA0F;QAC1F,sEAAsE;QACtE,EAAE;QACF,8FAA8F;QAC9F,2FAA2F;QAC3F,4FAA4F;QAC5F,gGAAgG;QAChG,IAAI,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,IAAK,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,CAAC,CAAC,4EAA4E;QACzF,CAAC;QACD,IAAI,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5C,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAChC,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/E,CAAC;QACD,MAAM,MAAM,GAAG,UAAU;aACtB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;aACpE,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS;YACzB,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK;YAC/B,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CACvD,CAAC;QACJ,iGAAiG;QACjG,+FAA+F;QAC/F,8FAA8F;QAC9F,2FAA2F;QAC3F,6FAA6F;QAC7F,0EAA0E;QAC1E,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC3F,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/D,CAAC;IAED,GAAG;QACD,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,+CAA+C;IAC/C,OAAO;QACL,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,CAAC,EAAU;QACf,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACpB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,gFAAgF;IAChF,UAAU,CAAC,EAAU;QACnB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK;QACH,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;OAOG;IACH,IAAI;QACF,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YAChC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACvD,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,QAAQ,QAAQ,OAAO,CAAC,GAAG,EAAE,CAAC;YAClD,IAAI,CAAC;gBACH,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;gBACxE,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,sCAAsC;gBACpE,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,qFAAqF;IACrF,IAAI;QACF,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,gDAAgD;IAChD,QAAQ;QACN,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAmB,CAAC;YAChF,KAAK,MAAM,MAAM,IAAI,IAAI;gBAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;CACF"}

package/dist/sqlite-backend.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-backend.d.ts","sourceRoot":"","sources":["../src/sqlite-backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE7E,OAAO,KAAK,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAwG/E,gFAAgF;AAChF,eAAO,MAAM,eAAe,gKAK3B,CAAC;AAEF,eAAO,MAAM,qBAAqB,oLAKjC,CAAC;AASF,eAAO,MAAM,OAAO,iCAAiC,CAAC;AACtD,eAAO,MAAM,SAAS,+CAA+C,CAAC;AACtE,eAAO,MAAM,YAAY,oDAAoD,CAAC;AAE9E;;;;;;;;;;;GAWG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE;IAAE,GAAG,CAAC,EAAE,GAAG,CAAC;IAAC,QAAQ,CAAC,EAAE,GAAG,CAAC;IAAC,GAAG,EAAE,GAAG,CAAC;IAAC,OAAO,EAAE,GAAG,CAAA;CAAE,EAC5D,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,WAAW,GACjB,YAAY,EAAE,CAmEhB;AAED,iCAAiC;AACjC,MAAM,WAAW,oBAAoB;IACnC,wCAAwC;IACxC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,qBAAa,aAAc,YAAW,aAAa;IACjD,QAAQ,CAAC,IAAI,YAAY;IAEzB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAM;IACzB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAM;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAM;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAM;IAC9B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAM;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAM;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkB;IAChD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAkB;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;gBAEtB,EAAE,EAAE,GAAG;IAyBnB,4DAA4D;IAC5D,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa;IAW5C;;;;;;OAMG;IACH,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,mBAAmB,GAAG,aAAa;IAmBhF,GAAG,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAaxC,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAIlD;;;;OAIG;IACH,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,YAAY,EAAE;IAQ7C,GAAG,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;IAI9B,6CAA6C;IAC7C,OAAO,IAAI,YAAY,EAAE;IAIzB,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKjC,sIAAsI;IACtI,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAI5B,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAIxB,oEAAoE;IACpE,OAAO,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,GAAG,IAAI;IAiB/C,qCAAqC;IACrC,KAAK,IAAI,IAAI;CAGd;AAED,kDAAkD;AAClD,wBAAgB,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,YAAY,CAUlD"}
+{"version":3,"file":"sqlite-backend.d.ts","sourceRoot":"","sources":["../src/sqlite-backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE7E,OAAO,KAAK,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAiF/E,gFAAgF;AAChF,eAAO,MAAM,eAAe,gKAK3B,CAAC;AAEF,eAAO,MAAM,qBAAqB,oLAKjC,CAAC;AASF,eAAO,MAAM,OAAO,iCAAiC,CAAC;AACtD,eAAO,MAAM,SAAS,+CAA+C,CAAC;AACtE,eAAO,MAAM,YAAY,oDAAoD,CAAC;AAE9E;;;;;;;;;;;GAWG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE;IAAE,GAAG,CAAC,EAAE,GAAG,CAAC;IAAC,QAAQ,CAAC,EAAE,GAAG,CAAC;IAAC,GAAG,EAAE,GAAG,CAAC;IAAC,OAAO,EAAE,GAAG,CAAA;CAAE,EAC5D,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,WAAW,GACjB,YAAY,EAAE,CAqFhB;AAED,iCAAiC;AACjC,MAAM,WAAW,oBAAoB;IACnC,wCAAwC;IACxC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,qBAAa,aAAc,YAAW,aAAa;IACjD,QAAQ,CAAC,IAAI,YAAY;IAEzB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAM;IACzB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAM;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAM;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAM;IAC9B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAM;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAM;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkB;IAChD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAkB;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;gBAEtB,EAAE,EAAE,GAAG;IAyBnB,4DAA4D;IAC5D,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa;IAW5C;;;;;;OAMG;IACH,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,mBAAmB,GAAG,aAAa;IAmBhF,GAAG,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAaxC,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAIlD;;;;OAIG;IACH,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,YAAY,EAAE;IAQ7C,GAAG,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;IAI9B,6CAA6C;IAC7C,OAAO,IAAI,YAAY,EAAE;IAIzB,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKjC,sIAAsI;IACtI,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAI5B,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAIxB,oEAAoE;IACpE,OAAO,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,GAAG,IAAI;IAiB/C,qCAAqC;IACrC,KAAK,IAAI,IAAI;CAGd;AAED,kDAAkD;AAClD,wBAAgB,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,YAAY,CAUlD"}

package/dist/sqlite-backend.js

@@ -11,31 +11,9 @@ import { createRequire } from 'node:module';
 import { mkdirSync } from 'node:fs';
 import { dirname } from 'node:path';
 import { openSqliteReadOnly, SqliteReadOnlyStore } from './sqlite-readonly.js';
+import { tokenize, hasSearchableTerms } from './tokenize.js';
 const require = createRequire(import.meta.url);
 const DEFAULT_LIMIT = 20;
-/**
- * Split text into lowercase word tokens of length > 1.
- *
- * The class is `\p{L}\p{N}`, not `a-z0-9`. Until 2026-08-21 it was ASCII-only, so every non-Latin
- * letter was a SEPARATOR and a Cyrillic query produced ZERO tokens — the FTS5 branch was then skipped
- * entirely, `relevanceOf` returned 0 for every record, and the sort collapsed onto its confidence
- * tie-break. MEASURED on a 267-record clone of the real brain: RU top-1 0/10 against EN 10/10, while
- * 63% of real recall traffic is Cyrillic. The INDEX was never wrong — FTS5's own tokenizer handles
- * Cyrillic — so nothing on disk needed migrating; only the query was being stripped of its terms.
- *
- * `\p{L}` admits letters and `\p{N}` digits; it does NOT admit `"`, `*`, `(` or any other FTS5
- * operator, which is what keeps the joined terms safe to interpolate into a MATCH expression.
- */
-function tokenize(text) {
-    return text
-        .toLowerCase()
-        .split(/[^\p{L}\p{N}]+/u)
-        // Count CODE POINTS, not UTF-16 units. `token.length` counts units, so a single astral letter
-        // (`𐐀`, one character, two units) would slip past a floor meant to reject one-character words —
-        // an accidental threshold change smuggled in by the alphabet change (cross-family review,
-        // 2026-08-21). The promise was "the alphabet, not the thresholds"; this keeps it.
-        .filter((token) => [...token].length > 1);
-}
 /**
  * Crude prefix-stem for morphology-bearing languages — feature recall-ru-morphology.
  *
@@ -191,7 +169,23 @@ export function searchPreparedRecords(stmts, hasFts5, query) {
         }
     }
     // Keyword overlap fallback
-    const terms = query.text !== undefined ? tokenize(query.text) : [];
+    const noText = query.text === undefined;
+    const terms = noText ? [] : tokenize(query.text);
+    // FR-3 (recall-short-terms): a query that SUPPLIED text but tokenized to literally nothing
+    // (pure punctuation/whitespace/empty) has nothing to filter on, and until 2026-09-15 that
+    // silently dumped the WHOLE STORE — indistinguishable from "here is everything you asked for".
+    // That is the bug this feature names and fixes: honest behavior for an unsatisfiable search is
+    // EMPTY, the same "no-match-means-no-results" answer any other failed search gets (ADR-001). A
+    // query with NO text field at all is a different intent — "browse everything", never attempted
+    // as a search — and keeps the old whole-store answer below.
+    //
+    // Fix-round 1 (Codex HIGH-1): this used to reimplement the decision as `terms.length === 0` —
+    // a hand-rolled copy of `hasSearchableTerms`'s own logic, forkable exactly the way the old
+    // per-backend `tokenize()` was. Calling the shared helper directly closes that gap; `terms` is
+    // still computed above for `relevanceOf` below, but the BRANCH decision is never re-derived.
+    if (!noText && !hasSearchableTerms(query.text)) {
+        return []; // reason: 'no-searchable-terms' — see tokenize.ts `noSearchableTermsReason`
+    }
     let rows;
     if (query.skillId !== undefined) {
         rows = stmts.bySkill.all(query.skillId);
@@ -207,8 +201,9 @@ export function searchPreparedRecords(stmts, hasFts5, query) {
         b.record.timestamp.localeCompare(a.record.timestamp));
     // The SAME guard the JSON backend applies, so a store's answers never depend on which backend is
     // installed. This is the keyword FALLBACK; the FTS5 path above already returns zero honestly and
-    // is untouched. With no usable terms there was nothing to match on, so the store still comes back
-    // ranked by confidence — that distinction is the whole decision (ADR-001).
+    // is untouched. `noText` is the ONLY way `terms.length === 0` reaches this point now (the
+    // provided-but-unsearchable case returned above) — a genuine "no query at all" browse, so nothing
+    // was ever expressible to filter on and the store comes back ranked by confidence (ADR-001).
     const filtered = terms.length > 0 ? ranked.filter((entry) => entry.relevance > 0) : ranked;
     return filtered.slice(0, limit).map((entry) => entry.record);
 }

package/dist/sqlite-backend.js.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-backend.js","sourceRoot":"","sources":["../src/sqlite-backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAG/E,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAE/C,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB;;;;;;;;;;;;GAYG;AACH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI;SACR,WAAW,EAAE;SACb,KAAK,CAAC,iBAAiB,CAAC;QACzB,8FAA8F;QAC9F,iGAAiG;QACjG,0FAA0F;QAC1F,kFAAkF;SACjF,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,MAAM,CAAC,KAAa;IAC3B,MAAM,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACvB,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtD,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;mGACmG;AACnG,SAAS,WAAW,CAAC,MAAoB,EAAE,KAAwB;IACjE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,IAAI,IAAI,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QAChD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,IAAI,KAAK,GAAG,KAAK,CAAC;YAClB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBAAC,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;oBAAC,KAAK,GAAG,IAAI,CAAC;oBAAC,MAAM;gBAAC,CAAC;YAAC,CAAC;YAC5E,IAAI,KAAK;gBAAE,IAAI,IAAI,GAAG,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,4CAA4C;AAC5C,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,MAAM,QAAQ,GAAG;;;;;;;;;;;;0BAYS,cAAc;CACvC,CAAC;AAEF,wDAAwD;AACxD,MAAM,QAAQ,GAAG;;;;;;;;;;;;;;CAchB,CAAC;AAEF,gFAAgF;AAChF,MAAM,CAAC,MAAM,eAAe,GAAG;;;;;CAK9B,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG;;;;;CAKpC,CAAC;AAEF,MAAM,UAAU,GAAG;;;CAGlB,CAAC;AAEF,MAAM,UAAU,GAAG,yCAAyC,CAAC;AAE7D,MAAM,CAAC,MAAM,OAAO,GAAG,8BAA8B,CAAC;AACtD,MAAM,CAAC,MAAM,SAAS,GAAG,4CAA4C,CAAC;AACtE,MAAM,CAAC,MAAM,YAAY,GAAG,iDAAiD,CAAC;AAE9E;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,qBAAqB,CACnC,KAA4D,EAC5D,OAAgB,EAChB,KAAkB;IAElB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,aAAa,CAAC;IAE3C,iFAAiF;IACjF,IAAI,OAAO,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxE,IAAI,CAAC;YACH,oFAAoF;YACpF,0FAA0F;YAC1F,0FAA0F;YAC1F,oFAAoF;YACpF,sEAAsE;YACtE,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;iBAClC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;gBACb,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACvB,OAAO,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,GAAG,GAAG,CAAC,CAAC;YAC/C,CAAC,CAAC;iBACD,IAAI,CAAC,MAAM,CAAC,CAAC;YAChB,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,IAAI,IAAW,CAAC;gBAChB,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;oBAChC,IAAI,GAAG,KAAK,CAAC,QAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;gBACtD,CAAC;qBAAM,CAAC;oBACN,IAAI,GAAG,KAAK,CAAC,GAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAClC,CAAC;gBACD,uEAAuE;gBACvE,yEAAyE;gBACzE,uEAAuE;gBACvE,sEAAsE;gBACtE,MAAM,MAAM,GAAG,IAAI;qBAChB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,KAAe,EAAE,CAAC,CAAC;qBACvE,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI;oBACf,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK;oBAC/B,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CACvD,CAAC;gBACJ,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,6EAA6E;QAC/E,CAAC;IACH,CAAC;IAED,2BAA2B;IAC3B,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,IAAI,IAAW,CAAC;IAChB,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAChC,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;IACzB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,OAAO;SACnB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;SACpE,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS;QACzB,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK;QAC/B,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CACvD,CAAC;IACJ,iGAAiG;IACjG,iGAAiG;IACjG,kGAAkG;IAClG,2EAA2E;IAC3E,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAC3F,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AAC/D,CAAC;AAQD;;;;;GAKG;AACH,MAAM,OAAO,aAAa;IACf,IAAI,GAAG,QAAQ,CAAC;IAER,EAAE,CAAM,CAAC,mCAAmC;IAC5C,UAAU,CAAM;IAChB,UAAU,CAAM;IAChB,OAAO,CAAM;IACb,SAAS,CAAM;IACf,WAAW,CAAM;IACjB,aAAa,CAAkB;IAC/B,kBAAkB,CAAkB;IACpC,OAAO,CAAU;IAElC,YAAY,EAAO;QACjB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElB,yDAAyD;QACzD,IAAI,KAAK,GAAG,KAAK,CAAC;QAClB,IAAI,CAAC;YACH,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClB,oDAAoD;YACpD,EAAE,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;YACjE,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;YACjD,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC;YAC5D,KAAK,GAAG,IAAI,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,sDAAsD;QACxD,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QAErB,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACzC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAED,4DAA4D;IAC5D,MAAM,CAAC,IAAI,CAAC,QAAgB;QAC1B,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,gEAAgE;QAChE,iEAAiE;QACjE,MAAM,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC3C,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAClC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAChC,EAAE,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QAClC,OAAO,IAAI,aAAa,CAAC,EAAE,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,YAAY,CAAC,QAAgB,EAAE,IAA0B;QAC9D,MAAM,MAAM,GAAG,kBAAkB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC;YACH,OAAO,IAAI,mBAAmB,CAAC,MAAM,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,2FAA2F;YAC3F,mFAAmF;YACnF,0FAA0F;YAC1F,4CAA4C;YAC5C,IAAI,CAAC;gBACH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YACpB,CAAC;YAAC,MAAM,CAAC;gBACP,2FAA2F;YAC7F,CAAC;YACD,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,GAAG,CAAC,MAAoB;QACtB,IAAI,CAAC,UAAU,CAAC,GAAG,CACjB,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CACzD,CAAC;QACF,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,KAAkB;QACtB,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAChD,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,KAAkB;QAC1B,OAAO,qBAAqB,CAC1B,EAAE,GAAG,EAAE,IAAI,CAAC,aAAa,EAAE,QAAQ,EAAE,IAAI,CAAC,kBAAkB,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,EAC5G,IAAI,CAAC,OAAO,EACZ,KAAK,CACN,CAAC;IACJ,CAAC;IAED,GAAG;QACD,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,6CAA6C;IAC7C,OAAO;QACL,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,CAAC,EAAU;QACf,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACpB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,sIAAsI;IACtI,UAAU,CAAC,EAAU;QACnB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK;QACH,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;IACnD,CAAC;IAED,oEAAoE;IACpE,OAAO,CAAC,OAAgC;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,KAA8B,EAAE,EAAE;YACxE,KAAK,MAAM,MAAM,IAAI,KAAK,EAAE,CAAC;gBAC3B,IAAI,CAAC,UAAU,CAAC,GAAG,CACjB,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CACzD,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QACH,UAAU,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;IAED,qCAAqC;IACrC,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF;AAED,kDAAkD;AAClD,MAAM,UAAU,WAAW,CAAC,GAAQ;IAClC,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,OAAO,EAAE,GAAG,CAAC,QAAQ;QACrB,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS;KAC9D,CAAC;AACJ,CAAC"}
+{"version":3,"file":"sqlite-backend.js","sourceRoot":"","sources":["../src/sqlite-backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE/E,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAE7D,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAE/C,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB;;;;;;;;;;;;GAYG;AACH,SAAS,MAAM,CAAC,KAAa;IAC3B,MAAM,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACvB,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtD,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACvD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;mGACmG;AACnG,SAAS,WAAW,CAAC,MAAoB,EAAE,KAAwB;IACjE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,IAAI,IAAI,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QAChD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,IAAI,KAAK,GAAG,KAAK,CAAC;YAClB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBAAC,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;oBAAC,KAAK,GAAG,IAAI,CAAC;oBAAC,MAAM;gBAAC,CAAC;YAAC,CAAC;YAC5E,IAAI,KAAK;gBAAE,IAAI,IAAI,GAAG,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,4CAA4C;AAC5C,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,MAAM,QAAQ,GAAG;;;;;;;;;;;;0BAYS,cAAc;CACvC,CAAC;AAEF,wDAAwD;AACxD,MAAM,QAAQ,GAAG;;;;;;;;;;;;;;CAchB,CAAC;AAEF,gFAAgF;AAChF,MAAM,CAAC,MAAM,eAAe,GAAG;;;;;CAK9B,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG;;;;;CAKpC,CAAC;AAEF,MAAM,UAAU,GAAG;;;CAGlB,CAAC;AAEF,MAAM,UAAU,GAAG,yCAAyC,CAAC;AAE7D,MAAM,CAAC,MAAM,OAAO,GAAG,8BAA8B,CAAC;AACtD,MAAM,CAAC,MAAM,SAAS,GAAG,4CAA4C,CAAC;AACtE,MAAM,CAAC,MAAM,YAAY,GAAG,iDAAiD,CAAC;AAE9E;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,qBAAqB,CACnC,KAA4D,EAC5D,OAAgB,EAChB,KAAkB;IAElB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,aAAa,CAAC;IAE3C,iFAAiF;IACjF,IAAI,OAAO,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxE,IAAI,CAAC;YACH,oFAAoF;YACpF,0FAA0F;YAC1F,0FAA0F;YAC1F,oFAAoF;YACpF,sEAAsE;YACtE,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;iBAClC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;gBACb,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACvB,OAAO,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,GAAG,GAAG,CAAC,CAAC;YAC/C,CAAC,CAAC;iBACD,IAAI,CAAC,MAAM,CAAC,CAAC;YAChB,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,IAAI,IAAW,CAAC;gBAChB,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;oBAChC,IAAI,GAAG,KAAK,CAAC,QAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;gBACtD,CAAC;qBAAM,CAAC;oBACN,IAAI,GAAG,KAAK,CAAC,GAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAClC,CAAC;gBACD,uEAAuE;gBACvE,yEAAyE;gBACzE,uEAAuE;gBACvE,sEAAsE;gBACtE,MAAM,MAAM,GAAG,IAAI;qBAChB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,KAAe,EAAE,CAAC,CAAC;qBACvE,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI;oBACf,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK;oBAC/B,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CACvD,CAAC;gBACJ,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,6EAA6E;QAC/E,CAAC;IACH,CAAC;IAED,2BAA2B;IAC3B,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC;IACxC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAK,CAAC,CAAC;IAClD,2FAA2F;IAC3F,0FAA0F;IAC1F,+FAA+F;IAC/F,+FAA+F;IAC/F,+FAA+F;IAC/F,+FAA+F;IAC/F,4DAA4D;IAC5D,EAAE;IACF,8FAA8F;IAC9F,2FAA2F;IAC3F,+FAA+F;IAC/F,6FAA6F;IAC7F,IAAI,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,IAAK,CAAC,EAAE,CAAC;QAChD,OAAO,EAAE,CAAC,CAAC,4EAA4E;IACzF,CAAC;IAED,IAAI,IAAW,CAAC;IAChB,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAChC,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;IACzB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,OAAO;SACnB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;SACpE,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS;QACzB,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK;QAC/B,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CACvD,CAAC;IACJ,iGAAiG;IACjG,iGAAiG;IACjG,0FAA0F;IAC1F,kGAAkG;IAClG,6FAA6F;IAC7F,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAC3F,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AAC/D,CAAC;AAQD;;;;;GAKG;AACH,MAAM,OAAO,aAAa;IACf,IAAI,GAAG,QAAQ,CAAC;IAER,EAAE,CAAM,CAAC,mCAAmC;IAC5C,UAAU,CAAM;IAChB,UAAU,CAAM;IAChB,OAAO,CAAM;IACb,SAAS,CAAM;IACf,WAAW,CAAM;IACjB,aAAa,CAAkB;IAC/B,kBAAkB,CAAkB;IACpC,OAAO,CAAU;IAElC,YAAY,EAAO;QACjB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElB,yDAAyD;QACzD,IAAI,KAAK,GAAG,KAAK,CAAC;QAClB,IAAI,CAAC;YACH,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClB,oDAAoD;YACpD,EAAE,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;YACjE,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;YACjD,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC;YAC5D,KAAK,GAAG,IAAI,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,sDAAsD;QACxD,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QAErB,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACzC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAED,4DAA4D;IAC5D,MAAM,CAAC,IAAI,CAAC,QAAgB;QAC1B,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,gEAAgE;QAChE,iEAAiE;QACjE,MAAM,QAAQ,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC3C,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAClC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAChC,EAAE,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QAClC,OAAO,IAAI,aAAa,CAAC,EAAE,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,YAAY,CAAC,QAAgB,EAAE,IAA0B;QAC9D,MAAM,MAAM,GAAG,kBAAkB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC;YACH,OAAO,IAAI,mBAAmB,CAAC,MAAM,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,2FAA2F;YAC3F,mFAAmF;YACnF,0FAA0F;YAC1F,4CAA4C;YAC5C,IAAI,CAAC;gBACH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YACpB,CAAC;YAAC,MAAM,CAAC;gBACP,2FAA2F;YAC7F,CAAC;YACD,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,GAAG,CAAC,MAAoB;QACtB,IAAI,CAAC,UAAU,CAAC,GAAG,CACjB,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CACzD,CAAC;QACF,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,KAAkB;QACtB,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAChD,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,KAAkB;QAC1B,OAAO,qBAAqB,CAC1B,EAAE,GAAG,EAAE,IAAI,CAAC,aAAa,EAAE,QAAQ,EAAE,IAAI,CAAC,kBAAkB,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,EAC5G,IAAI,CAAC,OAAO,EACZ,KAAK,CACN,CAAC;IACJ,CAAC;IAED,GAAG;QACD,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,6CAA6C;IAC7C,OAAO;QACL,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,CAAC,EAAU;QACf,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACpB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,sIAAsI;IACtI,UAAU,CAAC,EAAU;QACnB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK;QACH,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;IACnD,CAAC;IAED,oEAAoE;IACpE,OAAO,CAAC,OAAgC;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,KAA8B,EAAE,EAAE;YACxE,KAAK,MAAM,MAAM,IAAI,KAAK,EAAE,CAAC;gBAC3B,IAAI,CAAC,UAAU,CAAC,GAAG,CACjB,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CACzD,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QACH,UAAU,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;IAED,qCAAqC;IACrC,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF;AAED,kDAAkD;AAClD,MAAM,UAAU,WAAW,CAAC,GAAQ;IAClC,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,OAAO,EAAE,GAAG,CAAC,QAAQ;QACrB,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS;KAC9D,CAAC;AACJ,CAAC"}

package/dist/tokenize.d.ts

@@ -0,0 +1,85 @@
+/**
+ * The shared lexical tokenizer — `sqlite-backend.ts` and `json-backend.ts` both import this
+ * instead of carrying their own copy (feature recall-short-terms, FR-1). A forked copy is exactly
+ * the class of bug this package has hit twice already: the two backends' tokenizers drifted once
+ * before (the Cyrillic-alphabet fix, 2026-08-21) and a reader whose ranking diverges from the
+ * writer's silently regresses recall.
+ *
+ * @packageDocumentation
+ */
+/**
+ * Split text into lowercase word tokens.
+ *
+ * The class is `\p{L}\p{N}`, not `a-z0-9` — see the 2026-08-21 Cyrillic-alphabet fix: an ASCII-only
+ * class made every non-Latin letter a SEPARATOR, so a Cyrillic query produced zero tokens.
+ *
+ * Until 2026-09-15 a token also had to be MORE THAN ONE code point (`[...token].length > 1`), so a
+ * query like `x`, `C`, or Cyrillic `на`'s single-letter cousin `и` produced ZERO usable terms and
+ * fell into the "nothing to search by" branch — which used to silently return the WHOLE STORE
+ * (backlog 529c31ab, MEASURED: a record whose entire text is `x` was never found by querying `x`,
+ * because the query never reached the ranker at all). Single- and two-character alphanumeric
+ * entities are real in this domain — `C`, `Go`, `R`, `ID`, `db`, ADR letter variants — so the floor
+ * is gone: ANY non-empty run of `\p{L}\p{N}` characters is now a token.
+ *
+ * This introduces no new noise on its own: each backend's `stemOf` already refuses to stem
+ * anything under 5 code points, so a 1-2 character term is only ever matched by EXACT token
+ * equality, never a prefix (FR-2) — the widened floor and the no-stemming-below-5 rule are
+ * independent decisions that happen to compose safely.
+ *
+ * Filtering on `token.length > 0` (a JS UTF-16-unit length) rather than counting code points is
+ * deliberately fine here, unlike the old `> 1` floor: emptiness does not depend on surrogate
+ * pairs — a non-empty string always has `.length >= 1` and an empty one always has `.length === 0`,
+ * on BOTH measures. The code-point-vs-UTF-16-unit distinction only mattered for a POSITIVE
+ * threshold like the old `> 1` (where a lone astral letter, one code point but two UTF-16 units,
+ * could slip past it); it is moot once the floor is simply "not empty".
+ *
+ * `.normalize('NFC')` runs BEFORE lowercasing (fix-round 1, Codex HIGH-2, MEASURED — not merely
+ * argued). Precomposed `café` (4 code points, the `é` is U+00E9) and decomposed `café` (5 code
+ * points, `e` + combining acute U+0301) are the SAME text to a reader and to shipping FTS5 — a
+ * `better-sqlite3` probe (`node -e` against an in-memory `fts5` table) confirmed both forms and
+ * the bare ASCII `cafe` all MATCH each other under FTS5's default `unicode61` tokenizer. Without
+ * NFC, this tokenizer disagreed: the combining mark U+0301 is category `\p{M}` (Mark), not
+ * `\p{L}`, so it fell into the SEPARATOR class above — decomposed `café` tokenized to `cafe`
+ * (accent silently dropped) while precomposed `café` tokenized to the distinct string `café`
+ * (accent kept), so the keyword-overlap path treated two representations of ONE word as two
+ * DIFFERENT words. NFC first collapses both representations to the single precomposed form
+ * before the letter/number split ever runs, so both now tokenize to the identical `café`.
+ *
+ * **Diacritics are folded for LATIN script only — exactly the shipping engine's rule (Codex r2/r3,
+ * lead fix, MEASURED 2026-09-15 with a `better-sqlite3` probe against an in-memory `fts5` table):**
+ * FTS5's default `unicode61` tokenizer (`remove_diacritics=1`) strips diacritics from Latin
+ * characters (`cafe` MATCHes `café`, `ano` MATCHes `año`) but leaves other scripts alone
+ * (`й` does NOT match `и`, `ё` does NOT match `е`). The keyword-overlap path (JSON backend, sqlite
+ * with FTS5 off) must answer IDENTICALLY — parity between the two backends of one store is the
+ * requirement — so this tokenizer does the same: NFD, strip combining marks that follow a LATIN
+ * base letter only, NFC back (so `й` = `и` + U+0306 recomposes and stays a distinct letter). A
+ * global `\p{M}` strip (the first lead attempt) would have folded Cyrillic `й`→`и`, `ё`→`е` where
+ * FTS5 does not — a divergence in the script that carries 63% of real recall traffic. Named
+ * limit, inherited from the engine: Latin words that differ only by an accent (`año`/`ano`,
+ * `côté`/`cote`) conflate in BOTH backends, as FTS5 already conflated them.
+ * Edge measured (Codex r4): `remove_diacritics=1` leaves Latin letters with TWO combining marks
+ * alone (`ộ`, `ố`, `ấ`, `ǘ` — `MATCH 'o'` does not return the `ộ` row), so the fold applies only
+ * to a single mark; `remove_diacritics=2` semantics (fold everything) are deliberately NOT used.
+ */
+export declare function tokenize(text: string): string[];
+/**
+ * Does `text` contain at least one searchable token?
+ *
+ * FR-3: the "nothing to search by" branch in each backend's keyword path must fire ONLY when this
+ * is `false` — i.e. only when tokenization finds literally nothing (pure punctuation, whitespace,
+ * or an empty string), never merely because every token happened to be short. That narrower branch
+ * used to answer with a silent full-store dump; it no longer does (see each backend's `querySync`),
+ * and this function is the one place that decides which queries land in it — named so the decision
+ * is discoverable, not reimplemented ad hoc at each call site.
+ */
+export declare function hasSearchableTerms(text: string): boolean;
+/**
+ * The named reason a query's keyword-overlap path fell into the "no searchable terms" branch, or
+ * `undefined` when it didn't. Fix-round 1 (Codex HIGH-1): both backends used to reimplement the
+ * `hasSearchableTerms(text) === false` check as a hand-rolled `terms.length === 0`, and the reason
+ * string it corresponds to lived nowhere a caller — including `dz recall`'s CLI printer — could
+ * name without re-deriving it. This is the single source of truth for that name: a backend or a
+ * printer that wants to know WHY an empty result came back calls this instead of re-tokenizing.
+ */
+export declare function noSearchableTermsReason(text: string): 'no-searchable-terms' | undefined;
+//# sourceMappingURL=tokenize.d.ts.map

package/dist/tokenize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../src/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqDG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAa/C;AAED;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAExD;AAED;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,qBAAqB,GAAG,SAAS,CAEvF"}

package/dist/tokenize.js

@@ -0,0 +1,102 @@
+/**
+ * The shared lexical tokenizer — `sqlite-backend.ts` and `json-backend.ts` both import this
+ * instead of carrying their own copy (feature recall-short-terms, FR-1). A forked copy is exactly
+ * the class of bug this package has hit twice already: the two backends' tokenizers drifted once
+ * before (the Cyrillic-alphabet fix, 2026-08-21) and a reader whose ranking diverges from the
+ * writer's silently regresses recall.
+ *
+ * @packageDocumentation
+ */
+/**
+ * Split text into lowercase word tokens.
+ *
+ * The class is `\p{L}\p{N}`, not `a-z0-9` — see the 2026-08-21 Cyrillic-alphabet fix: an ASCII-only
+ * class made every non-Latin letter a SEPARATOR, so a Cyrillic query produced zero tokens.
+ *
+ * Until 2026-09-15 a token also had to be MORE THAN ONE code point (`[...token].length > 1`), so a
+ * query like `x`, `C`, or Cyrillic `на`'s single-letter cousin `и` produced ZERO usable terms and
+ * fell into the "nothing to search by" branch — which used to silently return the WHOLE STORE
+ * (backlog 529c31ab, MEASURED: a record whose entire text is `x` was never found by querying `x`,
+ * because the query never reached the ranker at all). Single- and two-character alphanumeric
+ * entities are real in this domain — `C`, `Go`, `R`, `ID`, `db`, ADR letter variants — so the floor
+ * is gone: ANY non-empty run of `\p{L}\p{N}` characters is now a token.
+ *
+ * This introduces no new noise on its own: each backend's `stemOf` already refuses to stem
+ * anything under 5 code points, so a 1-2 character term is only ever matched by EXACT token
+ * equality, never a prefix (FR-2) — the widened floor and the no-stemming-below-5 rule are
+ * independent decisions that happen to compose safely.
+ *
+ * Filtering on `token.length > 0` (a JS UTF-16-unit length) rather than counting code points is
+ * deliberately fine here, unlike the old `> 1` floor: emptiness does not depend on surrogate
+ * pairs — a non-empty string always has `.length >= 1` and an empty one always has `.length === 0`,
+ * on BOTH measures. The code-point-vs-UTF-16-unit distinction only mattered for a POSITIVE
+ * threshold like the old `> 1` (where a lone astral letter, one code point but two UTF-16 units,
+ * could slip past it); it is moot once the floor is simply "not empty".
+ *
+ * `.normalize('NFC')` runs BEFORE lowercasing (fix-round 1, Codex HIGH-2, MEASURED — not merely
+ * argued). Precomposed `café` (4 code points, the `é` is U+00E9) and decomposed `café` (5 code
+ * points, `e` + combining acute U+0301) are the SAME text to a reader and to shipping FTS5 — a
+ * `better-sqlite3` probe (`node -e` against an in-memory `fts5` table) confirmed both forms and
+ * the bare ASCII `cafe` all MATCH each other under FTS5's default `unicode61` tokenizer. Without
+ * NFC, this tokenizer disagreed: the combining mark U+0301 is category `\p{M}` (Mark), not
+ * `\p{L}`, so it fell into the SEPARATOR class above — decomposed `café` tokenized to `cafe`
+ * (accent silently dropped) while precomposed `café` tokenized to the distinct string `café`
+ * (accent kept), so the keyword-overlap path treated two representations of ONE word as two
+ * DIFFERENT words. NFC first collapses both representations to the single precomposed form
+ * before the letter/number split ever runs, so both now tokenize to the identical `café`.
+ *
+ * **Diacritics are folded for LATIN script only — exactly the shipping engine's rule (Codex r2/r3,
+ * lead fix, MEASURED 2026-09-15 with a `better-sqlite3` probe against an in-memory `fts5` table):**
+ * FTS5's default `unicode61` tokenizer (`remove_diacritics=1`) strips diacritics from Latin
+ * characters (`cafe` MATCHes `café`, `ano` MATCHes `año`) but leaves other scripts alone
+ * (`й` does NOT match `и`, `ё` does NOT match `е`). The keyword-overlap path (JSON backend, sqlite
+ * with FTS5 off) must answer IDENTICALLY — parity between the two backends of one store is the
+ * requirement — so this tokenizer does the same: NFD, strip combining marks that follow a LATIN
+ * base letter only, NFC back (so `й` = `и` + U+0306 recomposes and stays a distinct letter). A
+ * global `\p{M}` strip (the first lead attempt) would have folded Cyrillic `й`→`и`, `ё`→`е` where
+ * FTS5 does not — a divergence in the script that carries 63% of real recall traffic. Named
+ * limit, inherited from the engine: Latin words that differ only by an accent (`año`/`ano`,
+ * `côté`/`cote`) conflate in BOTH backends, as FTS5 already conflated them.
+ * Edge measured (Codex r4): `remove_diacritics=1` leaves Latin letters with TWO combining marks
+ * alone (`ộ`, `ố`, `ấ`, `ǘ` — `MATCH 'o'` does not return the `ộ` row), so the fold applies only
+ * to a single mark; `remove_diacritics=2` semantics (fold everything) are deliberately NOT used.
+ */
+export function tokenize(text) {
+    return text
+        // Latin-only diacritic fold = FTS5 unicode61 remove_diacritics=1 (see the doc comment):
+        // NFD → drop marks after a Latin base letter → NFC (non-Latin letters recompose untouched).
+        .normalize('NFD')
+        // Codex r4 (lead, MEASURED 2026-09-15 on fts5 in-memory): remove_diacritics=1 folds a Latin
+        // letter carrying ONE mark (é→e, ü→u, ñ→n) but NOT one carrying two (Vietnamese ộ/ố/ấ, ǘ stay
+        // distinct) — so only a SINGLE mark after a Latin base is dropped; multi-mark letters recompose.
+        .replace(/(\p{Script=Latin})\p{M}(?!\p{M})/gu, '$1')
+        .normalize('NFC')
+        .toLowerCase()
+        .split(/[^\p{L}\p{N}]+/u)
+        .filter((token) => token.length > 0);
+}
+/**
+ * Does `text` contain at least one searchable token?
+ *
+ * FR-3: the "nothing to search by" branch in each backend's keyword path must fire ONLY when this
+ * is `false` — i.e. only when tokenization finds literally nothing (pure punctuation, whitespace,
+ * or an empty string), never merely because every token happened to be short. That narrower branch
+ * used to answer with a silent full-store dump; it no longer does (see each backend's `querySync`),
+ * and this function is the one place that decides which queries land in it — named so the decision
+ * is discoverable, not reimplemented ad hoc at each call site.
+ */
+export function hasSearchableTerms(text) {
+    return tokenize(text).length > 0;
+}
+/**
+ * The named reason a query's keyword-overlap path fell into the "no searchable terms" branch, or
+ * `undefined` when it didn't. Fix-round 1 (Codex HIGH-1): both backends used to reimplement the
+ * `hasSearchableTerms(text) === false` check as a hand-rolled `terms.length === 0`, and the reason
+ * string it corresponds to lived nowhere a caller — including `dz recall`'s CLI printer — could
+ * name without re-deriving it. This is the single source of truth for that name: a backend or a
+ * printer that wants to know WHY an empty result came back calls this instead of re-tokenizing.
+ */
+export function noSearchableTermsReason(text) {
+    return hasSearchableTerms(text) ? undefined : 'no-searchable-terms';
+}
+//# sourceMappingURL=tokenize.js.map

package/dist/tokenize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenize.js","sourceRoot":"","sources":["../src/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqDG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,OAAO,IAAI;QACT,wFAAwF;QACxF,4FAA4F;SAC3F,SAAS,CAAC,KAAK,CAAC;QACjB,4FAA4F;QAC5F,8FAA8F;QAC9F,iGAAiG;SAChG,OAAO,CAAC,oCAAoC,EAAE,IAAI,CAAC;SACnD,SAAS,CAAC,KAAK,CAAC;SAChB,WAAW,EAAE;SACb,KAAK,CAAC,iBAAiB,CAAC;SACxB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACzC,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;AACnC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAAY;IAClD,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,qBAAqB,CAAC;AACtE,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dzhechkov/memory",
-  "version": "0.2.21",
+  "version": "0.2.22",
   "description": "Harness memory layer - backend cascade, Reflexion feedback, and the host-memory bridge.",
   "type": "module",
   "license": "MIT",

package/sbom.json

@@ -25,7 +25,7 @@
       "hashes": [
         {
           "alg": "SHA-256",
-          "content": "6ae2da1d09bbe9ce6e3742079d659690f5c505a39eb187806c5f5be4c2d3a7d4"
+          "content": "4560bc1052213616df71c0eca90758f976eaf9683cfda760cfa67d701e57604b"
         }
       ]
     },
@@ -195,7 +195,7 @@
       "hashes": [
         {
           "alg": "SHA-256",
-          "content": "c4e8be344baf02933c8e453bdba9ec3f95062d28672c8d0dabdd8035ef39e58e"
+          "content": "e73b2a1fc8f83272323e24cc117943712c929c6e2daddaf407c2ed71ab7ca1b8"
         }
       ]
     },
@@ -205,7 +205,7 @@
       "hashes": [
         {
           "alg": "SHA-256",
-          "content": "9576e873ac4003f6b217e0eff04f2cd5f8633397e004164abe7f10f9d7886511"
+          "content": "b84d8f5995bd3a1285a69361a638af35eba6c64c9b4945268da942894d48aff6"
         }
       ]
     },
@@ -215,7 +215,7 @@
       "hashes": [
         {
           "alg": "SHA-256",
-          "content": "b6f72385ca8f3fbde2a5c11cf0deb98f58ea9836d216377e6585dc22b4b629b5"
+          "content": "be8ab9da014b2c9bf7472f342cebe2e9ae81d0867b3efbfb35f006ff811973c3"
         }
       ]
     },
@@ -225,7 +225,7 @@
       "hashes": [
         {
           "alg": "SHA-256",
-          "content": "92e221dc762befc999d234a173b09b2e5c09c7e927518fccdd8bbebe15923712"
+          "content": "1c78cb041d2f482319f7a691330b313ed0235441835ec9ae78f8165a27eb3e5d"
         }
       ]
     },
@@ -245,7 +245,7 @@
       "hashes": [
         {
           "alg": "SHA-256",
-          "content": "95adb5287a060d417ed83d2364d8f63b973790e9885dc29ea7182132f4231b7c"
+          "content": "158819665930ed2cd3f86f4fa257b8b7bfa69d7638a69cfcc6d947be2c7d014c"
         }
       ]
     },
@@ -255,7 +255,7 @@
       "hashes": [
         {
           "alg": "SHA-256",
-          "content": "d53fcda53b7a1d24ccff4c3e1d98794985a9de01c97f6c357eec91ad64c2a341"
+          "content": "eb621bac0d3447892064fefb8c9e0f118cef1bf3ca6da538177afe4614320bb9"
         }
       ]
     },
@@ -265,7 +265,7 @@
       "hashes": [
         {
           "alg": "SHA-256",
-          "content": "7e0a073bfc25ae6fea53f29ad518319c31cdac070a1215bfb8a7584cb7ce9c33"
+          "content": "662b463dd30f390aaa60527bca2df3b42894bb70de3162a55c24aee1718efb65"
         }
       ]
     },
@@ -325,7 +325,7 @@
       "hashes": [
         {
           "alg": "SHA-256",
-          "content": "0df025e4853d8420efc87d2b7aa8ea9f36e5b9a0a0be4351a7eb7614caf62b4c"
+          "content": "62b60365ba990ae1be666b5190e4f01d4a60989ddc62cea687e3823a4f0bb2cc"
         }
       ]
     },
@@ -335,7 +335,7 @@
       "hashes": [
         {
           "alg": "SHA-256",
-          "content": "4f2a5c61fc27a8aa39f1adfb06215af80bbde0236f1f2a7f73ab65d3ab3f3652"
+          "content": "664eb414c5438dda67b5e6b2e9c3281a20c38c4b85a1e9ee48b004a1e7c08d24"
         }
       ]
     },
@@ -345,7 +345,7 @@
       "hashes": [
         {
           "alg": "SHA-256",
-          "content": "4106d2a7cfa2ad50ea5497dab8ad18d2102019f05e5a3aca2469ab2ec325ce48"
+          "content": "12ba0851550864589cb8fad02950cfc7ff468bdf0e87b429ba4bfc6a7294e28d"
         }
       ]
     },
@@ -429,6 +429,46 @@
         }
       ]
     },
+    {
+      "type": "file",
+      "name": "dist/tokenize.d.ts",
+      "hashes": [
+        {
+          "alg": "SHA-256",
+          "content": "5c1fcc80c8d6836c3b80dba6de50e98b06c70bc2d15c46b1392bb305af6e5e57"
+        }
+      ]
+    },
+    {
+      "type": "file",
+      "name": "dist/tokenize.d.ts.map",
+      "hashes": [
+        {
+          "alg": "SHA-256",
+          "content": "ec47c8a2687a0f598c0e81a5871e9a011cf686880c8a34915af31a09701faddc"
+        }
+      ]
+    },
+    {
+      "type": "file",
+      "name": "dist/tokenize.js",
+      "hashes": [
+        {
+          "alg": "SHA-256",
+          "content": "bc1b9368d3e375bed2293afead3cc5d23409f8f83fae2e8f9eb790542ab02635"
+        }
+      ]
+    },
+    {
+      "type": "file",
+      "name": "dist/tokenize.js.map",
+      "hashes": [
+        {
+          "alg": "SHA-256",
+          "content": "8fd9e4236d32873ee9b9ea4b3d44f9e4dfffa09071dc3831e26956f88f662fcf"
+        }
+      ]
+    },
     {
       "type": "file",
       "name": "package.json",
@@ -439,7 +479,7 @@
         },
         {
           "name": "dz:canonical-json-sha256-v2",
-          "value": "cd706c6ae1def0fd6e4bdb9c84e0d95239a24bda69eb8f70b47377189c26fc14"
+          "value": "ea0e5da14d9fdddbb6a5a6278610f0bcda3b98f1cae6287def6e5565f01a3ab6"
         }
       ]
     },
@@ -489,7 +529,7 @@
       "hashes": [
         {
           "alg": "SHA-256",
-          "content": "9152ce2677f8b209bb26a90cf15b1f97dccd7e5fcdcc95e7cf417a698798fc56"
+          "content": "4ac9affbf4fbe220a7a23c08ce48a1d9ffc02c1ddc4774fd41abbeccd33d8993"
         }
       ]
     },
@@ -499,7 +539,7 @@
       "hashes": [
         {
           "alg": "SHA-256",
-          "content": "ba881b55f48dabe7d95ce87ee1c840180850899f551173cd5940e54a8d6c6cf6"
+          "content": "37b2e031e8dc9437b55b3b57ad83e2ebc962335d16c496834a7c4aed9a1770cc"
         }
       ]
     },
@@ -519,7 +559,7 @@
       "hashes": [
         {
           "alg": "SHA-256",
-          "content": "08b618bf7fd351120aac5c4140ae84c2d207c45d708dc42a98f175a99938a11c"
+          "content": "d91cff21f894ae44483b06807fdbbe4958bee00292c0cd85469e2c5f3374c055"
         }
       ]
     },
@@ -542,6 +582,16 @@
           "content": "c973b0eeae5567af4c519c178ccb48d496fa1cf2ad226363b26ecc2e7d11dc45"
         }
       ]
+    },
+    {
+      "type": "file",
+      "name": "src/tokenize.ts",
+      "hashes": [
+        {
+          "alg": "SHA-256",
+          "content": "16b59783d4cf4023d810584564063b195e1cae83a4f4c4fd9d11751ad3d1e7be"
+        }
+      ]
     }
   ]
 }

package/src/index.ts

@@ -11,6 +11,7 @@ export const MEMORY_VERSION: string =
   (createRequire(import.meta.url)('../package.json') as { version: string }).version;
 
 export type { MemoryBackend, MemoryQuery, MemoryRecord } from './backend.js';
+export { tokenize, hasSearchableTerms, noSearchableTermsReason } from './tokenize.js';
 export { JsonFileBackend } from './json-backend.js';
 export type { JsonFileBackendOptions } from './json-backend.js';
 export { selectBackend } from './cascade.js';

package/src/json-backend.ts

@@ -12,33 +12,10 @@ import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync
 import { dirname } from 'node:path';
 
 import type { MemoryBackend, MemoryQuery, MemoryRecord } from './backend.js';
+import { tokenize, hasSearchableTerms } from './tokenize.js';
 
 const DEFAULT_LIMIT = 20;
 
-/**
- * Split text into lowercase word tokens of length > 1.
- *
- * The class is `\p{L}\p{N}`, not `a-z0-9`. Until 2026-08-21 it was ASCII-only, so every non-Latin
- * letter was a SEPARATOR and a Cyrillic query produced ZERO tokens — the FTS5 branch was then skipped
- * entirely, `relevanceOf` returned 0 for every record, and the sort collapsed onto its confidence
- * tie-break. MEASURED on a 267-record clone of the real brain: RU top-1 0/10 against EN 10/10, while
- * 63% of real recall traffic is Cyrillic. The INDEX was never wrong — FTS5's own tokenizer handles
- * Cyrillic — so nothing on disk needed migrating; only the query was being stripped of its terms.
- *
- * `\p{L}` admits letters and `\p{N}` digits; it does NOT admit `"`, `*`, `(` or any other FTS5
- * operator, which is what keeps the joined terms safe to interpolate into a MATCH expression.
- */
-function tokenize(text: string): string[] {
-  return text
-    .toLowerCase()
-    .split(/[^\p{L}\p{N}]+/u)
-    // Count CODE POINTS, not UTF-16 units. `token.length` counts units, so a single astral letter
-    // (`𐐀`, one character, two units) would slip past a floor meant to reject one-character words —
-    // an accidental threshold change smuggled in by the alphabet change (cross-family review,
-    // 2026-08-21). The promise was "the alphabet, not the thresholds"; this keeps it.
-    .filter((token) => [...token].length > 1);
-}
-
 /**
  * Crude prefix-stem for morphology-bearing languages — feature recall-ru-morphology.
  *
@@ -125,7 +102,22 @@ export class JsonFileBackend implements MemoryBackend {
   /** Synchronous {@link JsonFileBackend.query} — same ranking, no Promise. */
   querySync(query: MemoryQuery): MemoryRecord[] {
     const limit = query.limit ?? DEFAULT_LIMIT;
-    const terms = query.text !== undefined ? tokenize(query.text) : [];
+    const noText = query.text === undefined;
+    const terms = noText ? [] : tokenize(query.text!);
+    // FR-3 (recall-short-terms): text was SUPPLIED but tokenized to nothing (pure
+    // punctuation/whitespace/empty). Until 2026-09-15 this silently returned the WHOLE STORE,
+    // indistinguishable from "here is everything you asked for". Honest behavior for an
+    // unsatisfiable search is EMPTY, same as any other failed search (ADR-001's own
+    // "no-match-means-no-results"). A query with NO text field at all is a different intent —
+    // "browse everything", never attempted as a search — untouched below.
+    //
+    // Fix-round 1 (Codex HIGH-1): this used to reimplement the decision as `terms.length === 0` —
+    // a hand-rolled copy of `hasSearchableTerms`'s own logic, forkable exactly the way the old
+    // per-backend `tokenize()` was. Calling the shared helper directly closes that gap; `terms`
+    // is still computed above for `relevanceOf` below, but the BRANCH decision is never re-derived.
+    if (!noText && !hasSearchableTerms(query.text!)) {
+      return []; // reason: 'no-searchable-terms' — see tokenize.ts `noSearchableTermsReason`
+    }
     let candidates = [...this.records.values()];
     if (query.skillId !== undefined) {
       candidates = candidates.filter((record) => record.skillId === query.skillId);
@@ -140,9 +132,10 @@ export class JsonFileBackend implements MemoryBackend {
       );
     // ASKING and finding nothing returns nothing. Without this the keyword path RANKS by overlap and
     // never EXCLUDES, so every query returned the whole store reordered — MEASURED on two records:
-    // `zebrafish` (matches neither) AND `hello` (matches one) both returned both. With no usable
-    // terms there was nothing to match on, and returning the store ranked by confidence stays the
-    // right answer; that distinction is the whole decision (ADR-001).
+    // `zebrafish` (matches neither) AND `hello` (matches one) both returned both. `noText` is the
+    // ONLY way `terms.length === 0` reaches this point now (the provided-but-unsearchable case
+    // returned above) — nothing was ever expressible to filter on, so the store still comes back
+    // ranked by confidence; that distinction is the whole decision (ADR-001).
     const filtered = terms.length > 0 ? ranked.filter((entry) => entry.relevance > 0) : ranked;
     return filtered.slice(0, limit).map((entry) => entry.record);
   }

package/src/sqlite-backend.ts

@@ -15,35 +15,12 @@ import { dirname } from 'node:path';
 import type { MemoryBackend, MemoryQuery, MemoryRecord } from './backend.js';
 import { openSqliteReadOnly, SqliteReadOnlyStore } from './sqlite-readonly.js';
 import type { OpenReadOnlyOptions, ReadOnlyStore } from './sqlite-readonly.js';
+import { tokenize, hasSearchableTerms } from './tokenize.js';
 
 const require = createRequire(import.meta.url);
 
 const DEFAULT_LIMIT = 20;
 
-/**
- * Split text into lowercase word tokens of length > 1.
- *
- * The class is `\p{L}\p{N}`, not `a-z0-9`. Until 2026-08-21 it was ASCII-only, so every non-Latin
- * letter was a SEPARATOR and a Cyrillic query produced ZERO tokens — the FTS5 branch was then skipped
- * entirely, `relevanceOf` returned 0 for every record, and the sort collapsed onto its confidence
- * tie-break. MEASURED on a 267-record clone of the real brain: RU top-1 0/10 against EN 10/10, while
- * 63% of real recall traffic is Cyrillic. The INDEX was never wrong — FTS5's own tokenizer handles
- * Cyrillic — so nothing on disk needed migrating; only the query was being stripped of its terms.
- *
- * `\p{L}` admits letters and `\p{N}` digits; it does NOT admit `"`, `*`, `(` or any other FTS5
- * operator, which is what keeps the joined terms safe to interpolate into a MATCH expression.
- */
-function tokenize(text: string): string[] {
-  return text
-    .toLowerCase()
-    .split(/[^\p{L}\p{N}]+/u)
-    // Count CODE POINTS, not UTF-16 units. `token.length` counts units, so a single astral letter
-    // (`𐐀`, one character, two units) would slip past a floor meant to reject one-character words —
-    // an accidental threshold change smuggled in by the alphabet change (cross-family review,
-    // 2026-08-21). The promise was "the alphabet, not the thresholds"; this keeps it.
-    .filter((token) => [...token].length > 1);
-}
-
 /**
  * Crude prefix-stem for morphology-bearing languages — feature recall-ru-morphology.
  *
@@ -204,7 +181,24 @@ export function searchPreparedRecords(
   }
 
   // Keyword overlap fallback
-  const terms = query.text !== undefined ? tokenize(query.text) : [];
+  const noText = query.text === undefined;
+  const terms = noText ? [] : tokenize(query.text!);
+  // FR-3 (recall-short-terms): a query that SUPPLIED text but tokenized to literally nothing
+  // (pure punctuation/whitespace/empty) has nothing to filter on, and until 2026-09-15 that
+  // silently dumped the WHOLE STORE — indistinguishable from "here is everything you asked for".
+  // That is the bug this feature names and fixes: honest behavior for an unsatisfiable search is
+  // EMPTY, the same "no-match-means-no-results" answer any other failed search gets (ADR-001). A
+  // query with NO text field at all is a different intent — "browse everything", never attempted
+  // as a search — and keeps the old whole-store answer below.
+  //
+  // Fix-round 1 (Codex HIGH-1): this used to reimplement the decision as `terms.length === 0` —
+  // a hand-rolled copy of `hasSearchableTerms`'s own logic, forkable exactly the way the old
+  // per-backend `tokenize()` was. Calling the shared helper directly closes that gap; `terms` is
+  // still computed above for `relevanceOf` below, but the BRANCH decision is never re-derived.
+  if (!noText && !hasSearchableTerms(query.text!)) {
+    return []; // reason: 'no-searchable-terms' — see tokenize.ts `noSearchableTermsReason`
+  }
+
   let rows: any[];
   if (query.skillId !== undefined) {
     rows = stmts.bySkill.all(query.skillId);
@@ -223,8 +217,9 @@ export function searchPreparedRecords(
     );
   // The SAME guard the JSON backend applies, so a store's answers never depend on which backend is
   // installed. This is the keyword FALLBACK; the FTS5 path above already returns zero honestly and
-  // is untouched. With no usable terms there was nothing to match on, so the store still comes back
-  // ranked by confidence — that distinction is the whole decision (ADR-001).
+  // is untouched. `noText` is the ONLY way `terms.length === 0` reaches this point now (the
+  // provided-but-unsearchable case returned above) — a genuine "no query at all" browse, so nothing
+  // was ever expressible to filter on and the store comes back ranked by confidence (ADR-001).
   const filtered = terms.length > 0 ? ranked.filter((entry) => entry.relevance > 0) : ranked;
   return filtered.slice(0, limit).map((entry) => entry.record);
 }

package/src/tokenize.ts

@@ -0,0 +1,104 @@
+/**
+ * The shared lexical tokenizer — `sqlite-backend.ts` and `json-backend.ts` both import this
+ * instead of carrying their own copy (feature recall-short-terms, FR-1). A forked copy is exactly
+ * the class of bug this package has hit twice already: the two backends' tokenizers drifted once
+ * before (the Cyrillic-alphabet fix, 2026-08-21) and a reader whose ranking diverges from the
+ * writer's silently regresses recall.
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * Split text into lowercase word tokens.
+ *
+ * The class is `\p{L}\p{N}`, not `a-z0-9` — see the 2026-08-21 Cyrillic-alphabet fix: an ASCII-only
+ * class made every non-Latin letter a SEPARATOR, so a Cyrillic query produced zero tokens.
+ *
+ * Until 2026-09-15 a token also had to be MORE THAN ONE code point (`[...token].length > 1`), so a
+ * query like `x`, `C`, or Cyrillic `на`'s single-letter cousin `и` produced ZERO usable terms and
+ * fell into the "nothing to search by" branch — which used to silently return the WHOLE STORE
+ * (backlog 529c31ab, MEASURED: a record whose entire text is `x` was never found by querying `x`,
+ * because the query never reached the ranker at all). Single- and two-character alphanumeric
+ * entities are real in this domain — `C`, `Go`, `R`, `ID`, `db`, ADR letter variants — so the floor
+ * is gone: ANY non-empty run of `\p{L}\p{N}` characters is now a token.
+ *
+ * This introduces no new noise on its own: each backend's `stemOf` already refuses to stem
+ * anything under 5 code points, so a 1-2 character term is only ever matched by EXACT token
+ * equality, never a prefix (FR-2) — the widened floor and the no-stemming-below-5 rule are
+ * independent decisions that happen to compose safely.
+ *
+ * Filtering on `token.length > 0` (a JS UTF-16-unit length) rather than counting code points is
+ * deliberately fine here, unlike the old `> 1` floor: emptiness does not depend on surrogate
+ * pairs — a non-empty string always has `.length >= 1` and an empty one always has `.length === 0`,
+ * on BOTH measures. The code-point-vs-UTF-16-unit distinction only mattered for a POSITIVE
+ * threshold like the old `> 1` (where a lone astral letter, one code point but two UTF-16 units,
+ * could slip past it); it is moot once the floor is simply "not empty".
+ *
+ * `.normalize('NFC')` runs BEFORE lowercasing (fix-round 1, Codex HIGH-2, MEASURED — not merely
+ * argued). Precomposed `café` (4 code points, the `é` is U+00E9) and decomposed `café` (5 code
+ * points, `e` + combining acute U+0301) are the SAME text to a reader and to shipping FTS5 — a
+ * `better-sqlite3` probe (`node -e` against an in-memory `fts5` table) confirmed both forms and
+ * the bare ASCII `cafe` all MATCH each other under FTS5's default `unicode61` tokenizer. Without
+ * NFC, this tokenizer disagreed: the combining mark U+0301 is category `\p{M}` (Mark), not
+ * `\p{L}`, so it fell into the SEPARATOR class above — decomposed `café` tokenized to `cafe`
+ * (accent silently dropped) while precomposed `café` tokenized to the distinct string `café`
+ * (accent kept), so the keyword-overlap path treated two representations of ONE word as two
+ * DIFFERENT words. NFC first collapses both representations to the single precomposed form
+ * before the letter/number split ever runs, so both now tokenize to the identical `café`.
+ *
+ * **Diacritics are folded for LATIN script only — exactly the shipping engine's rule (Codex r2/r3,
+ * lead fix, MEASURED 2026-09-15 with a `better-sqlite3` probe against an in-memory `fts5` table):**
+ * FTS5's default `unicode61` tokenizer (`remove_diacritics=1`) strips diacritics from Latin
+ * characters (`cafe` MATCHes `café`, `ano` MATCHes `año`) but leaves other scripts alone
+ * (`й` does NOT match `и`, `ё` does NOT match `е`). The keyword-overlap path (JSON backend, sqlite
+ * with FTS5 off) must answer IDENTICALLY — parity between the two backends of one store is the
+ * requirement — so this tokenizer does the same: NFD, strip combining marks that follow a LATIN
+ * base letter only, NFC back (so `й` = `и` + U+0306 recomposes and stays a distinct letter). A
+ * global `\p{M}` strip (the first lead attempt) would have folded Cyrillic `й`→`и`, `ё`→`е` where
+ * FTS5 does not — a divergence in the script that carries 63% of real recall traffic. Named
+ * limit, inherited from the engine: Latin words that differ only by an accent (`año`/`ano`,
+ * `côté`/`cote`) conflate in BOTH backends, as FTS5 already conflated them.
+ * Edge measured (Codex r4): `remove_diacritics=1` leaves Latin letters with TWO combining marks
+ * alone (`ộ`, `ố`, `ấ`, `ǘ` — `MATCH 'o'` does not return the `ộ` row), so the fold applies only
+ * to a single mark; `remove_diacritics=2` semantics (fold everything) are deliberately NOT used.
+ */
+export function tokenize(text: string): string[] {
+  return text
+    // Latin-only diacritic fold = FTS5 unicode61 remove_diacritics=1 (see the doc comment):
+    // NFD → drop marks after a Latin base letter → NFC (non-Latin letters recompose untouched).
+    .normalize('NFD')
+    // Codex r4 (lead, MEASURED 2026-09-15 on fts5 in-memory): remove_diacritics=1 folds a Latin
+    // letter carrying ONE mark (é→e, ü→u, ñ→n) but NOT one carrying two (Vietnamese ộ/ố/ấ, ǘ stay
+    // distinct) — so only a SINGLE mark after a Latin base is dropped; multi-mark letters recompose.
+    .replace(/(\p{Script=Latin})\p{M}(?!\p{M})/gu, '$1')
+    .normalize('NFC')
+    .toLowerCase()
+    .split(/[^\p{L}\p{N}]+/u)
+    .filter((token) => token.length > 0);
+}
+
+/**
+ * Does `text` contain at least one searchable token?
+ *
+ * FR-3: the "nothing to search by" branch in each backend's keyword path must fire ONLY when this
+ * is `false` — i.e. only when tokenization finds literally nothing (pure punctuation, whitespace,
+ * or an empty string), never merely because every token happened to be short. That narrower branch
+ * used to answer with a silent full-store dump; it no longer does (see each backend's `querySync`),
+ * and this function is the one place that decides which queries land in it — named so the decision
+ * is discoverable, not reimplemented ad hoc at each call site.
+ */
+export function hasSearchableTerms(text: string): boolean {
+  return tokenize(text).length > 0;
+}
+
+/**
+ * The named reason a query's keyword-overlap path fell into the "no searchable terms" branch, or
+ * `undefined` when it didn't. Fix-round 1 (Codex HIGH-1): both backends used to reimplement the
+ * `hasSearchableTerms(text) === false` check as a hand-rolled `terms.length === 0`, and the reason
+ * string it corresponds to lived nowhere a caller — including `dz recall`'s CLI printer — could
+ * name without re-deriving it. This is the single source of truth for that name: a backend or a
+ * printer that wants to know WHY an empty result came back calls this instead of re-tokenizing.
+ */
+export function noSearchableTermsReason(text: string): 'no-searchable-terms' | undefined {
+  return hasSearchableTerms(text) ? undefined : 'no-searchable-terms';
+}