@avantf/dsh-mem 0.6.4 → 0.8.0

package/lib/engine/mem/store/lexical.d.ts

@@ -1,15 +1,13 @@
 /**
- * The sync relevance probe behind the plugin's conditional hints (DESIGN §12).
+ * The LEXICAL TERM SET and its term-count probe.
  *
- * WHY THIS IS LEXICAL, AND WHY IT MUST BE SYNC. The hint has to be in place *before* the step's
- * prompt is assembled — `dsh-agent-loop`'s `preStep` calls `systemPrompt.assemble()` immediately
- * after the message is admitted, and the section/context provider that renders the hint is
- * SYNCHRONOUS (it cannot await). Of the retrieval legs, only the FTS one is synchronous: the
- * semantic leg needs an `encode()`, and a warm encode is ~10–20 ms, which loses the race and lands
- * the hint one step late — i.e. after the model already decided whether to call a tool. So the
- * gate is a lexical one, and paraphrase-style questions score 0 here. That is the accepted trade:
- * it stays SILENT when unsure, and the always-on usage sections remain the fallback that tells the
- * model to query anyway.
+ * NOT the hint's decision any more. The conditional hint is `AvantfRuntime.wouldFind`, which runs
+ * the SAME retrieval pipeline as the tools and answers "would it return a hit?" (DESIGN §12). This
+ * module used to be that decision's cheap first half; it now provides the term definition the
+ * retrieval legs and floors consume ({@link relevanceTerms} / {@link gradedTerms} /
+ * {@link substringTerms}) plus a diagnostic term-count probe ({@link looksRelevant} /
+ * {@link probeStoreTerms}) that the regression test uses to show what the OLD hint would have
+ * answered (`test/retrieve.spec.ts`, shape B: lexical hit blocked by `min_score`).
  *
  * THE BAR, AND HOW IT WAS MEASURED. Terms are latin/digit words of >= 5 chars plus every CJK
  * 3-gram (the FTS tables are trigram-tokenised, so a 2-char CJK term cannot be expressed and a
@@ -24,36 +22,27 @@
  * would not have saved it; BOTH the floor and the two-term bar are load-bearing. Hence
  * `matched >= 2`, with the margin between 2 and 1 rather than a tuned threshold inside a
  * continuum. A single-word query (`cgroup`) scores 1 and stays silent on purpose: one term is not
- * evidence.
+ * evidence. The numbers stay here as the record of that calibration; the hint no longer branches
+ * on them.
  *
- * THE SHORT-CJK FALLBACK (and why the hint probe does NOT use it). The trigram index has no term
- * shorter than three characters — measured on the live store: `fts5vocab` over `facts_fts` holds
- * 27906 distinct terms and EVERY one is 3 characters (and a 2-char `MATCH` returns nothing, even as
- * a prefix query). So a 2-char CJK query produces no FTS term at all and that leg is empty by
- * construction. The FTS LEG therefore falls back to the one finer predicate the FTS5 trigram TABLE
- * does expose: `content LIKE '%…%'`, which matches a 2-char run by scanning (no index can serve a
- * 2-char pattern). See {@link substringTerms} / {@link gradedTerms}.
+ * THE SHORT-CJK FALLBACK. The trigram index has no term shorter than three characters — measured on
+ * the live store: `fts5vocab` over `facts_fts` holds 27906 distinct terms and EVERY one is 3
+ * characters (and a 2-char `MATCH` returns nothing, even as a prefix query). So a 2-char CJK query
+ * produces no FTS term at all and that leg is empty by construction. The FTS LEG therefore falls
+ * back to the one finer predicate the FTS5 trigram TABLE does expose: `content LIKE '%…%'`, which
+ * matches a 2-char run by scanning (no index can serve a 2-char pattern). See
+ * {@link substringTerms} / {@link gradedTerms}.
  *
- * The conditional-hint probe deliberately keeps the index-expressible terms only: it is
- * SYNCHRONOUS and runs on every step's prompt assembly, so it must stay a bounded number of
- * `LIMIT 1` index lookups — an O(corpus) scan per 2-char term would put a full scan on the prompt
- * path. Behaviour is unchanged there: a 2-char query scored 0 matched terms before and still does
- * (`{terms: 0, matched: 0}`), so the hint stays silent exactly as it did.
+ * The term-count probe keeps the index-expressible terms only: it is a bounded number of
+ * `LIMIT 1` index lookups, never an O(corpus) scan — there is no prompt-path budget reason for that
+ * any more, but it is still the right cost for a diagnostic.
  */
-import { type FtsTokenizer } from '../db/tokenizer.js';
+import type { Awaitable } from '../interfaces/common.js';
 /** How many of an input's terms a store holds. `terms === 0` means the input had nothing to test. */
 export interface LexicalProbe {
     terms: number;
     matched: number;
 }
-/**
- * Which stores hold something relevant — the ONE answer the plugin's hint layer consumes.
- *
- * A boolean on purpose: `kb_query` retrieves from BOTH stores, so the hint only has to decide
- * "worth mentioning or not". Which store matched is nobody's business downstream — and asking for
- * it cost a distinction the probe would have to keep alive in two places.
- */
-export type RelevanceHit = boolean;
 /**
  * The distinct terms of `text` that the FTS tables can express.
  *
@@ -92,28 +81,30 @@ export declare function substringTerms(text: string, cap?: number): string[];
  */
 export declare function gradedTerms(text: string, cap?: number): string[];
 /**
- * The gate itself: is this store worth telling the model about?
+ * The old hint's bar, kept as the lexical ORACLE the regression test compares against: does this
+ * store hold at least two of the text's distinct terms?
  *
  * Two distinct terms is the measured bar (see the module comment). Deliberately not "at least one"
  * — a single hit is what a common word produces — and deliberately not a score threshold, because
  * bm25 sums over OR'd terms and therefore rewards a long question for merely containing many
- * 3-grams.
+ * 3-grams. NOTHING on the retrieval or hint path branches on this: `wouldFind` asks the retrieval
+ * pipeline itself, and a lexical hit that cannot reach `min_score` must NOT fire the hint.
  */
 export declare function looksRelevant(probe: LexicalProbe): boolean;
 /**
  * Count how many of `text`'s terms ONE store holds.
  *
- * The loop both stores used to carry verbatim. `holds` is handed the raw TERM, not a built MATCH
- * expression: turning a term into a query depends on the tokenizer the store's FTS table was actually
- * built with (see `db/tokenizer.ts`), which is the store's business, not this module's. Each store
- * asks its own table with `LIMIT 1` — the question is whether a term exists at all, not where it ranks. `stopAt` lets a caller that only asks "≥ 2?" stop early (`relevance()` does, and the prompt
- * path is synchronous); the default keeps an exact count for the calibration recorded above.
+ * The loop both stores used to carry verbatim. `holds` is handed the raw TERM: turning a term into
+ * an index expression depends on the backend and its tokenizer, which is the backend's business
+ * (DB-3 made that explicit — the adapter owns the query builder). Each store asks its own index
+ * with `LIMIT 1` — the question is whether a term exists at all, not where it ranks. `stopAt` lets a
+ * caller that only asks "≥ 2?" stop early; the default keeps an exact count for the calibration
+ * recorded above.
  */
-export declare function probeTerms(text: string, holds: (term: string) => boolean, stopAt?: number): LexicalProbe;
+export declare function probeTerms(text: string, holds: (term: string) => Awaitable<boolean>, stopAt?: number): Promise<LexicalProbe>;
 /**
- * The store probe both stores' `lexicalProbe` used to spell out verbatim: build THIS store's MATCH
- * expression for one term (the tokenizer is a property of the store's FTS table) and ask that
- * table whether anything holds it. `probeTerms` owns the term set and the counting; this owns the
- * one part that has to differ, which is now a single `holds` closure per store.
+ * The store probe both stores' `lexicalProbe` used to spell out verbatim: ask THIS store's index
+ * whether anything holds a term. `probeTerms` owns the term set and the counting; this owns the one
+ * part that has to differ, which is now a single `holds` closure per store.
  */
-export declare function probeStoreTerms(text: string, tokenizer: FtsTokenizer, holds: (ftsQuery: string) => boolean, stopAt?: number): LexicalProbe;
+export declare function probeStoreTerms(text: string, holds: (term: string) => Awaitable<boolean>, stopAt?: number): Promise<LexicalProbe>;