@gmickel/gno 2.8.0 → 2.8.2

package/src/core/memory-recall.ts

@@ -30,6 +30,9 @@ import {
 } from "./memory-fence";
 import {
   MEMORY_EMPTY_RECALL_HINT,
+  MEMORY_NO_MATCH_RECALL_HINT,
+  MEMORY_OVER_BUDGET_RECALL_HINT,
+  MEMORY_RECALL_ANY_TERM_MIN_RELATIVE_SCORE,
   MEMORY_RECALL_MAX_FACTS,
   MEMORY_RECALL_MAX_TOKENS,
   MEMORY_RECALL_RETRIEVAL_LIMIT,
@@ -40,6 +43,218 @@ import {
 
 type RetrievalLeg = { source: "bm25" | "vector"; results: SearchResult[] };
 
+/**
+ * Function words dropped from the lexical leg so a question-shaped turn
+ * retrieves on its content terms. English plus the common German, French,
+ * and Italian question and function words.
+ */
+const RECALL_STOPWORDS = new Set([
+  // English
+  "a",
+  "about",
+  "am",
+  "an",
+  "and",
+  "any",
+  "anything",
+  "are",
+  "as",
+  "at",
+  "be",
+  "been",
+  "but",
+  "by",
+  "can",
+  "could",
+  "did",
+  "do",
+  "does",
+  "for",
+  "from",
+  "had",
+  "has",
+  "have",
+  "how",
+  "how's",
+  "i",
+  "if",
+  "in",
+  "into",
+  "is",
+  "it",
+  "it's",
+  "its",
+  "know",
+  "me",
+  "my",
+  "of",
+  "on",
+  "or",
+  "our",
+  "please",
+  "should",
+  "so",
+  "tell",
+  "that",
+  "the",
+  "their",
+  "them",
+  "there",
+  "these",
+  "they",
+  "this",
+  "those",
+  "to",
+  "us",
+  "was",
+  "we",
+  "were",
+  "what",
+  "what's",
+  "when",
+  "where",
+  "where's",
+  "which",
+  "who",
+  "who's",
+  "whom",
+  "whose",
+  "why",
+  "will",
+  "with",
+  "would",
+  "you",
+  "your",
+  // German
+  "das",
+  "dem",
+  "den",
+  "der",
+  "des",
+  "ein",
+  "eine",
+  "ist",
+  "mit",
+  "oder",
+  "sind",
+  "und",
+  "uns",
+  "von",
+  "wann",
+  "warum",
+  "welche",
+  "welcher",
+  "welches",
+  "wer",
+  "wie",
+  "wir",
+  "wo",
+  "zu",
+  // French
+  "avec",
+  "comment",
+  "dans",
+  "de",
+  "du",
+  "est",
+  "et",
+  "la",
+  "le",
+  "les",
+  "nous",
+  "ou",
+  "où",
+  "pour",
+  "pourquoi",
+  "quand",
+  "que",
+  "quel",
+  "quelle",
+  "qui",
+  "quoi",
+  "sont",
+  "sur",
+  "un",
+  "une",
+  "vous",
+  // Italian
+  "che",
+  "chi",
+  "come",
+  "con",
+  "cosa",
+  "da",
+  "del",
+  "della",
+  "di",
+  "dove",
+  "il",
+  "per",
+  "perché",
+  "quale",
+  "quali",
+  "sono",
+]);
+
+/** Whitespace tokens, keeping a quoted phrase (optionally negated) whole. */
+const QUERY_TOKEN_PATTERN = /-?"[^"]*"|\S+/g;
+/** Same character class the FTS term sanitizer keeps. */
+const NON_TERM_CHARS = /[^\p{L}\p{N}'_]/gu;
+
+/**
+ * Content terms of a recall query: bare stopword tokens are dropped; quoted
+ * phrases, negations, and compounds pass through. A query with no positive
+ * content term left is returned unchanged.
+ */
+function recallContentQuery(query: string): string {
+  const tokens = query.match(QUERY_TOKEN_PATTERN) ?? [];
+  const kept = tokens.filter((token) => {
+    if (token.startsWith("-") || token.includes('"')) return true;
+    return !RECALL_STOPWORDS.has(
+      token.replace(NON_TERM_CHARS, "").toLowerCase()
+    );
+  });
+  const hasPositiveTerm = kept.some(
+    (token) =>
+      !token.startsWith("-") && token.replace(NON_TERM_CHARS, "").length > 0
+  );
+  return hasPositiveTerm ? kept.join(" ") : query;
+}
+
+/**
+ * Lexical leg: BM25 over the content terms, every term required first. When
+ * no fact carries all of them (a question-shaped turn), fall back to
+ * any-term matching so facts sharing a content term still rank, best BM25
+ * first, above a relative score floor. `null` means the query has no
+ * searchable terms.
+ */
+async function searchLexical(
+  deps: MemoryServiceDeps,
+  input: { query: string; collection: string; scopes: string[] }
+): Promise<SearchResult[] | null> {
+  const contentQuery = recallContentQuery(input.query);
+  const run = async (anyTerm: boolean) => {
+    const bm25 = await searchBm25(deps.store, contentQuery, {
+      collection: input.collection,
+      limit: MEMORY_RECALL_RETRIEVAL_LIMIT,
+      memoryFilter: { scopes: input.scopes, excludeSuperseded: true },
+      ...(anyTerm
+        ? {
+            anyTerm,
+            minRelativeScore: MEMORY_RECALL_ANY_TERM_MIN_RELATIVE_SCORE,
+          }
+        : {}),
+    });
+    if (bm25.ok) return bm25.value.results;
+    if (bm25.error.code !== "INVALID_INPUT") {
+      throw new MemoryError("MEMORY_QUERY_FAILED", bm25.error.message);
+    }
+    return null;
+  };
+  const allTerms = await run(false);
+  return allTerms?.length === 0 ? run(true) : allTerms;
+}
+
 /**
  * Retrieval legs: BM25 always; vectors when an embedding port and a searchable
  * vector index are present. The eligible set is one unbounded in-query
@@ -52,17 +267,9 @@ async function retrieveLegs(
   const { store, config } = deps;
   const { query, collection, scopes } = input;
   const legs: RetrievalLeg[] = [];
-  const bm25 = await searchBm25(store, query, {
-    collection,
-    limit: MEMORY_RECALL_RETRIEVAL_LIMIT,
-    memoryFilter: { scopes, excludeSuperseded: true },
-  });
-  if (!bm25.ok) {
-    if (bm25.error.code !== "INVALID_INPUT") {
-      throw new MemoryError("MEMORY_QUERY_FAILED", bm25.error.message);
-    }
-  } else {
-    legs.push({ source: "bm25", results: bm25.value.results });
+  const lexical = await searchLexical(deps, input);
+  if (lexical) {
+    legs.push({ source: "bm25", results: lexical });
   }
 
   const retrieval: RecallResult["retrieval"] = { mode: "lexical" };
@@ -159,7 +366,11 @@ async function materializeFacts(
   return materialized;
 }
 
-/** Token budget via the shared context-evidence selector, then the fact cap. */
+/**
+ * Token budget via the shared context-evidence selector, then the fact cap.
+ * Facts carry no facets, so the selector fills the budget in retrieval-rank
+ * order (per-fact facets would make it prefer the shortest facts).
+ */
 function selectWithinBudget(
   materialized: Array<{ fact: RecalledFact; rank: number }>,
   maxFacts: number,
@@ -176,11 +387,11 @@ function selectWithinBudget(
       sourceHash: fact.contentHash,
       mirrorHash: fact.contentHash,
       text: fact.text,
-      facets: [fact.uri],
+      facets: [],
       retrievalRank: rank,
       value: fact,
     })),
-    requestedFacets: materialized.map(({ fact }) => fact.uri),
+    requestedFacets: [],
     limits: {
       requestedBytes: maxTokens * MEMORY_TOKEN_BYTES_ESTIMATE,
       requestedTokens: maxTokens,
@@ -201,6 +412,28 @@ function selectWithinBudget(
   return selection.selected.slice(0, maxFacts).map((item) => item.value);
 }
 
+/**
+ * Why nothing came back: facts matched but none fit the budget, the scope
+ * holds facts but none matched, or the scope holds no current fact at all.
+ */
+async function emptyRecallHint(
+  deps: MemoryServiceDeps,
+  input: { collection: string; scopes: string[]; matched: number }
+): Promise<string> {
+  if (input.matched > 0) return MEMORY_OVER_BUDGET_RECALL_HINT;
+  const eligible = await deps.store.listMemoryEligibleDocuments({
+    collection: input.collection,
+    scopes: input.scopes,
+    excludeSuperseded: true,
+  });
+  if (!eligible.ok) {
+    throw new MemoryError("MEMORY_QUERY_FAILED", eligible.error.message);
+  }
+  return eligible.value.length > 0
+    ? MEMORY_NO_MATCH_RECALL_HINT
+    : MEMORY_EMPTY_RECALL_HINT;
+}
+
 export async function recallFacts(
   deps: MemoryServiceDeps,
   rawInput: RecallInput
@@ -264,6 +497,12 @@ export async function recallFacts(
             facts.map((fact) => fact.egressLineage)
           ),
         }
-      : { hint: MEMORY_EMPTY_RECALL_HINT }),
+      : {
+          hint: await emptyRecallHint(deps, {
+            collection: collection.name,
+            scopes,
+            matched: materialized.length,
+          }),
+        }),
   };
 }

package/src/core/memory-types.ts

@@ -33,11 +33,23 @@ export const MEMORY_RECALL_MAX_TOKENS = 512;
 /** Retrieval depth per leg before fusion and budgeting. */
 export const MEMORY_RECALL_RETRIEVAL_LIMIT = 32;
 export const MEMORY_RRF_K = 60;
+/**
+ * Any-term fallback floor: a fact must score at least this fraction of the
+ * best fact's raw BM25, so a term shared by most facts cannot pull them in.
+ */
+export const MEMORY_RECALL_ANY_TERM_MIN_RELATIVE_SCORE = 0.1;
 export const MEMORY_DEFAULT_LOCK_WAIT_MS = 120_000;
 export const MEMORY_TOKEN_BYTES_ESTIMATE = 4;
 
+/** Recall hint when the scope holds no current fact at all. */
 export const MEMORY_EMPTY_RECALL_HINT =
   'No memories in scope yet. Store one with: gno remember "<fact>" --scope <scope> --decision add';
+/** Recall hint when the scope holds facts but none matched the query. */
+export const MEMORY_NO_MATCH_RECALL_HINT =
+  'No memories in scope matched this query. Rephrase with words the fact uses, or store one with: gno remember "<fact>" --scope <scope> --decision add';
+/** Recall hint when facts matched but none fit the token budget. */
+export const MEMORY_OVER_BUDGET_RECALL_HINT =
+  "Matching memories did not fit the token budget. Raise --max-tokens (maxTokens) to return them.";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Errors

package/src/core/memory.ts

@@ -34,6 +34,8 @@ export {
   MEMORY_CANDIDATE_POOL,
   MEMORY_EMPTY_RECALL_HINT,
   MEMORY_LEXICAL_LIKELY_THRESHOLD,
+  MEMORY_NO_MATCH_RECALL_HINT,
+  MEMORY_OVER_BUDGET_RECALL_HINT,
   MEMORY_RECALL_MAX_FACTS,
   MEMORY_RECALL_MAX_TOKENS,
   MEMORY_SEMANTIC_LIKELY_THRESHOLD,

package/src/ingestion/strip.ts

@@ -4,6 +4,7 @@
  * Identifies regions to exclude from link/tag extraction:
  * - YAML frontmatter
  * - Fenced code blocks (CommonMark backtick and tilde fences)
+ * - Indented code blocks
  * - Inline code
  * - HTML comments
  *
@@ -13,6 +14,10 @@
  * @module src/ingestion/strip
  */
 
+import { fromMarkdown } from "mdast-util-from-markdown";
+import { gfmFromMarkdown } from "mdast-util-gfm";
+import { gfm } from "micromark-extension-gfm";
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Types
 // ─────────────────────────────────────────────────────────────────────────────
@@ -20,6 +25,7 @@
 export type ExcludedRangeKind =
   | "frontmatter"
   | "fenced_code"
+  | "indented_code"
   | "inline_code"
   | "html_comment";
 
@@ -39,150 +45,100 @@ export interface ExcludedRange {
 /** Frontmatter at start of file (YAML between --- delimiters) */
 const FRONTMATTER_REGEX = /^---\r?\n[\s\S]*?(?:\r?\n)?---(?:\r?\n|$)/;
 
-/** CommonMark fence opener: 0–3 spaces, then 3+ backticks or tildes + info. */
-const FENCE_OPEN_REGEX = /^ {0,3}(`{3,}|~{3,})(.*)$/u;
-
-/** CommonMark fence closer: matching character, length ≥ opener, trailing space/tabs only. */
-const FENCE_CLOSE_REGEX = /^ {0,3}(`{3,}|~{3,})[\t ]*$/u;
-
 /** HTML comments */
 const HTML_COMMENT_REGEX = /<!--[\s\S]*?-->/g;
 
-interface OpenFence {
-  marker: "`" | "~";
-  length: number;
-  start: number;
-}
-
+/** Backtick (code span or fence) or tilde fence anywhere in the text. */
+const INLINE_OR_FENCE_TRIGGER_REGEX = /`|~~~/;
+/** Blockquote markers and indentation before a line's content. */
+const CONTAINER_PREFIX_REGEX = /^[\t >]*/;
+/** Four columns of indentation after any blockquote markers. */
+const INDENTED_LINE_REGEX = /^(?:>[\t ]?)*(?:\t| {4})/;
 /**
- * Collect CommonMark fenced code ranges (backtick and tilde). A closer must
- * use the same character and be at least as long as the opener; when omitted,
- * CommonMark extends the fenced block through end of input.
+ * Lines after which an indented line may start an indented code block: an
+ * ATX heading, table row, HTML or definition line, or a thematic break or
+ * setext underline. After prose or a list item it continues the paragraph.
  */
-const collectFencedCodeRanges = (markdown: string): ExcludedRange[] => {
-  const ranges: ExcludedRange[] = [];
-  let offset = 0;
-  let open: OpenFence | null = null;
-
-  while (offset <= markdown.length) {
-    const nextNl = markdown.indexOf("\n", offset);
-    const lineEnd = nextNl === -1 ? markdown.length : nextNl;
-    const rawLine = markdown.slice(offset, lineEnd);
-    const logical = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine;
+const BLOCK_END_LINE_REGEX =
+  /^(?:[#|<[]|([-*_=])[\t ]*(?:\1[\t ]*){2,}$)|>[\t ]*$/u;
 
-    if (open) {
-      const closeRun = FENCE_CLOSE_REGEX.exec(logical)?.[1];
-      if (
-        closeRun &&
-        closeRun[0] === open.marker &&
-        closeRun.length >= open.length
-      ) {
-        const end = nextNl === -1 ? markdown.length : nextNl + 1;
-        ranges.push({ start: open.start, end, kind: "fenced_code" });
-        open = null;
-      }
-    } else {
-      const openMatch = FENCE_OPEN_REGEX.exec(logical);
-      const run = openMatch?.[1];
-      const suffix = openMatch?.[2] ?? "";
-      // Backtick info strings cannot contain backticks (CommonMark).
-      if (run && !(run[0] === "`" && suffix.includes("`"))) {
-        open = {
-          marker: run[0] as OpenFence["marker"],
-          length: run.length,
-          start: offset,
-        };
-      }
+/**
+ * Cheap superset test for code: a backtick, a tilde fence, or an indented
+ * line that can open an indented code block (not a list continuation). When
+ * it fails the note has no code and the parser is skipped.
+ */
+const mayContainCode = (markdown: string): boolean => {
+  if (INLINE_OR_FENCE_TRIGGER_REGEX.test(markdown)) return true;
+  let previous: string | null = null;
+  for (const rawLine of markdown.split("\n")) {
+    const line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine;
+    const content = line.replace(CONTAINER_PREFIX_REGEX, "");
+    if (
+      content !== "" &&
+      INDENTED_LINE_REGEX.test(line) &&
+      (previous === null ||
+        previous === "" ||
+        BLOCK_END_LINE_REGEX.test(previous))
+    ) {
+      return true;
     }
-
-    if (nextNl === -1) break;
-    offset = nextNl + 1;
-  }
-
-  if (open) {
-    ranges.push({
-      start: open.start,
-      end: markdown.length,
-      kind: "fenced_code",
-    });
+    previous = content;
   }
-
-  return ranges;
+  return false;
 };
 
-interface BacktickRun {
-  end: number;
-  length: number;
-  start: number;
+/** Opening fence at the start of a code node: backtick or tilde fence. */
+const FENCE_START_REGEX = /^[\t ]*(?:`{3,}|~{3,})/;
+
+interface CodeNode {
+  children?: CodeNode[];
+  position?: { start: { offset?: number }; end: { offset?: number } };
+  type: string;
 }
 
-/** CommonMark code spans close only on a backtick run of equal length. */
-const collectInlineCodeRanges = (
+/**
+ * Code regions as the CommonMark + GFM parser sees them: inline code spans,
+ * fenced code blocks and indented code blocks, with their block context
+ * (paragraphs, blockquotes, list items) handled by the parser. Frontmatter is
+ * blanked first (same length, newlines kept) so it cannot open a fence.
+ */
+const collectCodeRanges = (
   markdown: string,
-  excludedRanges: ExcludedRange[]
+  frontmatterEnd: number
 ): ExcludedRange[] => {
-  const runs: BacktickRun[] = [];
-  let cursor = 0;
-  let excludedIndex = 0;
-  while (cursor < markdown.length) {
-    while (
-      excludedRanges[excludedIndex] &&
-      excludedRanges[excludedIndex]!.end <= cursor
-    ) {
-      excludedIndex += 1;
-    }
-    const excluded = excludedRanges[excludedIndex];
-    if (excluded && cursor >= excluded.start && cursor < excluded.end) {
-      cursor = excluded.end;
-      continue;
-    }
-    if (markdown[cursor] !== "`") {
-      cursor += 1;
-      continue;
-    }
-    const start = cursor;
-    while (markdown[cursor] === "`") cursor += 1;
-    let backslashes = 0;
-    for (let i = start - 1; i >= 0 && markdown[i] === "\\"; i -= 1) {
-      backslashes += 1;
-    }
-    if (backslashes % 2 === 0) {
-      runs.push({ start, end: cursor, length: cursor - start });
-    }
-  }
-
-  const nextMatchingRun = Array.from<number | undefined>({
-    length: runs.length,
-  });
-  const latestByLength = new Map<number, number>();
-  for (let index = runs.length - 1; index >= 0; index -= 1) {
-    const run = runs[index];
-    if (!run) continue;
-    nextMatchingRun[index] = latestByLength.get(run.length);
-    latestByLength.set(run.length, index);
-  }
-
+  if (!mayContainCode(markdown.slice(frontmatterEnd))) return [];
+  const source =
+    frontmatterEnd > 0
+      ? markdown.slice(0, frontmatterEnd).replace(/[^\r\n]/gu, " ") +
+        markdown.slice(frontmatterEnd)
+      : markdown;
+  const root = fromMarkdown(source, {
+    extensions: [gfm()],
+    mdastExtensions: [gfmFromMarkdown()],
+  }) as CodeNode;
   const ranges: ExcludedRange[] = [];
-  let index = 0;
-  while (index < runs.length) {
-    const closeIndex = nextMatchingRun[index];
-    const opener = runs[index];
-    if (closeIndex === undefined || !opener) {
-      index += 1;
-      continue;
-    }
-    const closer = runs[closeIndex];
-    if (!closer) {
-      index += 1;
-      continue;
+  const visit = (node: CodeNode): void => {
+    const start = node.position?.start.offset;
+    const end = node.position?.end.offset;
+    if (start !== undefined && end !== undefined && end > start) {
+      if (node.type === "inlineCode") {
+        ranges.push({ start, end, kind: "inline_code" });
+        return;
+      }
+      if (node.type === "code") {
+        ranges.push({
+          start,
+          end,
+          kind: FENCE_START_REGEX.test(source.slice(start, end))
+            ? "fenced_code"
+            : "indented_code",
+        });
+        return;
+      }
     }
-    ranges.push({
-      start: opener.start,
-      end: closer.end,
-      kind: "inline_code",
-    });
-    index = closeIndex + 1;
-  }
+    for (const child of node.children ?? []) visit(child);
+  };
+  visit(root);
   return ranges;
 };
 
@@ -208,10 +164,7 @@ export function getExcludedRanges(markdown: string): ExcludedRange[] {
     });
   }
 
-  // 2. Fenced code blocks (backtick + tilde, CommonMark matching rules)
-  ranges.push(...collectFencedCodeRanges(markdown));
-
-  // 3. HTML comments
+  // 2. HTML comments
   HTML_COMMENT_REGEX.lastIndex = 0;
   let match: RegExpExecArray | null;
   while ((match = HTML_COMMENT_REGEX.exec(markdown)) !== null) {
@@ -222,10 +175,10 @@ export function getExcludedRanges(markdown: string): ExcludedRange[] {
     });
   }
 
-  // 4. Inline code. Pair delimiters only in visible prose so unmatched
-  // backticks inside already-excluded blocks cannot consume later content.
-  ranges.sort((a, b) => a.start - b.start);
-  ranges.push(...collectInlineCodeRanges(markdown, ranges));
+  // 3. Code spans and fenced or indented code blocks, from the parser.
+  ranges.push(
+    ...collectCodeRanges(markdown, frontmatterMatch?.[0].length ?? 0)
+  );
 
   // Sort by start position for efficient lookup
   ranges.sort((a, b) => a.start - b.start);