@gmickel/gno 2.8.0 → 2.8.1

package/src/core/audit-outside-index.ts

@@ -0,0 +1,215 @@
+/**
+ * Existence-only classification of unresolved workspace wiki links whose
+ * target is a file inside the link workspace that is not an indexed document
+ * (an attachment, or a note in an unindexed or excluded folder). Obsidian
+ * resolves such links, so the link audit reports them as `outside-index`
+ * instead of unresolved. Only file names are listed: no file is opened,
+ * indexed, or returned, and no graph edge is created.
+ *
+ * @module src/core/audit-outside-index
+ */
+
+import type { Database } from "bun:sqlite";
+
+// node:fs/promises readdir/realpath/stat: directory enumeration and symlink
+// resolution; Bun has no equivalent that can skip an unreadable folder
+// instead of failing the scan.
+import { readdir, realpath, stat } from "node:fs/promises";
+// node:path join: platform path algebra; no Bun equivalent.
+import { join } from "node:path";
+
+import type { AuditLinkSnapshot } from "../store/sqlite/graph-link-resolver";
+
+import {
+  createWorkspaceFileMatcher,
+  loadLinkWorkspaceMemberships,
+} from "../store/sqlite/workspace-link-resolver";
+import { pathContains, placeDocument } from "./link-workspace";
+
+/** Upper bound of files listed per workspace; beyond it the listing is partial. */
+export const WORKSPACE_FILE_LISTING_MAX_FILES = 200_000;
+
+export interface WorkspaceFileListing {
+  /** Workspace-relative POSIX paths (NFC) of every non-hidden file. */
+  files: string[];
+  /** False when a folder could not be read or the file bound was reached. */
+  complete: boolean;
+}
+
+interface WorkspaceDirectoryEntry {
+  name: string;
+  isFile(): boolean;
+  isDirectory(): boolean;
+  isSymbolicLink(): boolean;
+}
+
+/** Filesystem reads the listing needs; injectable for tests. */
+export interface WorkspaceFileSystem {
+  readDirectory(path: string): Promise<WorkspaceDirectoryEntry[]>;
+  realPath(path: string): Promise<string>;
+  isRegularFile(path: string): Promise<boolean>;
+}
+
+const nodeWorkspaceFileSystem: WorkspaceFileSystem = {
+  readDirectory: (path) => readdir(path, { withFileTypes: true }),
+  realPath: (path) => realpath(path),
+  isRegularFile: async (path) => (await stat(path)).isFile(),
+};
+
+/** A symlink target that does not exist (or loops) is not a file, not an error. */
+const MISSING_TARGET_CODES = new Set(["ENOENT", "ENOTDIR", "ELOOP"]);
+
+/**
+ * A symlink counts as a file only when it resolves to an existing regular
+ * file inside the workspace. Dangling links, links to folders and links that
+ * leave the workspace are not files. Other errors are reported as such.
+ */
+const symlinkIsWorkspaceFile = async (
+  fs: WorkspaceFileSystem,
+  realRoot: string,
+  path: string
+): Promise<"file" | "not-file" | "error"> => {
+  try {
+    const target = await fs.realPath(path);
+    if (!pathContains(realRoot, target)) return "not-file";
+    return (await fs.isRegularFile(target)) ? "file" : "not-file";
+  } catch (cause) {
+    const code = (cause as NodeJS.ErrnoException | undefined)?.code;
+    return code !== undefined && MISSING_TARGET_CODES.has(code)
+      ? "not-file"
+      : "error";
+  }
+};
+
+/**
+ * List every file below a workspace root once. Hidden files and folders
+ * (`.obsidian`, `.trash`, `.git`) are skipped, as Obsidian does; symlinked
+ * folders are not descended. Filesystem calls use the names as stored on
+ * disk; only the returned paths are NFC-normalized for matching. An
+ * unreadable folder is skipped and marks the listing incomplete, so links
+ * into it stay unresolved.
+ */
+export const listWorkspaceFiles = async (
+  root: string,
+  options: {
+    maxFiles?: number;
+    signal?: AbortSignal;
+    fileSystem?: WorkspaceFileSystem;
+  } = {}
+): Promise<WorkspaceFileListing> => {
+  const maxFiles = options.maxFiles ?? WORKSPACE_FILE_LISTING_MAX_FILES;
+  const fs = options.fileSystem ?? nodeWorkspaceFileSystem;
+  const files: string[] = [];
+  let complete = true;
+  let realRoot: string;
+  try {
+    realRoot = await fs.realPath(root);
+  } catch {
+    return { files, complete: false };
+  }
+  // Each folder keeps its on-disk segments; matching uses the NFC path.
+  const queue: string[][] = [[]];
+  for (let head = 0; head < queue.length; head += 1) {
+    options.signal?.throwIfAborted();
+    const segments = queue[head] as string[];
+    let entries;
+    try {
+      entries = await fs.readDirectory(join(root, ...segments));
+    } catch {
+      complete = false;
+      continue;
+    }
+    for (const entry of entries) {
+      if (entry.name.startsWith(".")) continue;
+      const entrySegments = [...segments, entry.name];
+      let isFile = entry.isFile();
+      if (entry.isDirectory()) {
+        queue.push(entrySegments);
+        continue;
+      }
+      if (entry.isSymbolicLink()) {
+        const kind = await symlinkIsWorkspaceFile(
+          fs,
+          realRoot,
+          join(root, ...entrySegments)
+        );
+        if (kind === "error") complete = false;
+        isFile = kind === "file";
+      }
+      if (!isFile) continue;
+      if (files.length >= maxFiles) return { files, complete: false };
+      files.push(entrySegments.join("/").normalize("NFC"));
+    }
+  }
+  return { files, complete };
+};
+
+/**
+ * Mark unresolved plain wiki links of audited documents inside a link
+ * workspace whose target exists as a workspace file. Each involved workspace
+ * is listed once per call. A failed or partial listing never marks a link
+ * that is not in it; the snapshot then carries one diagnostic.
+ */
+export const markOutsideIndexLinks = async (
+  db: Database,
+  snapshot: AuditLinkSnapshot,
+  options: { maxFiles?: number; signal?: AbortSignal } = {}
+): Promise<AuditLinkSnapshot> => {
+  const audited = snapshot.auditedDocumentIds
+    ? new Set(snapshot.auditedDocumentIds)
+    : null;
+  const candidates = snapshot.links
+    .map((link, index) => ({ link, index }))
+    .filter(
+      ({ link }) =>
+        link.resolved === null &&
+        link.linkType === "wiki" &&
+        link.explicitCollection !== true &&
+        (audited === null || audited.has(link.sourceId))
+    );
+  if (candidates.length === 0) return snapshot;
+  const memberships = loadLinkWorkspaceMemberships(db);
+  const documentRelPaths = new Map(
+    snapshot.documents.map((document) => [document.id, document.relPath])
+  );
+  const placed = candidates.flatMap(({ link, index }) => {
+    const placement = placeDocument(
+      memberships.get(link.sourceCollection),
+      documentRelPaths.get(link.sourceId) ?? link.sourceRelPath
+    );
+    return placement.key === null
+      ? []
+      : [{ link, index, key: placement.key, sourcePath: placement.path }];
+  });
+  if (placed.length === 0) return snapshot;
+  const matchers = new Map<
+    string,
+    ReturnType<typeof createWorkspaceFileMatcher>
+  >();
+  const maxFiles = options.maxFiles ?? WORKSPACE_FILE_LISTING_MAX_FILES;
+  let incomplete = 0;
+  for (const key of new Set(placed.map((entry) => entry.key))) {
+    const listing = await listWorkspaceFiles(key, {
+      maxFiles,
+      signal: options.signal,
+    });
+    if (!listing.complete) incomplete += 1;
+    matchers.set(key, createWorkspaceFileMatcher(listing.files));
+  }
+  const links = [...snapshot.links];
+  for (const entry of placed) {
+    const matches = matchers.get(entry.key);
+    if (matches?.(entry.link.targetRefNorm, entry.sourcePath)) {
+      links[entry.index] = { ...entry.link, outsideIndex: true };
+    }
+  }
+  return {
+    ...snapshot,
+    links,
+    ...(incomplete > 0
+      ? {
+          outsideIndexDiagnostic: `The file listing of ${incomplete} link workspace${incomplete === 1 ? " was" : "s were"} incomplete (an unreadable folder or link, or more than ${maxFiles} files); links to files it missed stay unresolved`,
+        }
+      : {}),
+  };
+};

package/src/core/audit-workspace.ts

@@ -32,6 +32,7 @@ import {
 } from "./audit";
 import { evaluateFreshnessAudit } from "./audit-freshness";
 import { evaluateLinkAudit } from "./audit-links";
+import { markOutsideIndexLinks } from "./audit-outside-index";
 import { evaluateProvenanceAudit } from "./audit-provenance";
 import {
   extractCaptureSourceFromFrontmatter,
@@ -429,15 +430,20 @@ const loadWorkspaceSnapshot = async (
       ? captureAuditLinkSnapshot(options.store.getRawDb())
       : emptyLinkSnapshot();
   const selectedIds = new Set(selected.documents.map(({ id }) => id));
+  const links = filterLinkSnapshot(
+    rawLinks,
+    selectedIds,
+    selected.documents,
+    selected.truncated,
+    filters.collections
+  );
   return {
     documents: observed,
-    links: filterLinkSnapshot(
-      rawLinks,
-      selectedIds,
-      selected.documents,
-      selected.truncated,
-      filters.collections
-    ),
+    links: options.categories.includes("links")
+      ? await markOutsideIndexLinks(options.store.getRawDb(), links, {
+          signal: options.signal,
+        })
+      : links,
     truncated: selected.truncated,
   };
 };

package/src/core/audit.ts

@@ -226,8 +226,16 @@ const materializeRule = (
     )
     .sort(compareAuditFindings);
 
+  // Only warning/error findings fail a passing rule; info findings of an
+  // informational rule (links.outside-index) leave it passing.
   let status = contribution.status;
-  if (findings.length > 0 && status === "pass") {
+  if (
+    status === "pass" &&
+    findings.some(
+      (finding) =>
+        finding.severity === "error" || finding.severity === "warning"
+    )
+  ) {
     status = "fail";
   }
 

package/src/core/link-inventory-markdown.ts

@@ -23,7 +23,7 @@ import {
   pushInventoryToken,
 } from "./link-inventory-opaque";
 import { isRelevantDestination } from "./link-relevance";
-import { parseTargetParts } from "./links";
+import { parseTargetParts, splitWikiLinkContent } from "./links";
 
 const EXTERNAL_URL_REGEX = /^[a-z][a-z0-9+.-]*:/i;
 
@@ -51,8 +51,7 @@ export function inventoryWikiLinks(
     }
 
     const content = match[1] ?? "";
-    const pipeIndex = content.indexOf("|");
-    const targetPart = pipeIndex >= 0 ? content.slice(0, pipeIndex) : content;
+    const targetPart = splitWikiLinkContent(content).target;
     const trimmedTarget = targetPart.trim();
     if (!trimmedTarget) continue;
     const leadingWs = targetPart.length - targetPart.trimStart().length;

package/src/core/links.ts

@@ -88,6 +88,8 @@ const LOGSEQ_EMBED_REGEX =
  * Markdown inline link: [text](url)
  * Captures: 1=text, 2=url (path and optional anchor)
  * Negative lookbehind to avoid image links ![]()
+ * Link text may contain balanced square brackets one level deep
+ * (`[see [1]](note.md)`), as CommonMark allows.
  *
  * SCOPE LIMITATIONS:
  * - Only matches simple inline links [text](url)
@@ -95,7 +97,10 @@ const LOGSEQ_EMBED_REGEX =
  * - Does NOT match autolinks <url> or bare URLs
  * - Parens in URLs not supported (use %28 %29 encoding)
  */
-const MARKDOWN_LINK_REGEX = /(?<!!)\[([^\]]*)\]\(([^)]+)\)/g;
+const MARKDOWN_LINK_REGEX = /(?<!!)\[((?:[^[\]]|\[[^[\]]*\])*)\]\(([^)]+)\)/g;
+
+/** Square brackets in a destination mean the text was split, not a path. */
+const BRACKET_IN_DESTINATION_REGEX = /[[\]]/;
 
 /** External URL pattern (http:// https:// mailto: etc.) */
 const EXTERNAL_URL_REGEX = /^[a-z][a-z0-9+.-]*:/i;
@@ -144,6 +149,26 @@ export function extractWikiBasename(ref: string): string {
   return stripWikiMdExt(base);
 }
 
+/**
+ * Split the content of a wiki link into target and alias. Obsidian also reads
+ * `\|` (the pipe escaped inside a Markdown table) as the alias separator,
+ * so `[[Note\|Alias]]` targets `Note`, not `Note\`.
+ */
+export function splitWikiLinkContent(content: string): {
+  target: string;
+  alias?: string;
+} {
+  const pipeIndex = content.indexOf("|");
+  if (pipeIndex < 0) {
+    return { target: content };
+  }
+  const escaped = pipeIndex > 0 && content[pipeIndex - 1] === "\\";
+  return {
+    target: content.slice(0, escaped ? pipeIndex - 1 : pipeIndex),
+    alias: content.slice(pipeIndex + 1),
+  };
+}
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Path Normalization
 // ─────────────────────────────────────────────────────────────────────────────
@@ -338,22 +363,14 @@ export function parseLinks(
     const content = match[1];
     if (!content) continue;
 
-    // Parse [[target|alias]] format
-    const pipeIndex = content.indexOf("|");
-    let targetPart: string;
-    let displayText: string | undefined;
-
-    if (pipeIndex >= 0) {
-      targetPart = content.slice(0, pipeIndex);
-      const aliasText = content.slice(pipeIndex + 1);
-      // Only set displayText if different from target
-      displayText =
-        aliasText !== targetPart
-          ? truncateText(aliasText, MAX_DISPLAY_TEXT_GRAPHEMES)
-          : undefined;
-    } else {
-      targetPart = content;
-    }
+    // Parse [[target|alias]] (and table-escaped [[target\|alias]]) format
+    const { target: targetPart, alias: aliasText } =
+      splitWikiLinkContent(content);
+    // Only set displayText if different from target
+    const displayText =
+      aliasText !== undefined && aliasText !== targetPart
+        ? truncateText(aliasText, MAX_DISPLAY_TEXT_GRAPHEMES)
+        : undefined;
 
     const trimmedTarget = targetPart.trim();
     if (!trimmedTarget) {
@@ -427,6 +444,12 @@ export function parseLinks(
       continue;
     }
 
+    // Brackets in the destination come from link text split at an
+    // unbalanced bracket (`[a [b](c](d)`): unparseable, not a missing target.
+    if (BRACKET_IN_DESTINATION_REGEX.test(url)) {
+      continue;
+    }
+
     // Skip URLs that look like protocol-relative (//example.com)
     if (url.startsWith("//")) {
       continue;

package/src/core/memory-recall.ts

@@ -30,6 +30,9 @@ import {
 } from "./memory-fence";
 import {
   MEMORY_EMPTY_RECALL_HINT,
+  MEMORY_NO_MATCH_RECALL_HINT,
+  MEMORY_OVER_BUDGET_RECALL_HINT,
+  MEMORY_RECALL_ANY_TERM_MIN_RELATIVE_SCORE,
   MEMORY_RECALL_MAX_FACTS,
   MEMORY_RECALL_MAX_TOKENS,
   MEMORY_RECALL_RETRIEVAL_LIMIT,
@@ -40,6 +43,218 @@ import {
 
 type RetrievalLeg = { source: "bm25" | "vector"; results: SearchResult[] };
 
+/**
+ * Function words dropped from the lexical leg so a question-shaped turn
+ * retrieves on its content terms. English plus the common German, French,
+ * and Italian question and function words.
+ */
+const RECALL_STOPWORDS = new Set([
+  // English
+  "a",
+  "about",
+  "am",
+  "an",
+  "and",
+  "any",
+  "anything",
+  "are",
+  "as",
+  "at",
+  "be",
+  "been",
+  "but",
+  "by",
+  "can",
+  "could",
+  "did",
+  "do",
+  "does",
+  "for",
+  "from",
+  "had",
+  "has",
+  "have",
+  "how",
+  "how's",
+  "i",
+  "if",
+  "in",
+  "into",
+  "is",
+  "it",
+  "it's",
+  "its",
+  "know",
+  "me",
+  "my",
+  "of",
+  "on",
+  "or",
+  "our",
+  "please",
+  "should",
+  "so",
+  "tell",
+  "that",
+  "the",
+  "their",
+  "them",
+  "there",
+  "these",
+  "they",
+  "this",
+  "those",
+  "to",
+  "us",
+  "was",
+  "we",
+  "were",
+  "what",
+  "what's",
+  "when",
+  "where",
+  "where's",
+  "which",
+  "who",
+  "who's",
+  "whom",
+  "whose",
+  "why",
+  "will",
+  "with",
+  "would",
+  "you",
+  "your",
+  // German
+  "das",
+  "dem",
+  "den",
+  "der",
+  "des",
+  "ein",
+  "eine",
+  "ist",
+  "mit",
+  "oder",
+  "sind",
+  "und",
+  "uns",
+  "von",
+  "wann",
+  "warum",
+  "welche",
+  "welcher",
+  "welches",
+  "wer",
+  "wie",
+  "wir",
+  "wo",
+  "zu",
+  // French
+  "avec",
+  "comment",
+  "dans",
+  "de",
+  "du",
+  "est",
+  "et",
+  "la",
+  "le",
+  "les",
+  "nous",
+  "ou",
+  "où",
+  "pour",
+  "pourquoi",
+  "quand",
+  "que",
+  "quel",
+  "quelle",
+  "qui",
+  "quoi",
+  "sont",
+  "sur",
+  "un",
+  "une",
+  "vous",
+  // Italian
+  "che",
+  "chi",
+  "come",
+  "con",
+  "cosa",
+  "da",
+  "del",
+  "della",
+  "di",
+  "dove",
+  "il",
+  "per",
+  "perché",
+  "quale",
+  "quali",
+  "sono",
+]);
+
+/** Whitespace tokens, keeping a quoted phrase (optionally negated) whole. */
+const QUERY_TOKEN_PATTERN = /-?"[^"]*"|\S+/g;
+/** Same character class the FTS term sanitizer keeps. */
+const NON_TERM_CHARS = /[^\p{L}\p{N}'_]/gu;
+
+/**
+ * Content terms of a recall query: bare stopword tokens are dropped; quoted
+ * phrases, negations, and compounds pass through. A query with no positive
+ * content term left is returned unchanged.
+ */
+function recallContentQuery(query: string): string {
+  const tokens = query.match(QUERY_TOKEN_PATTERN) ?? [];
+  const kept = tokens.filter((token) => {
+    if (token.startsWith("-") || token.includes('"')) return true;
+    return !RECALL_STOPWORDS.has(
+      token.replace(NON_TERM_CHARS, "").toLowerCase()
+    );
+  });
+  const hasPositiveTerm = kept.some(
+    (token) =>
+      !token.startsWith("-") && token.replace(NON_TERM_CHARS, "").length > 0
+  );
+  return hasPositiveTerm ? kept.join(" ") : query;
+}
+
+/**
+ * Lexical leg: BM25 over the content terms, every term required first. When
+ * no fact carries all of them (a question-shaped turn), fall back to
+ * any-term matching so facts sharing a content term still rank, best BM25
+ * first, above a relative score floor. `null` means the query has no
+ * searchable terms.
+ */
+async function searchLexical(
+  deps: MemoryServiceDeps,
+  input: { query: string; collection: string; scopes: string[] }
+): Promise<SearchResult[] | null> {
+  const contentQuery = recallContentQuery(input.query);
+  const run = async (anyTerm: boolean) => {
+    const bm25 = await searchBm25(deps.store, contentQuery, {
+      collection: input.collection,
+      limit: MEMORY_RECALL_RETRIEVAL_LIMIT,
+      memoryFilter: { scopes: input.scopes, excludeSuperseded: true },
+      ...(anyTerm
+        ? {
+            anyTerm,
+            minRelativeScore: MEMORY_RECALL_ANY_TERM_MIN_RELATIVE_SCORE,
+          }
+        : {}),
+    });
+    if (bm25.ok) return bm25.value.results;
+    if (bm25.error.code !== "INVALID_INPUT") {
+      throw new MemoryError("MEMORY_QUERY_FAILED", bm25.error.message);
+    }
+    return null;
+  };
+  const allTerms = await run(false);
+  return allTerms?.length === 0 ? run(true) : allTerms;
+}
+
 /**
  * Retrieval legs: BM25 always; vectors when an embedding port and a searchable
  * vector index are present. The eligible set is one unbounded in-query
@@ -52,17 +267,9 @@ async function retrieveLegs(
   const { store, config } = deps;
   const { query, collection, scopes } = input;
   const legs: RetrievalLeg[] = [];
-  const bm25 = await searchBm25(store, query, {
-    collection,
-    limit: MEMORY_RECALL_RETRIEVAL_LIMIT,
-    memoryFilter: { scopes, excludeSuperseded: true },
-  });
-  if (!bm25.ok) {
-    if (bm25.error.code !== "INVALID_INPUT") {
-      throw new MemoryError("MEMORY_QUERY_FAILED", bm25.error.message);
-    }
-  } else {
-    legs.push({ source: "bm25", results: bm25.value.results });
+  const lexical = await searchLexical(deps, input);
+  if (lexical) {
+    legs.push({ source: "bm25", results: lexical });
   }
 
   const retrieval: RecallResult["retrieval"] = { mode: "lexical" };
@@ -159,7 +366,11 @@ async function materializeFacts(
   return materialized;
 }
 
-/** Token budget via the shared context-evidence selector, then the fact cap. */
+/**
+ * Token budget via the shared context-evidence selector, then the fact cap.
+ * Facts carry no facets, so the selector fills the budget in retrieval-rank
+ * order (per-fact facets would make it prefer the shortest facts).
+ */
 function selectWithinBudget(
   materialized: Array<{ fact: RecalledFact; rank: number }>,
   maxFacts: number,
@@ -176,11 +387,11 @@ function selectWithinBudget(
       sourceHash: fact.contentHash,
       mirrorHash: fact.contentHash,
       text: fact.text,
-      facets: [fact.uri],
+      facets: [],
       retrievalRank: rank,
       value: fact,
     })),
-    requestedFacets: materialized.map(({ fact }) => fact.uri),
+    requestedFacets: [],
     limits: {
       requestedBytes: maxTokens * MEMORY_TOKEN_BYTES_ESTIMATE,
       requestedTokens: maxTokens,
@@ -201,6 +412,28 @@ function selectWithinBudget(
   return selection.selected.slice(0, maxFacts).map((item) => item.value);
 }
 
+/**
+ * Why nothing came back: facts matched but none fit the budget, the scope
+ * holds facts but none matched, or the scope holds no current fact at all.
+ */
+async function emptyRecallHint(
+  deps: MemoryServiceDeps,
+  input: { collection: string; scopes: string[]; matched: number }
+): Promise<string> {
+  if (input.matched > 0) return MEMORY_OVER_BUDGET_RECALL_HINT;
+  const eligible = await deps.store.listMemoryEligibleDocuments({
+    collection: input.collection,
+    scopes: input.scopes,
+    excludeSuperseded: true,
+  });
+  if (!eligible.ok) {
+    throw new MemoryError("MEMORY_QUERY_FAILED", eligible.error.message);
+  }
+  return eligible.value.length > 0
+    ? MEMORY_NO_MATCH_RECALL_HINT
+    : MEMORY_EMPTY_RECALL_HINT;
+}
+
 export async function recallFacts(
   deps: MemoryServiceDeps,
   rawInput: RecallInput
@@ -264,6 +497,12 @@ export async function recallFacts(
             facts.map((fact) => fact.egressLineage)
           ),
         }
-      : { hint: MEMORY_EMPTY_RECALL_HINT }),
+      : {
+          hint: await emptyRecallHint(deps, {
+            collection: collection.name,
+            scopes,
+            matched: materialized.length,
+          }),
+        }),
   };
 }