@hraness/wordcell 0.23.0 → 0.24.0

package/skills/wordcell/references/query.md

@@ -54,7 +54,7 @@ returned:
 ```sh
 wordcell context packages/parser/src/index.ts --root "$KB_ROOT" --repo "$KB_REPO"
 wordcell search "why parser retries stop" --root "$KB_ROOT" --mode exact \
-  --history --repo "$KB_REPO" --json
+  --selected-passage --history --repo "$KB_REPO" --json
 wordcell backlinks notes/parser-contract --root "$KB_ROOT" --json
 wordcell history notes/parser-contract --root "$KB_ROOT" \
   --repo "$KB_REPO" --json
@@ -80,7 +80,7 @@ read them with the commands under
   resolve note identities before returning authored relationships.
 - A whole-vault structural question or relationship audit: use `wordcell graph --json`,
   then inspect the smallest relevant portion of its canonical output.
-- A phrase, identity, or concept expressed with different vocabulary: use `wordcell search`, whose default hybrid result preserves exact and QMD evidence separately.
+- A phrase, identity, or concept expressed with different vocabulary: use `wordcell search`, whose default hybrid result preserves exact and QMD evidence separately. Add `--selected-passage` when you will read, quote, or act on results: each hit then carries the note's best-matching excerpt with its line range and section headings.
 - Direct provenance for one note or repository path: use `wordcell history` or
   `wordcell history search` without changing authored metadata or links.
 - Recent captures awaiting maintained disposition: use the advisory `wordcell inbox` view.
@@ -96,7 +96,7 @@ wordcell backlinks "Plan title or path" --root "$KB_ROOT" --json
 wordcell links "Plan title or path" --root "$KB_ROOT" --direction both --depth 1 --limit 25 --json
 wordcell relation list "Plan title or path" --root "$KB_ROOT" --json
 wordcell graph --root "$KB_ROOT" --json
-wordcell search "why browser capture uses the current tab" --root "$KB_ROOT" --json
+wordcell search "why browser capture uses the current tab" --root "$KB_ROOT" --selected-passage --json
 wordcell search "accepted ingestion plans" --root "$KB_ROOT" --where type=plan --where status=accepted --tag ingestion --json
 wordcell search "notes/write-path" --root "$KB_ROOT" --mode exact --no-history --json
 wordcell history "notes/write-path" --root "$KB_ROOT" --repo "$KB_REPO" --json
@@ -211,6 +211,20 @@ For a repository's ordinary KB searches, prefer its approved, pinned
 choice. Read returned notes before acting; model probabilities are ranking
 signals, not proof of truth. Explicit priority rules still run last.
 
+Add `--selected-passage` to attach one source excerpt to each returned note.
+Wordcell chooses it locally, without model calls: the paragraph, or window of
+at most 512 UTF-8 bytes, that contains the most distinct query words, outside
+frontmatter. The excerpt is unedited note text with its byte offsets, line
+range, the SHA-256 of the note text, and the enclosing Markdown headings as
+separate spans. Offsets count from the start of the note text, after any
+byte-order mark. Ranking, scores, the `snippet` field, and reranker input do not
+change. A note without matching body text reports `status: "none"`; a note
+past a size limit reports `status: "unavailable"` with a reason. Both keep
+their snippet. Terminal output shows CRLF as LF and tabs as four spaces, and
+reports `display-sanitized` instead of an excerpt that credential redaction
+would alter. An excerpt shows where query words appear; it is not a verified
+answer. SDK and MCP `search` callers pass `selectedPassage: true`.
+
 `wordcell history <note>` returns the bounded commit history already associated with
 one resolved note. `wordcell history search <query-or-path>` searches the bounded Git
 projection directly and retains hashes, subjects, matched paths, co-change

package/src/cli-help.ts

@@ -114,6 +114,7 @@ export const commandHelp: readonly CommandHelp[] = [
       ["--min-score <score>", "Drop results scoring below this (0 to 1)"],
       ["--rerank <typesafe>", "Rerank results with the hosted TypeSafe service"],
       ["--rerank-limit <2..25>", "How many results the reranker sees"],
+      ["--selected-passage", "Add a local source excerpt to each result"],
       JSON_OPTION,
     ],
     examples: ['wordcell search "retries" --root kb --mode exact', 'wordcell search "retry policy" --root kb --limit 5'],

package/src/cli-program.ts

@@ -385,6 +385,7 @@ type ParsedCommand =
       readonly minScore?: number;
       readonly rerank?: "typesafe";
       readonly rerankLimit?: number;
+      readonly selectedPassage?: boolean;
       readonly query: string;
       readonly json: boolean;
     }
@@ -1274,6 +1275,7 @@ function parseSemanticCommand(command: "index" | "search", arguments_: readonly
   let minScore: number | undefined;
   let rerank: "typesafe" | undefined;
   let rerankLimit: number | undefined;
+  let selectedPassage = false;
   let graphDepth: number | undefined;
   let noGraph = false;
   let history = false;
@@ -1300,6 +1302,10 @@ function parseSemanticCommand(command: "index" | "search", arguments_: readonly
       noGraph = true;
       continue;
     }
+    if (argument === "--selected-passage" && command === "search") {
+      selectedPassage = true;
+      continue;
+    }
     if (argument === "--no-history" && command === "search") {
       noHistory = true;
       continue;
@@ -1535,6 +1541,7 @@ function parseSemanticCommand(command: "index" | "search", arguments_: readonly
       ...(minScore === undefined ? {} : { minScore }),
       ...(rerank === undefined ? {} : { rerank }),
       ...(rerankLimit === undefined ? {} : { rerankLimit }),
+      ...(selectedPassage ? { selectedPassage: true } : {}),
       query,
       json,
     },
@@ -2418,6 +2425,22 @@ function renderSemanticIndex(result: SemanticIndexResult): string {
   ].join("\n");
 }
 
+function searchForDisplay(result: KnowledgeBaseSearchResult): KnowledgeBaseSearchResult {
+  return {
+    ...result,
+    results: result.results.map((hit) => {
+      const passage = hit.selectedPassage;
+      if (passage?.status !== "selected") return hit;
+      const texts = [passage.text, ...passage.headings.map(({ text }) => text)];
+      // Terminal output always renders CRLF as LF and tabs as four spaces; only other changes lose content.
+      if (texts.every((text) => sanitizeTerminalText(redactSensitiveText(text))
+        === text.replaceAll("\r\n", "\n").replaceAll("\t", "    "))) return hit;
+      // Redacted display text cannot retain a claim of exact source bytes.
+      return { ...hit, selectedPassage: { status: "unavailable", reason: "display-sanitized" } as const };
+    }),
+  };
+}
+
 function renderKnowledgeBaseSearch(result: KnowledgeBaseSearchResult): string {
   const lines = [
     `${result.mode[0]?.toLocaleUpperCase("en-US") ?? ""}${result.mode.slice(1)} results for “${safe(result.query)}” (${result.results.length})${result.partial ? " [partial]" : ""}`,
@@ -2445,7 +2468,20 @@ function renderKnowledgeBaseSearch(result: KnowledgeBaseSearchResult): string {
       .map((item) => `${item.kind}#${searchEvidenceRank(item)}`)
       .join(", ");
     lines.push(`  ${hit.rank}. ${hit.score.toFixed(3)}  ${location} — ${safe(hit.title)} [${safe(evidence)}]`);
-    if (hit.snippet !== "") lines.push(`    ${safe(hit.snippet)}`);
+    const passage = hit.selectedPassage;
+    if (passage?.status === "selected") {
+      lines.push(`    Selected passage: ${safe(hit.path)}:${passage.startLine}-${passage.endLine}`
+        + (passage.clippedStart || passage.clippedEnd ? " (clipped excerpt)" : ""));
+      for (const heading of passage.headings) {
+        lines.push(`    Section: ${safe(heading.text)}${heading.truncated ? " (clipped)" : ""}`);
+      }
+      for (const line of passage.text.split(/\r?\n/u)) lines.push(`    ${safe(line)}`);
+    } else {
+      if (hit.snippet !== "") lines.push(`    ${safe(hit.snippet)}`);
+      if (passage !== undefined) {
+        lines.push(`    Selected passage: ${passage.status} (${safe(passage.reason)}); showing the search snippet.`);
+      }
+    }
   }
   if ((result.graph?.related.length ?? 0) > 0) {
     lines.push(`  Related graph context: ${result.graph?.related.length ?? 0}`);
@@ -2523,6 +2559,7 @@ async function runSemantic(
         ? {}
         : { candidateLimit: command.candidateLimit }),
       ...(command.minScore === undefined ? {} : { minScore: command.minScore }),
+      ...(command.selectedPassage === undefined ? {} : { selectedPassage: command.selectedPassage }),
       ...(command.rerank === undefined ? {} : {
         rerank: {
           engine: command.rerank,
@@ -2530,9 +2567,10 @@ async function runSemantic(
         },
       }),
     });
+    const display = searchForDisplay(result);
     output.stdout(command.json
-      ? terminalSafeJson(result)
-      : sanitizeTerminalText(renderKnowledgeBaseSearch(result)));
+      ? terminalSafeJson(display)
+      : sanitizeTerminalText(renderKnowledgeBaseSearch(display)));
     if (withoutIndex && person) {
       output.stderr("Searched without an index, so only exact words matched. "
         + `For meaning-based search, run wordcell index --root ${safe(shellWord(command.root))} `

package/src/mcp-tools.ts

@@ -229,6 +229,15 @@ export function optionalText(arguments_: Arguments, key: string, maxBytes: numbe
   return has(arguments_, key) ? checkedText(arguments_[key], key, maxBytes) : undefined;
 }
 
+function optionalBoolean(arguments_: Arguments, key: string): boolean | undefined {
+  if (!has(arguments_, key)) return undefined;
+  const value = arguments_[key];
+  if (typeof value !== "boolean") {
+    throw new ToolArgumentError(`Invalid argument "${key}": expected a boolean.`);
+  }
+  return value;
+}
+
 export function optionalInteger(
   arguments_: Arguments,
   key: string,
@@ -631,6 +640,11 @@ export function readTools(root: string, sessions: VaultSessions): readonly ToolE
           properties: {
             query: { type: "string", minLength: 1, maxLength: MAX_QUERY_TEXT_UTF8_BYTES },
             mode: { type: "string", enum: SEARCH_MODES, default: "hybrid" },
+            selectedPassage: {
+              type: "boolean",
+              default: false,
+              description: "Include a local source excerpt with line references and section headings. Ranking is unchanged.",
+            },
             limit: { type: "integer", minimum: 1, maximum: MAX_SEARCH_RESULTS, default: DEFAULT_SEARCH_RESULTS },
             ...filterProperties,
           },
@@ -638,10 +652,11 @@ export function readTools(root: string, sessions: VaultSessions): readonly ToolE
         annotations: { title: "Search notes", ...READ_ONLY },
       },
       handler: async (arguments_) => {
-        allowOnly(arguments_, ["query", "mode", "limit", "tags", "where", "has", "scope"]);
+        allowOnly(arguments_, ["query", "mode", "limit", "tags", "where", "has", "scope", "selectedPassage"]);
         const query = requiredText(arguments_, "query", MAX_QUERY_TEXT_UTF8_BYTES);
         const mode = optionalChoice(arguments_, "mode", SEARCH_MODES) ?? "hybrid";
         const limit = optionalInteger(arguments_, "limit", 1, MAX_SEARCH_RESULTS) ?? DEFAULT_SEARCH_RESULTS;
+        const selectedPassage = optionalBoolean(arguments_, "selectedPassage");
         const filters = metadataFilters(arguments_);
         const tags = optionalTextList(arguments_, "tags", MAX_QUERY_TAGS, MAX_QUERY_TEXT_UTF8_BYTES);
         const scopes = optionalTextList(arguments_, "scope", MAX_REPOSITORY_SCOPES, MAX_QUERY_TEXT_UTF8_BYTES);
@@ -650,6 +665,7 @@ export function readTools(root: string, sessions: VaultSessions): readonly ToolE
           query,
           mode,
           limit,
+          ...(selectedPassage === undefined ? {} : { selectedPassage }),
           ...(filters.length === 0 ? {} : { filters }),
           ...(tags === undefined ? {} : { tags }),
           ...(scopes === undefined ? {} : { repositoryScopes: scopes }),

package/src/sdk.ts

@@ -52,6 +52,15 @@ import {
   type SearchRuleRequest,
   type SearchRulesV1,
 } from "./search-rules.js";
+import {
+  createSelectedPassageBudget,
+  SELECTED_PASSAGE_MAX_BYTES,
+  SELECTED_PASSAGE_MAX_HEADING_BYTES,
+  SELECTED_PASSAGE_NONE_REASONS,
+  SELECTED_PASSAGE_UNAVAILABLE_REASONS,
+  selectSearchPassage,
+  type SelectedPassageResult,
+} from "./selected-passage.js";
 import {
   applyRerank,
   MAX_RERANK_CANDIDATES,
@@ -82,6 +91,14 @@ import {
   type UntrustedStructuredContent,
 } from "./untrusted-content.js";
 
+export type {
+  SelectedPassage,
+  SelectedPassageHeading,
+  SelectedPassageNone,
+  SelectedPassageResult,
+  SelectedPassageUnavailable,
+} from "./selected-passage.js";
+
 export const MAX_SEARCH_RESULTS = 100;
 export const MAX_SEARCH_CANDIDATES = 500;
 export const DEFAULT_SEARCH_RESULTS = 10;
@@ -131,6 +148,8 @@ export type KnowledgeBaseSearchOptions = {
   readonly history?: false | "auto" | "required" | KnowledgeBaseHistoryOptions;
   /** Opt-in rerank engine over a bounded result window. */
   readonly rerank?: KnowledgeBaseSearchRerankOptions;
+  /** Add one local query-selected source passage after ranking; never changes reranker input. */
+  readonly selectedPassage?: boolean;
 };
 
 export type KnowledgeBaseExactEvidence = {
@@ -179,6 +198,8 @@ export type KnowledgeBaseSearchHit = {
   readonly identity: boolean;
   readonly line?: number;
   readonly snippet: string;
+  /** Present only when requested; legacy snippet, line, and retrieval evidence retain their meanings. */
+  readonly selectedPassage?: SelectedPassageResult;
   readonly tags: readonly string[];
   readonly metadata: MetadataObject;
   readonly evidence: readonly KnowledgeBaseSearchEvidence[];
@@ -729,6 +750,10 @@ export async function openKnowledgeBase(
           searchRules,
         );
     const effectiveOptions = expansion.request as KnowledgeBaseSearchOptions;
+    if (effectiveOptions.selectedPassage !== undefined
+      && typeof effectiveOptions.selectedPassage !== "boolean") {
+      throw new TypeError("Search selectedPassage must be a boolean.");
+    }
     const { query: effectiveQuery } = validateSearchQuery(effectiveOptions.query);
     const mode = checkedSearchMode(effectiveOptions.mode);
     const ordering = checkedSearchOrdering(effectiveOptions.ordering);
@@ -1029,6 +1054,17 @@ export async function openKnowledgeBase(
           hit.rank === index + 1 ? hit : { ...hit, rank: index + 1 }));
       }
     }
+    if (effectiveOptions.selectedPassage === true) {
+      const budget = createSelectedPassageBudget();
+      results = Object.freeze(results.map((hit) => {
+        const note = notesById.get(hit.id);
+        if (note === undefined) throw new Error("Selected search hit has no snapshot note.");
+        return Object.freeze({
+          ...hit,
+          selectedPassage: selectSearchPassage({ query: effectiveQuery, text: note.content, budget }),
+        });
+      }));
+    }
     let graph: GraphContext | null = null;
     if (graphOptions !== null) {
       try {
@@ -1273,7 +1309,16 @@ export function packSearchContext(
       if (!append(String(searchEvidenceRank(item)))) return writer.result();
     }
     if (!append("\n\n")) return writer.result();
-    if (!append(hit.snippet)) return writer.result();
+    const selected = selectedPassageForContext(hit, `Knowledge-base search result ${hit.rank}`);
+    if (selected?.status === "selected") {
+      for (const heading of selected.headings) {
+        if (!append(`Section (level ${heading.level}, lines ${heading.startLine}-${heading.endLine}${heading.truncated ? ", shortened" : ""}):\n`)) return writer.result();
+        if (!append(heading.text)) return writer.result();
+        if (!append("\n\n")) return writer.result();
+      }
+      if (!append(`Selected source (UTF-8 snapshot ${selected.sourceSha256}; bytes ${selected.startByte}-${selected.endByte}; lines ${selected.startLine}-${selected.endLine}${selected.clippedStart || selected.clippedEnd ? "; excerpt clipped" : ""}):\n`)) return writer.result();
+      if (!append(selected.text)) return writer.result();
+    } else if (!append(hit.snippet)) return writer.result();
   }
 
   const graph = result.graph;
@@ -1420,6 +1465,107 @@ function optionalDataNumber(
   return dataNumber(record[key], `${label}.${key}`);
 }
 
+function dataCount(value: unknown, label: string, minimum = 0): number {
+  if (typeof value !== "number" || !Number.isSafeInteger(value) || value < minimum) {
+    throw new TypeError(`${label} must be an integer of at least ${minimum}.`);
+  }
+  return value;
+}
+
+function dataSourceText(value: unknown, label: string, maximumBytes: number): string {
+  const text = dataString(value, label);
+  if (/\p{Surrogate}/u.test(text)) throw new TypeError(`${label} must be well-formed Unicode.`);
+  if (Buffer.byteLength(text, "utf8") > maximumBytes) {
+    throw new RangeError(`${label} exceeds ${maximumBytes} UTF-8 bytes.`);
+  }
+  return text;
+}
+
+function sourceSpan(
+  record: InspectedDataRecord,
+  label: string,
+  text: string,
+): { startByte: number; endByte: number; startLine: number; endLine: number } {
+  const startByte = dataCount(requiredData(record, "startByte", label), `${label}.startByte`);
+  const endByte = dataCount(requiredData(record, "endByte", label), `${label}.endByte`);
+  const startLine = dataCount(requiredData(record, "startLine", label), `${label}.startLine`, 1);
+  const endLine = dataCount(requiredData(record, "endLine", label), `${label}.endLine`, startLine);
+  if (endByte - startByte !== Buffer.byteLength(text, "utf8")) {
+    throw new TypeError(`${label} byte span must match its text.`);
+  }
+  return { startByte, endByte, startLine, endLine };
+}
+
+function selectedPassageData(value: unknown, label: string): SelectedPassageResult {
+  const record = inspectedDataRecord(value, label);
+  const status = requiredData(record, "status", label);
+  if (status === "none" || status === "unavailable") {
+    const reason = requiredData(record, "reason", label);
+    const reasons: readonly string[] = status === "none"
+      ? SELECTED_PASSAGE_NONE_REASONS
+      : SELECTED_PASSAGE_UNAVAILABLE_REASONS;
+    if (typeof reason !== "string" || !reasons.includes(reason)) {
+      throw new TypeError(`${label}.reason is not a known ${status} reason.`);
+    }
+    return Object.freeze({ status, reason }) as SelectedPassageResult;
+  }
+  if (status !== "selected") throw new TypeError(`${label}.status is not a known selected-passage status.`);
+  const text = dataSourceText(requiredData(record, "text", label), `${label}.text`, SELECTED_PASSAGE_MAX_BYTES);
+  if (text === "") throw new TypeError(`${label}.text must not be empty.`);
+  const span = sourceSpan(record, label, text);
+  const sourceSha256 = dataString(requiredData(record, "sourceSha256", label), `${label}.sourceSha256`);
+  if (!/^[0-9a-f]{64}$/u.test(sourceSha256)) throw new TypeError(`${label}.sourceSha256 must be a SHA-256 digest.`);
+  if (requiredData(record, "sourceEncoding", label) !== "utf8-snapshot") {
+    throw new TypeError(`${label}.sourceEncoding must be utf8-snapshot.`);
+  }
+  let previousLevel = 0;
+  let previousEnd = 0;
+  const headings = inspectedDataArray(requiredData(record, "headings", label), `${label}.headings`, 6)
+    .map((entry, index) => {
+      const headingLabel = `${label}.headings[${index}]`;
+      const heading = inspectedDataRecord(entry, headingLabel);
+      const level = dataCount(requiredData(heading, "level", headingLabel), `${headingLabel}.level`, previousLevel + 1);
+      if (level > 6) throw new TypeError(`${headingLabel}.level must be at most 6.`);
+      const headingText = dataSourceText(
+        requiredData(heading, "text", headingLabel),
+        `${headingLabel}.text`,
+        SELECTED_PASSAGE_MAX_HEADING_BYTES,
+      );
+      const headingSpan = sourceSpan(heading, headingLabel, headingText);
+      if (headingSpan.startByte < previousEnd || headingSpan.endByte > span.startByte) {
+        throw new TypeError(`${headingLabel} must precede the selected passage in source order.`);
+      }
+      previousLevel = level;
+      previousEnd = headingSpan.endByte;
+      return Object.freeze({
+        level,
+        text: headingText,
+        ...headingSpan,
+        truncated: dataBoolean(requiredData(heading, "truncated", headingLabel), `${headingLabel}.truncated`),
+      });
+    });
+  return Object.freeze({
+    status,
+    text,
+    ...span,
+    sourceSha256,
+    sourceEncoding: "utf8-snapshot",
+    clippedStart: dataBoolean(requiredData(record, "clippedStart", label), `${label}.clippedStart`),
+    clippedEnd: dataBoolean(requiredData(record, "clippedEnd", label), `${label}.clippedEnd`),
+    headings: Object.freeze(headings),
+  });
+}
+
+/** Read an optional hit passage as untrusted data without invoking accessors. */
+function selectedPassageForContext(hit: object, label: string): SelectedPassageResult | undefined {
+  const descriptor = Object.getOwnPropertyDescriptor(hit, "selectedPassage");
+  if (descriptor === undefined) return undefined;
+  if (!("value" in descriptor)) throw new TypeError(`${label} contains an accessor property.`);
+  return descriptor.value === undefined
+    ? undefined
+    : selectedPassageData(descriptor.value, `${label}.selectedPassage`);
+}
+
 function searchEvidenceSummary(value: unknown, label: string): Readonly<Record<string, unknown>> {
   const evidence = inspectedDataRecord(value, label);
   return Object.freeze({
@@ -1438,6 +1584,7 @@ function searchHitContextRecord(value: unknown, index: number): Readonly<Record<
   ).map((entry, evidenceIndex) =>
     searchEvidenceSummary(entry, `${label}.evidence[${evidenceIndex}]`));
   const line = optionalDataNumber(hit, "line", label);
+  const selectedPassage = selectedPassageForContext(hit, label);
   return Object.freeze({
     kind: "search-result",
     id: dataString(requiredData(hit, "id", label), `${label}.id`),
@@ -1449,6 +1596,7 @@ function searchHitContextRecord(value: unknown, index: number): Readonly<Record<
     ...(line === undefined ? {} : { line }),
     evidence: Object.freeze(evidence),
     snippet: dataString(requiredData(hit, "snippet", label), `${label}.snippet`),
+    ...(selectedPassage === undefined ? {} : { selectedPassage }),
   });
 }