@gmickel/gno 2.8.1 → 2.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/README.md +1 -1
  2. package/assets/skill/SKILL.md +4 -2
  3. package/browser-extension/artifacts/{gno-browser-clipper-v2.8.1.zip → gno-browser-clipper-v2.8.3.zip} +0 -0
  4. package/browser-extension/artifacts/gno-browser-clipper-v2.8.3.zip.sha256 +1 -0
  5. package/browser-extension/dist/manifest.json +1 -1
  6. package/package.json +9 -1
  7. package/spec/cli.md +81 -8
  8. package/spec/mcp.md +5 -6
  9. package/src/cli/commands/daemon.ts +7 -2
  10. package/src/cli/commands/doctor.ts +42 -24
  11. package/src/cli/commands/index-cmd.ts +10 -2
  12. package/src/cli/commands/links.ts +5 -22
  13. package/src/cli/commands/shared.ts +26 -1
  14. package/src/cli/commands/status.ts +60 -0
  15. package/src/cli/commands/update.ts +6 -1
  16. package/src/cli/commands/vec.ts +42 -2
  17. package/src/cli/program.ts +2 -2
  18. package/src/config/types.ts +19 -0
  19. package/src/converters/adapters/markitdownTs/adapter.ts +1 -52
  20. package/src/converters/adapters/shared/ooxml-protection.ts +58 -0
  21. package/src/converters/adapters/xlsx/adapter.ts +132 -0
  22. package/src/converters/adapters/xlsx/sheet-markdown.ts +312 -0
  23. package/src/converters/native/markdown.ts +3 -1
  24. package/src/converters/registry.ts +62 -6
  25. package/src/converters/versions.ts +2 -0
  26. package/src/core/audit-freshness.ts +1 -1
  27. package/src/core/audit-links.ts +2 -1
  28. package/src/core/audit-outside-index.ts +91 -18
  29. package/src/core/change-diff.ts +18 -6
  30. package/src/core/link-destination-parse.ts +21 -4
  31. package/src/core/link-inventory-markdown.ts +23 -3
  32. package/src/core/link-inventory-opaque.ts +3 -1
  33. package/src/core/links.ts +74 -16
  34. package/src/core/network-boundary-inventory.ts +18 -0
  35. package/src/index.ts +10 -0
  36. package/src/ingestion/document-metadata.ts +256 -0
  37. package/src/ingestion/file-child-env.ts +9 -0
  38. package/src/ingestion/file-child.ts +106 -0
  39. package/src/ingestion/file-processor.ts +546 -0
  40. package/src/ingestion/frontmatter.ts +6 -2
  41. package/src/ingestion/index.ts +1 -0
  42. package/src/ingestion/prepare-convert.ts +32 -0
  43. package/src/ingestion/prepare-file.ts +272 -0
  44. package/src/ingestion/prepare-worker.ts +43 -0
  45. package/src/ingestion/record-container.ts +7 -2
  46. package/src/ingestion/strip.ts +149 -133
  47. package/src/ingestion/sync-options.ts +7 -1
  48. package/src/ingestion/sync.ts +208 -385
  49. package/src/ingestion/types.ts +18 -0
  50. package/src/llm/native-worker/embedding-identity.ts +15 -3
  51. package/src/mcp/http-transport.ts +11 -0
  52. package/src/mcp/tools/links.ts +10 -38
  53. package/src/serve/resident-request.ts +19 -0
  54. package/src/serve/resident-runtime.ts +84 -8
  55. package/src/serve/routes/links.ts +6 -24
  56. package/src/serve/server.ts +3 -2
  57. package/src/sessions/config-refresh.ts +3 -2
  58. package/src/store/sqlite/graph-similarity.ts +5 -13
  59. package/src/store/vector/partition-index.ts +100 -0
  60. package/src/store/vector/stored-vectors.ts +20 -15
  61. package/browser-extension/artifacts/gno-browser-clipper-v2.8.1.zip.sha256 +0 -1
@@ -0,0 +1,312 @@
1
+ /**
2
+ * Worksheet to Markdown table, linear in cells (fn-198).
3
+ *
4
+ * Reproduces the Markdown the markitdown-ts chain produces for a worksheet
5
+ * (SheetJS `sheet_to_html` -> jsdom -> turndown + Joplin GFM tables) without
6
+ * building a DOM. That chain is O(columns x rows^2) and uses about 20 KB per
7
+ * cell: the GFM plugin's column-alignment scan re-reads `table.rows.length`
8
+ * through a jsdom proxy for every row of every column. Here each cell is
9
+ * rendered once from the sheet data:
10
+ *
11
+ * - cell text is chosen as `sheet_to_html` does (`h`, or escaped `w`),
12
+ * - plain text goes through turndown's whitespace collapsing and Markdown
13
+ * escaping, per line (a line break starts a new text node),
14
+ * - cells whose HTML carries markup (rich-text runs, hyperlinks) are
15
+ * converted by turndown itself with the same rules, one cell at a time,
16
+ * - rows, the empty heading row, `---` borders (SheetJS never sets an
17
+ * alignment), merges (`colspan` padding, covered cells skipped) and the
18
+ * single-cell-table case follow the GFM plugin.
19
+ *
20
+ * @module src/converters/adapters/xlsx/sheet-markdown
21
+ */
22
+
23
+ import type { CellObject, WorkSheet } from "xlsx";
24
+
25
+ // @ts-expect-error -- @joplin/turndown-plugin-gfm ships no types
26
+ import turndownPluginGfm from "@joplin/turndown-plugin-gfm";
27
+ // @ts-expect-error -- turndown ships no types
28
+ import TurndownService from "turndown";
29
+ import * as XLSX from "xlsx";
30
+
31
+ /** The part of turndown's API this module uses. */
32
+ interface Turndown {
33
+ turndown(html: string): string;
34
+ use(plugin: unknown): Turndown;
35
+ addRule(
36
+ key: string,
37
+ rule: {
38
+ filter: string[];
39
+ replacement: (
40
+ content: string,
41
+ node: { getAttribute(name: string): string | null; title: string }
42
+ ) => string;
43
+ }
44
+ ): Turndown;
45
+ }
46
+ const TurndownConstructor = TurndownService as new (options: {
47
+ headingStyle: "atx";
48
+ }) => Turndown;
49
+
50
+ /** SheetJS `escapehtml` (xlsx 0.20.3), used to recognise plain cell HTML. */
51
+ const HTML_ENCODINGS: Record<string, string> = {
52
+ '"': "&quot;",
53
+ "'": "&apos;",
54
+ ">": "&gt;",
55
+ "<": "&lt;",
56
+ "&": "&amp;",
57
+ };
58
+ const escapeHtml = (text: string): string =>
59
+ text
60
+ .replace(/[&<>'"]/g, (char) => HTML_ENCODINGS[char] ?? char)
61
+ .replace(/\n/g, "<br/>")
62
+ .replace(
63
+ // oxlint-disable-next-line no-control-regex -- SheetJS htmlcharegex: C0 controls become numeric references
64
+ /[\u0000-\u001f]/g,
65
+ (char) => `&#x${`000${char.charCodeAt(0).toString(16)}`.slice(-4)};`
66
+ );
67
+
68
+ /** turndown 7.2 `escapeMarkdown`, applied to each text node. */
69
+ const MARKDOWN_ESCAPES: Array<[RegExp, string]> = [
70
+ [/\\/g, "\\\\"],
71
+ [/\*/g, "\\*"],
72
+ [/^-/g, "\\-"],
73
+ [/^\+ /g, "\\+ "],
74
+ [/^(=+)/g, "\\$1"],
75
+ [/^(#{1,6}) /g, "\\$1 "],
76
+ [/`/g, "\\`"],
77
+ [/^~~~/g, "\\~~~"],
78
+ [/\[/g, "\\["],
79
+ [/\]/g, "\\]"],
80
+ [/^>/g, "\\>"],
81
+ [/_/g, "\\_"],
82
+ [/^(\d+)\. /g, "$1\\. "],
83
+ ];
84
+ /** Cheap pre-test: text that no escape rule can touch skips the chain. */
85
+ const MAY_NEED_ESCAPE = /[\\*`[\]_]|^[-+=#~>\d]/;
86
+
87
+ const escapeMarkdown = (text: string): string =>
88
+ MAY_NEED_ESCAPE.test(text)
89
+ ? MARKDOWN_ESCAPES.reduce(
90
+ (value, [pattern, replacement]) => value.replace(pattern, replacement),
91
+ text
92
+ )
93
+ : text;
94
+
95
+ /** turndown's `<br>` replacement with the default `br: " "` option. */
96
+ const LINE_BREAK = " \n";
97
+
98
+ /**
99
+ * Plain cell text as turndown renders it inside a table cell: each line is a
100
+ * text node whose whitespace runs collapse to one space, with the leading
101
+ * and trailing space dropped (cells and line breaks are block boundaries),
102
+ * escaped on its own; every line break renders, empty lines included.
103
+ */
104
+ const plainCellContent = (text: string): string => {
105
+ const lines: string[] = [];
106
+ for (const raw of text.split("\n")) {
107
+ // The HTML parser turns the escaped NUL (&#x0000;) into U+FFFD.
108
+ const line = raw
109
+ .replaceAll("\u0000", "\uFFFD")
110
+ .replace(/[ \r\n\t]+/g, " ")
111
+ .replace(/^ | $/g, "");
112
+ lines.push(line ? escapeMarkdown(line) : "");
113
+ }
114
+ return lines.join(LINE_BREAK);
115
+ };
116
+
117
+ let markupTurndown: Turndown | null = null;
118
+ const markupCache = new Map<string, string>();
119
+
120
+ /**
121
+ * Cell HTML with markup (rich-text runs, a hyperlink), converted by turndown
122
+ * with markitdown's rules. A paragraph gives the same block context as a
123
+ * table cell.
124
+ */
125
+ const markupCellContent = (html: string): string => {
126
+ const cached = markupCache.get(html);
127
+ if (cached !== undefined) return cached;
128
+ markupTurndown ??= createMarkitdownTurndown();
129
+ const content = markupTurndown.turndown(`<p>${html}</p>`);
130
+ markupCache.set(html, content);
131
+ return content;
132
+ };
133
+
134
+ /** markitdown-ts `CustomTurnDown` configuration (vendored dist). */
135
+ function createMarkitdownTurndown(): Turndown {
136
+ const service = new TurndownConstructor({ headingStyle: "atx" });
137
+ service.use((turndownPluginGfm as { gfm: unknown }).gfm);
138
+ service.addRule("anchor tags", {
139
+ filter: ["a"],
140
+ replacement(content, node) {
141
+ if (content === "") return "";
142
+ const prefix = content[0] === " " ? " " : "";
143
+ const suffix = content.at(-1) === " " ? " " : "";
144
+ const text = content.trim().replace(/\n\n.*/g, "");
145
+ if (text === "") return "";
146
+ const href = node.getAttribute("href");
147
+ let title = node.title;
148
+ if (href) {
149
+ try {
150
+ const parsed = new URL(href);
151
+ if (!["https:", "http:", "file:"].includes(parsed.protocol)) {
152
+ return `${prefix}${text}${suffix}`;
153
+ }
154
+ } catch {
155
+ if (!/^https?:|^file:/.test(href)) {
156
+ return `${prefix}[${text}](${href} "${title}")${suffix}`;
157
+ }
158
+ return `${prefix}${text}${suffix}`;
159
+ }
160
+ }
161
+ if (text.replace(/\\_/g, "_") === href && !title) return `<${href}>`;
162
+ if (!title && href) title = href;
163
+ const titlePart = title ? ` "${title}"` : "";
164
+ return `${prefix}[${text}](${href}${titlePart})${suffix}`;
165
+ },
166
+ });
167
+ return service;
168
+ }
169
+
170
+ /** Joplin GFM `cell()`: trim, `<br>` for newlines, escape pipes, pad. */
171
+ const formatCell = (content: string, first: boolean, colspan: number) => {
172
+ let value = content
173
+ .trim()
174
+ .replace(/\n\r/g, "<br>")
175
+ .replace(/\n/g, "<br>")
176
+ .replace(/\|+/g, "\\|");
177
+ while (value.length < 3) value += " ";
178
+ for (let extra = 1; extra < colspan; extra += 1) value += " | ";
179
+ return `${first ? "| " : " "}${value} |`;
180
+ };
181
+
182
+ /** SheetJS error text for non-finite numbers (`make_html_row`). */
183
+ const nonFiniteText = (value: number): string =>
184
+ Number.isNaN(value) ? "#NUM!" : "#DIV/0!";
185
+
186
+ /**
187
+ * Cell Markdown content (before `formatCell`), from the HTML `make_html_row`
188
+ * (xlsx 0.20.3) writes for it: `h` if present, else the escaped formatted
189
+ * text, wrapped in a link for an external hyperlink. HTML that is exactly
190
+ * the escaped text is rendered from the text; any other HTML goes through
191
+ * turndown.
192
+ */
193
+ const cellContent = (cell: CellObject | undefined): string => {
194
+ if (!cell || cell.v == null) return "";
195
+ let text: string;
196
+ let html: string;
197
+ if (cell.t === "n" && !Number.isFinite(cell.v as number)) {
198
+ // SheetJS swaps in an error cell, which has no `h` and no hyperlink.
199
+ text = nonFiniteText(cell.v as number);
200
+ return text ? plainCellContent(text) : "";
201
+ }
202
+ if (cell.h) {
203
+ html = cell.h;
204
+ text = typeof cell.v === "string" ? cell.v : "";
205
+ } else {
206
+ text =
207
+ cell.w ||
208
+ (XLSX.utils.format_cell(cell), cell.w as string | undefined) ||
209
+ "";
210
+ html = escapeHtml(text);
211
+ }
212
+ if (!html) return "";
213
+ const target = cell.l?.Target;
214
+ if (target && target.charAt(0) !== "#") {
215
+ return markupCellContent(`<a href="${escapeHtml(target)}">${html}</a>`);
216
+ }
217
+ return escapeHtml(text) === html
218
+ ? plainCellContent(text)
219
+ : markupCellContent(html);
220
+ };
221
+
222
+ interface MergeSpan {
223
+ rowspan: number;
224
+ colspan: number;
225
+ }
226
+
227
+ /**
228
+ * Merge lookup as `make_html_row` resolves it: the first merge (in sheet
229
+ * order) containing a cell decides; its top-left cell carries the span and
230
+ * the other covered cells are skipped.
231
+ */
232
+ const mergeLookup = (
233
+ merges: XLSX.Range[]
234
+ ): ((row: number, col: number) => MergeSpan | "covered" | undefined) => {
235
+ if (merges.length === 0) return () => undefined;
236
+ const byCell = new Map<string, MergeSpan | "covered">();
237
+ for (let index = merges.length - 1; index >= 0; index -= 1) {
238
+ const merge = merges[index];
239
+ if (!merge) continue;
240
+ for (let row = merge.s.r; row <= merge.e.r; row += 1) {
241
+ for (let col = merge.s.c; col <= merge.e.c; col += 1) {
242
+ byCell.set(
243
+ `${row}:${col}`,
244
+ row === merge.s.r && col === merge.s.c
245
+ ? {
246
+ rowspan: merge.e.r - merge.s.r + 1,
247
+ colspan: merge.e.c - merge.s.c + 1,
248
+ }
249
+ : "covered"
250
+ );
251
+ }
252
+ }
253
+ }
254
+ return (row, col) => byCell.get(`${row}:${col}`);
255
+ };
256
+
257
+ const readCell = (
258
+ sheet: WorkSheet,
259
+ row: number,
260
+ col: number
261
+ ): CellObject | undefined => {
262
+ const dense = (sheet as { "!data"?: CellObject[][] })["!data"];
263
+ if (dense) return dense[row]?.[col];
264
+ return sheet[XLSX.utils.encode_cell({ r: row, c: col })] as
265
+ | CellObject
266
+ | undefined;
267
+ };
268
+
269
+ /**
270
+ * Markdown for one worksheet with a `!ref`, trimmed as markitdown trims it.
271
+ * Output matches `turndown(sheet_to_html(sheet))` for the same sheet.
272
+ */
273
+ export function sheetToMarkdown(sheet: WorkSheet): string {
274
+ // Memoized markup is per sheet: the file processor reuses one worker across
275
+ // files, so a process-wide cache would retain every earlier workbook.
276
+ markupCache.clear();
277
+ const range = XLSX.utils.decode_range(sheet["!ref"] ?? "A1");
278
+ const mergeAt = mergeLookup(sheet["!merges"] ?? []);
279
+ const rows: string[] = [];
280
+ let columnCount = 0;
281
+ let onlyCell: string | null = null;
282
+ let firstRowCells = 0;
283
+ for (let row = range.s.r; row <= range.e.r; row += 1) {
284
+ const cells: string[] = [];
285
+ let cellCount = 0;
286
+ for (let col = range.s.c; col <= range.e.c; col += 1) {
287
+ const merge = mergeAt(row, col);
288
+ if (merge === "covered") continue;
289
+ const content = cellContent(readCell(sheet, row, col));
290
+ cells.push(formatCell(content, cellCount === 0, merge?.colspan ?? 1));
291
+ cellCount += 1;
292
+ onlyCell = content;
293
+ }
294
+ columnCount = Math.max(columnCount, cellCount);
295
+ if (row === range.s.r) firstRowCells = cellCount;
296
+ // A row whose cells are all covered by merges renders as an empty line,
297
+ // which the GFM plugin collapses away.
298
+ if (cells.length > 0) rows.push(cells.join(""));
299
+ }
300
+
301
+ // A one-row table with at most one cell is left as plain content.
302
+ if (range.e.r === range.s.r && firstRowCells <= 1) {
303
+ return (onlyCell ?? "").trim();
304
+ }
305
+
306
+ const secondLine = rows[1] ?? "";
307
+ const header =
308
+ columnCount && !/\| :?---/.test(secondLine)
309
+ ? `|${" |".repeat(columnCount)}\n|${" --- |".repeat(columnCount)}\n`
310
+ : "";
311
+ return `${header}${rows.join("\n")}`.trim();
312
+ }
@@ -8,7 +8,9 @@ import type { Converter, ConvertInput, ConvertResult } from "../types";
8
8
  import { basenameWithoutExt } from "../path";
9
9
  import { NATIVE_VERSIONS } from "../versions";
10
10
 
11
- const CONVERTER_ID = "native/markdown" as const;
11
+ /** Converter id of Markdown sources, the only documents with link edges. */
12
+ export const MARKDOWN_CONVERTER_ID = "native/markdown" as const;
13
+ const CONVERTER_ID = MARKDOWN_CONVERTER_ID;
12
14
  const CONVERTER_VERSION = NATIVE_VERSIONS.markdown;
13
15
 
14
16
  /** UTF-8 BOM character */
@@ -10,7 +10,7 @@ import type {
10
10
  RecordAdapter,
11
11
  } from "./types";
12
12
 
13
- import { unsupportedError } from "./errors";
13
+ import { adapterError, unsupportedError } from "./errors";
14
14
 
15
15
  export class ConverterRegistry {
16
16
  private readonly converters: Converter[] = [];
@@ -71,13 +71,48 @@ export class ConverterRegistry {
71
71
  }
72
72
  }
73
73
 
74
+ /**
75
+ * Load an adapter module; if it cannot load in this runtime, stand in a
76
+ * converter that fails its file types with ADAPTER_FAILURE and the reason.
77
+ */
78
+ export async function loadAdapter(
79
+ load: () => Promise<Converter>,
80
+ id: string,
81
+ extensions: readonly string[],
82
+ mimes: readonly string[]
83
+ ): Promise<Converter> {
84
+ try {
85
+ return await load();
86
+ } catch (cause) {
87
+ const reason = cause instanceof Error ? cause.message : String(cause);
88
+ return {
89
+ id,
90
+ version: "unavailable",
91
+ // Same matching as the real adapter, so a sniffed MIME without an
92
+ // extension still reports ADAPTER_FAILURE, not UNSUPPORTED.
93
+ canHandle: (mime, ext) =>
94
+ extensions.includes(ext) || mimes.includes(mime),
95
+ convert: (input) =>
96
+ Promise.resolve({
97
+ ok: false,
98
+ error: adapterError(
99
+ input,
100
+ id,
101
+ `${id} is unavailable in this build: ${reason}`
102
+ ),
103
+ }),
104
+ };
105
+ }
106
+ }
107
+
74
108
  /**
75
109
  * Create the default registry with all MVP converters.
76
110
  * Priority order per PRD §8.6:
77
111
  * 1. native/markdown - handles .md
78
112
  * 2. native/plaintext - handles .txt
79
- * 3. adapter/markitdown-ts - handles .pdf, .docx, .xlsx
80
- * 4. adapter/officeparser - handles .pptx
113
+ * 3. adapter/xlsx - handles .xlsx (linear SheetJS -> Markdown tables)
114
+ * 4. adapter/markitdown-ts - handles .pdf, .docx
115
+ * 5. adapter/officeparser - handles .pptx
81
116
  */
82
117
  export async function createDefaultRegistry(): Promise<ConverterRegistry> {
83
118
  const registry = new ConverterRegistry();
@@ -85,9 +120,29 @@ export async function createDefaultRegistry(): Promise<ConverterRegistry> {
85
120
  // Import converters dynamically to avoid circular deps
86
121
  const { markdownConverter } = await import("./native/markdown");
87
122
  const { plaintextConverter } = await import("./native/plaintext");
88
- const { markitdownAdapter } = await import("./adapters/markitdownTs/adapter");
89
- const { officeparserAdapter } =
90
- await import("./adapters/officeparser/adapter");
123
+ const { xlsxAdapter } = await import("./adapters/xlsx/adapter");
124
+ // The PDF/Office adapters load pdf.js, which cannot initialize where its
125
+ // native canvas binding is missing (a standalone compiled executable). A
126
+ // failed adapter only fails its own file types.
127
+ const markitdownAdapter = await loadAdapter(
128
+ async () =>
129
+ (await import("./adapters/markitdownTs/adapter")).markitdownAdapter,
130
+ "adapter/markitdown-ts",
131
+ [".pdf", ".docx"],
132
+ [
133
+ "application/pdf",
134
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
135
+ ]
136
+ );
137
+ const officeparserAdapter = await loadAdapter(
138
+ async () =>
139
+ (await import("./adapters/officeparser/adapter")).officeparserAdapter,
140
+ "adapter/officeparser",
141
+ [".pptx"],
142
+ [
143
+ "application/vnd.openxmlformats-officedocument.presentationml.presentation",
144
+ ]
145
+ );
91
146
  const { jsonlAdapter } = await import("./adapters/jsonl/adapter");
92
147
  const { transcriptAdapter } = await import("./adapters/transcript/adapter");
93
148
  const { emailRecordAdapter } = await import("./adapters/email/adapter");
@@ -98,6 +153,7 @@ export async function createDefaultRegistry(): Promise<ConverterRegistry> {
98
153
  // Register in priority order
99
154
  registry.register(markdownConverter);
100
155
  registry.register(plaintextConverter);
156
+ registry.register(xlsxAdapter);
101
157
  registry.register(markitdownAdapter);
102
158
  registry.register(officeparserAdapter);
103
159
  registry.registerRecordAdapter(jsonlAdapter);
@@ -18,5 +18,7 @@ export const NATIVE_VERSIONS = {
18
18
  */
19
19
  export const ADAPTER_VERSIONS = {
20
20
  "markitdown-ts": "0.0.10+xlsx.0.20.3",
21
+ /** Direct SheetJS -> Markdown tables (fn-198); output matches markitdown. */
22
+ xlsx: "sheetjs-0.20.3+md.1",
21
23
  officeparser: "7.8.0+pdfjs.6.3.289",
22
24
  } as const;
@@ -144,7 +144,7 @@ export const evaluateFreshnessAudit = (
144
144
  document,
145
145
  kind: "stale-indexed-revision",
146
146
  message: document.lastErrorCode
147
- ? "The latest indexing attempt recorded an error"
147
+ ? `The latest indexing attempt recorded an error (${document.lastErrorCode})`
148
148
  : "The document has no indexed-at revision evidence",
149
149
  detail: {
150
150
  indexedAt: document.indexedAt,
@@ -16,8 +16,9 @@ import {
16
16
  /**
17
17
  * 1.1: workspace-wide resolution and reference/resolution/scope evidence.
18
18
  * 1.2: `links.outside-index` for workspace files that are not indexed.
19
+ * 1.3: relative Markdown links to such files are `outside-index` too.
19
20
  */
20
- export const LINK_AUDIT_RULE_VERSION = "1.2" as const;
21
+ export const LINK_AUDIT_RULE_VERSION = "1.3" as const;
21
22
  /** Default per-rule cap; runs pass the effective `maxFindings` instead. */
22
23
  export const LINK_AUDIT_MAX_FINDINGS_PER_RULE = 1000;
23
24
 
@@ -1,7 +1,8 @@
1
1
  /**
2
- * Existence-only classification of unresolved workspace wiki links whose
3
- * target is a file inside the link workspace that is not an indexed document
4
- * (an attachment, or a note in an unindexed or excluded folder). Obsidian
2
+ * Existence-only classification of unresolved workspace wiki links and
3
+ * relative Markdown links whose target is a file inside the link workspace
4
+ * that is not an indexed document (an attachment, a script beside a note, or
5
+ * a note in an unindexed or excluded folder). Obsidian
5
6
  * resolves such links, so the link audit reports them as `outside-index`
6
7
  * instead of unresolved. Only file names are listed: no file is opened,
7
8
  * indexed, or returned, and no graph edge is created.
@@ -18,7 +19,10 @@ import { readdir, realpath, stat } from "node:fs/promises";
18
19
  // node:path join: platform path algebra; no Bun equivalent.
19
20
  import { join } from "node:path";
20
21
 
21
- import type { AuditLinkSnapshot } from "../store/sqlite/graph-link-resolver";
22
+ import type {
23
+ AuditLinkSnapshot,
24
+ AuditLinkSnapshotLink,
25
+ } from "../store/sqlite/graph-link-resolver";
22
26
 
23
27
  import {
24
28
  createWorkspaceFileMatcher,
@@ -26,6 +30,30 @@ import {
26
30
  } from "../store/sqlite/workspace-link-resolver";
27
31
  import { pathContains, placeDocument } from "./link-workspace";
28
32
 
33
+ /** Path identity for exact-path matching: NFC, case-insensitive. */
34
+ const pathKey = (path: string): string => path.normalize("NFC").toLowerCase();
35
+
36
+ /**
37
+ * A Markdown link names the file's exact path; as with wiki links, `.md` is
38
+ * optional, so `[x](Note)` matches an unindexed `Note.md`. An indexed
39
+ * `Note.md` is not outside the index, so that link keeps its status. Indexed
40
+ * paths are compared under the same key as listed files (NFC, Unicode
41
+ * lowercase), never SQLite's ASCII-only NOCASE.
42
+ */
43
+ const markdownTargetExists = (
44
+ link: AuditLinkSnapshotLink,
45
+ targetPath: string,
46
+ paths: ReadonlySet<string> | undefined,
47
+ indexedKeys: (collection: string) => ReadonlySet<string>
48
+ ): boolean => {
49
+ const key = pathKey(targetPath);
50
+ if (paths?.has(key) === true) return true;
51
+ if (key.endsWith(".md") || paths?.has(`${key}.md`) !== true) return false;
52
+ return !indexedKeys(link.targetCollection).has(
53
+ pathKey(`${link.targetRefNorm}.md`)
54
+ );
55
+ };
56
+
29
57
  /** Upper bound of files listed per workspace; beyond it the listing is partial. */
30
58
  export const WORKSPACE_FILE_LISTING_MAX_FILES = 200_000;
31
59
 
@@ -145,8 +173,10 @@ export const listWorkspaceFiles = async (
145
173
  };
146
174
 
147
175
  /**
148
- * Mark unresolved plain wiki links of audited documents inside a link
149
- * workspace whose target exists as a workspace file. Each involved workspace
176
+ * Mark unresolved plain wiki links and relative Markdown links of audited
177
+ * documents inside a link workspace whose target exists as a workspace file.
178
+ * A wiki link matches under workspace resolution rules; a Markdown link must
179
+ * name the file's exact path (NFC, case-insensitive), `.md` optional. Each involved workspace
150
180
  * is listed once per call. A failed or partial listing never marks a link
151
181
  * that is not in it; the snapshot then carries one diagnostic.
152
182
  */
@@ -163,8 +193,8 @@ export const markOutsideIndexLinks = async (
163
193
  .filter(
164
194
  ({ link }) =>
165
195
  link.resolved === null &&
166
- link.linkType === "wiki" &&
167
- link.explicitCollection !== true &&
196
+ (link.linkType === "markdown" ||
197
+ (link.linkType === "wiki" && link.explicitCollection !== true)) &&
168
198
  (audited === null || audited.has(link.sourceId))
169
199
  );
170
200
  if (candidates.length === 0) return snapshot;
@@ -172,19 +202,34 @@ export const markOutsideIndexLinks = async (
172
202
  const documentRelPaths = new Map(
173
203
  snapshot.documents.map((document) => [document.id, document.relPath])
174
204
  );
175
- const placed = candidates.flatMap(({ link, index }) => {
205
+ type PlacedLink = {
206
+ link: AuditLinkSnapshot["links"][number];
207
+ index: number;
208
+ key: string;
209
+ } & ({ sourcePath: string } | { targetPath: string });
210
+ const placed = candidates.flatMap(({ link, index }): PlacedLink[] => {
211
+ const membership = memberships.get(link.sourceCollection);
176
212
  const placement = placeDocument(
177
- memberships.get(link.sourceCollection),
213
+ membership,
178
214
  documentRelPaths.get(link.sourceId) ?? link.sourceRelPath
179
215
  );
180
- return placement.key === null
181
- ? []
182
- : [{ link, index, key: placement.key, sourcePath: placement.path }];
216
+ if (placement.key === null) return [];
217
+ if (link.linkType === "wiki") {
218
+ return [{ link, index, key: placement.key, sourcePath: placement.path }];
219
+ }
220
+ // A Markdown target is stored collection-relative; place it the same way.
221
+ const target = placeDocument(membership, link.targetRefNorm);
222
+ return target.key === placement.key
223
+ ? [{ link, index, key: placement.key, targetPath: target.path }]
224
+ : [];
183
225
  });
184
226
  if (placed.length === 0) return snapshot;
185
- const matchers = new Map<
227
+ const listings = new Map<
186
228
  string,
187
- ReturnType<typeof createWorkspaceFileMatcher>
229
+ {
230
+ matches: ReturnType<typeof createWorkspaceFileMatcher>;
231
+ paths: Set<string>;
232
+ }
188
233
  >();
189
234
  const maxFiles = options.maxFiles ?? WORKSPACE_FILE_LISTING_MAX_FILES;
190
235
  let incomplete = 0;
@@ -194,12 +239,40 @@ export const markOutsideIndexLinks = async (
194
239
  signal: options.signal,
195
240
  });
196
241
  if (!listing.complete) incomplete += 1;
197
- matchers.set(key, createWorkspaceFileMatcher(listing.files));
242
+ listings.set(key, {
243
+ matches: createWorkspaceFileMatcher(listing.files),
244
+ paths: new Set(listing.files.map(pathKey)),
245
+ });
198
246
  }
247
+ const indexedByCollection = new Map<string, Set<string>>();
248
+ const indexedKeys = (collection: string): ReadonlySet<string> => {
249
+ let keys = indexedByCollection.get(collection);
250
+ if (!keys) {
251
+ keys = new Set(
252
+ db
253
+ .query<{ relPath: string }, [string]>(
254
+ "SELECT rel_path AS relPath FROM documents WHERE active = 1 AND collection = ?"
255
+ )
256
+ .all(collection)
257
+ .map(({ relPath }) => pathKey(relPath))
258
+ );
259
+ indexedByCollection.set(collection, keys);
260
+ }
261
+ return keys;
262
+ };
199
263
  const links = [...snapshot.links];
200
264
  for (const entry of placed) {
201
- const matches = matchers.get(entry.key);
202
- if (matches?.(entry.link.targetRefNorm, entry.sourcePath)) {
265
+ const listing = listings.get(entry.key);
266
+ const exists =
267
+ "targetPath" in entry
268
+ ? markdownTargetExists(
269
+ entry.link,
270
+ entry.targetPath,
271
+ listing?.paths,
272
+ indexedKeys
273
+ )
274
+ : listing?.matches(entry.link.targetRefNorm, entry.sourcePath) === true;
275
+ if (exists) {
203
276
  links[entry.index] = { ...entry.link, outsideIndex: true };
204
277
  }
205
278
  }
@@ -12,7 +12,7 @@ import type {
12
12
 
13
13
  import { parseFrontmatter } from "../ingestion/frontmatter";
14
14
  import { buildLineOffsets } from "../ingestion/position";
15
- import { getExcludedRanges } from "../ingestion/strip";
15
+ import { type ExcludedRange, getExcludedRanges } from "../ingestion/strip";
16
16
  import { normalizeMarkdownPath, normalizeWikiName, parseLinks } from "./links";
17
17
  import { extractSections } from "./sections";
18
18
 
@@ -104,15 +104,27 @@ const extractTypedEdges = (markdown: string): string[] => {
104
104
  return [...new Set(values)].sort();
105
105
  };
106
106
 
107
+ export interface DocumentStructureOptions {
108
+ /** Only Markdown sources carry links (matches link extraction at sync). */
109
+ markdownSource: boolean;
110
+ excludedRanges?: ExcludedRange[];
111
+ }
112
+
107
113
  export const extractDocumentStructure = (
108
114
  markdown: string,
109
115
  relPath: string,
110
- dateFields: Readonly<Record<string, string>> | null | undefined
116
+ dateFields: Readonly<Record<string, string>> | null | undefined,
117
+ { markdownSource, excludedRanges }: DocumentStructureOptions
111
118
  ): DocumentStructureSnapshot => {
112
- const excludedRanges = getExcludedRanges(markdown);
113
- const links = parseLinks(markdown, buildLineOffsets(markdown), excludedRanges)
114
- .map((link) => linkSummary(link, relPath))
115
- .filter((value): value is string => value !== null);
119
+ const links = markdownSource
120
+ ? parseLinks(
121
+ markdown,
122
+ buildLineOffsets(markdown),
123
+ excludedRanges ?? getExcludedRanges(markdown)
124
+ )
125
+ .map((link) => linkSummary(link, relPath))
126
+ .filter((value): value is string => value !== null)
127
+ : [];
116
128
 
117
129
  return {
118
130
  headings: withOccurrences(