@gmickel/gno 2.8.1 → 2.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/skill/SKILL.md +4 -2
- package/browser-extension/artifacts/{gno-browser-clipper-v2.8.1.zip → gno-browser-clipper-v2.8.3.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.8.3.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +9 -1
- package/spec/cli.md +81 -8
- package/spec/mcp.md +5 -6
- package/src/cli/commands/daemon.ts +7 -2
- package/src/cli/commands/doctor.ts +42 -24
- package/src/cli/commands/index-cmd.ts +10 -2
- package/src/cli/commands/links.ts +5 -22
- package/src/cli/commands/shared.ts +26 -1
- package/src/cli/commands/status.ts +60 -0
- package/src/cli/commands/update.ts +6 -1
- package/src/cli/commands/vec.ts +42 -2
- package/src/cli/program.ts +2 -2
- package/src/config/types.ts +19 -0
- package/src/converters/adapters/markitdownTs/adapter.ts +1 -52
- package/src/converters/adapters/shared/ooxml-protection.ts +58 -0
- package/src/converters/adapters/xlsx/adapter.ts +132 -0
- package/src/converters/adapters/xlsx/sheet-markdown.ts +312 -0
- package/src/converters/native/markdown.ts +3 -1
- package/src/converters/registry.ts +62 -6
- package/src/converters/versions.ts +2 -0
- package/src/core/audit-freshness.ts +1 -1
- package/src/core/audit-links.ts +2 -1
- package/src/core/audit-outside-index.ts +91 -18
- package/src/core/change-diff.ts +18 -6
- package/src/core/link-destination-parse.ts +21 -4
- package/src/core/link-inventory-markdown.ts +23 -3
- package/src/core/link-inventory-opaque.ts +3 -1
- package/src/core/links.ts +74 -16
- package/src/core/network-boundary-inventory.ts +18 -0
- package/src/index.ts +10 -0
- package/src/ingestion/document-metadata.ts +256 -0
- package/src/ingestion/file-child-env.ts +9 -0
- package/src/ingestion/file-child.ts +106 -0
- package/src/ingestion/file-processor.ts +546 -0
- package/src/ingestion/frontmatter.ts +6 -2
- package/src/ingestion/index.ts +1 -0
- package/src/ingestion/prepare-convert.ts +32 -0
- package/src/ingestion/prepare-file.ts +272 -0
- package/src/ingestion/prepare-worker.ts +43 -0
- package/src/ingestion/record-container.ts +7 -2
- package/src/ingestion/strip.ts +149 -133
- package/src/ingestion/sync-options.ts +7 -1
- package/src/ingestion/sync.ts +208 -385
- package/src/ingestion/types.ts +18 -0
- package/src/llm/native-worker/embedding-identity.ts +15 -3
- package/src/mcp/http-transport.ts +11 -0
- package/src/mcp/tools/links.ts +10 -38
- package/src/serve/resident-request.ts +19 -0
- package/src/serve/resident-runtime.ts +84 -8
- package/src/serve/routes/links.ts +6 -24
- package/src/serve/server.ts +3 -2
- package/src/sessions/config-refresh.ts +3 -2
- package/src/store/sqlite/graph-similarity.ts +5 -13
- package/src/store/vector/partition-index.ts +100 -0
- package/src/store/vector/stored-vectors.ts +20 -15
- package/browser-extension/artifacts/gno-browser-clipper-v2.8.1.zip.sha256 +0 -1
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worksheet to Markdown table, linear in cells (fn-198).
|
|
3
|
+
*
|
|
4
|
+
* Reproduces the Markdown the markitdown-ts chain produces for a worksheet
|
|
5
|
+
* (SheetJS `sheet_to_html` -> jsdom -> turndown + Joplin GFM tables) without
|
|
6
|
+
* building a DOM. That chain is O(columns x rows^2) and uses about 20 KB per
|
|
7
|
+
* cell: the GFM plugin's column-alignment scan re-reads `table.rows.length`
|
|
8
|
+
* through a jsdom proxy for every row of every column. Here each cell is
|
|
9
|
+
* rendered once from the sheet data:
|
|
10
|
+
*
|
|
11
|
+
* - cell text is chosen as `sheet_to_html` does (`h`, or escaped `w`),
|
|
12
|
+
* - plain text goes through turndown's whitespace collapsing and Markdown
|
|
13
|
+
* escaping, per line (a line break starts a new text node),
|
|
14
|
+
* - cells whose HTML carries markup (rich-text runs, hyperlinks) are
|
|
15
|
+
* converted by turndown itself with the same rules, one cell at a time,
|
|
16
|
+
* - rows, the empty heading row, `---` borders (SheetJS never sets an
|
|
17
|
+
* alignment), merges (`colspan` padding, covered cells skipped) and the
|
|
18
|
+
* single-cell-table case follow the GFM plugin.
|
|
19
|
+
*
|
|
20
|
+
* @module src/converters/adapters/xlsx/sheet-markdown
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import type { CellObject, WorkSheet } from "xlsx";
|
|
24
|
+
|
|
25
|
+
// @ts-expect-error -- @joplin/turndown-plugin-gfm ships no types
|
|
26
|
+
import turndownPluginGfm from "@joplin/turndown-plugin-gfm";
|
|
27
|
+
// @ts-expect-error -- turndown ships no types
|
|
28
|
+
import TurndownService from "turndown";
|
|
29
|
+
import * as XLSX from "xlsx";
|
|
30
|
+
|
|
31
|
+
/** The part of turndown's API this module uses. */
|
|
32
|
+
interface Turndown {
|
|
33
|
+
turndown(html: string): string;
|
|
34
|
+
use(plugin: unknown): Turndown;
|
|
35
|
+
addRule(
|
|
36
|
+
key: string,
|
|
37
|
+
rule: {
|
|
38
|
+
filter: string[];
|
|
39
|
+
replacement: (
|
|
40
|
+
content: string,
|
|
41
|
+
node: { getAttribute(name: string): string | null; title: string }
|
|
42
|
+
) => string;
|
|
43
|
+
}
|
|
44
|
+
): Turndown;
|
|
45
|
+
}
|
|
46
|
+
const TurndownConstructor = TurndownService as new (options: {
|
|
47
|
+
headingStyle: "atx";
|
|
48
|
+
}) => Turndown;
|
|
49
|
+
|
|
50
|
+
/** SheetJS `escapehtml` (xlsx 0.20.3), used to recognise plain cell HTML. */
|
|
51
|
+
const HTML_ENCODINGS: Record<string, string> = {
|
|
52
|
+
'"': """,
|
|
53
|
+
"'": "'",
|
|
54
|
+
">": ">",
|
|
55
|
+
"<": "<",
|
|
56
|
+
"&": "&",
|
|
57
|
+
};
|
|
58
|
+
const escapeHtml = (text: string): string =>
|
|
59
|
+
text
|
|
60
|
+
.replace(/[&<>'"]/g, (char) => HTML_ENCODINGS[char] ?? char)
|
|
61
|
+
.replace(/\n/g, "<br/>")
|
|
62
|
+
.replace(
|
|
63
|
+
// oxlint-disable-next-line no-control-regex -- SheetJS htmlcharegex: C0 controls become numeric references
|
|
64
|
+
/[\u0000-\u001f]/g,
|
|
65
|
+
(char) => `&#x${`000${char.charCodeAt(0).toString(16)}`.slice(-4)};`
|
|
66
|
+
);
|
|
67
|
+
|
|
68
|
+
/** turndown 7.2 `escapeMarkdown`, applied to each text node. */
|
|
69
|
+
const MARKDOWN_ESCAPES: Array<[RegExp, string]> = [
|
|
70
|
+
[/\\/g, "\\\\"],
|
|
71
|
+
[/\*/g, "\\*"],
|
|
72
|
+
[/^-/g, "\\-"],
|
|
73
|
+
[/^\+ /g, "\\+ "],
|
|
74
|
+
[/^(=+)/g, "\\$1"],
|
|
75
|
+
[/^(#{1,6}) /g, "\\$1 "],
|
|
76
|
+
[/`/g, "\\`"],
|
|
77
|
+
[/^~~~/g, "\\~~~"],
|
|
78
|
+
[/\[/g, "\\["],
|
|
79
|
+
[/\]/g, "\\]"],
|
|
80
|
+
[/^>/g, "\\>"],
|
|
81
|
+
[/_/g, "\\_"],
|
|
82
|
+
[/^(\d+)\. /g, "$1\\. "],
|
|
83
|
+
];
|
|
84
|
+
/** Cheap pre-test: text that no escape rule can touch skips the chain. */
|
|
85
|
+
const MAY_NEED_ESCAPE = /[\\*`[\]_]|^[-+=#~>\d]/;
|
|
86
|
+
|
|
87
|
+
const escapeMarkdown = (text: string): string =>
|
|
88
|
+
MAY_NEED_ESCAPE.test(text)
|
|
89
|
+
? MARKDOWN_ESCAPES.reduce(
|
|
90
|
+
(value, [pattern, replacement]) => value.replace(pattern, replacement),
|
|
91
|
+
text
|
|
92
|
+
)
|
|
93
|
+
: text;
|
|
94
|
+
|
|
95
|
+
/** turndown's `<br>` replacement with the default `br: " "` option. */
|
|
96
|
+
const LINE_BREAK = " \n";
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Plain cell text as turndown renders it inside a table cell: each line is a
|
|
100
|
+
* text node whose whitespace runs collapse to one space, with the leading
|
|
101
|
+
* and trailing space dropped (cells and line breaks are block boundaries),
|
|
102
|
+
* escaped on its own; every line break renders, empty lines included.
|
|
103
|
+
*/
|
|
104
|
+
const plainCellContent = (text: string): string => {
|
|
105
|
+
const lines: string[] = [];
|
|
106
|
+
for (const raw of text.split("\n")) {
|
|
107
|
+
// The HTML parser turns the escaped NUL (�) into U+FFFD.
|
|
108
|
+
const line = raw
|
|
109
|
+
.replaceAll("\u0000", "\uFFFD")
|
|
110
|
+
.replace(/[ \r\n\t]+/g, " ")
|
|
111
|
+
.replace(/^ | $/g, "");
|
|
112
|
+
lines.push(line ? escapeMarkdown(line) : "");
|
|
113
|
+
}
|
|
114
|
+
return lines.join(LINE_BREAK);
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
let markupTurndown: Turndown | null = null;
|
|
118
|
+
const markupCache = new Map<string, string>();
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Cell HTML with markup (rich-text runs, a hyperlink), converted by turndown
|
|
122
|
+
* with markitdown's rules. A paragraph gives the same block context as a
|
|
123
|
+
* table cell.
|
|
124
|
+
*/
|
|
125
|
+
const markupCellContent = (html: string): string => {
|
|
126
|
+
const cached = markupCache.get(html);
|
|
127
|
+
if (cached !== undefined) return cached;
|
|
128
|
+
markupTurndown ??= createMarkitdownTurndown();
|
|
129
|
+
const content = markupTurndown.turndown(`<p>${html}</p>`);
|
|
130
|
+
markupCache.set(html, content);
|
|
131
|
+
return content;
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
/** markitdown-ts `CustomTurnDown` configuration (vendored dist). */
|
|
135
|
+
function createMarkitdownTurndown(): Turndown {
|
|
136
|
+
const service = new TurndownConstructor({ headingStyle: "atx" });
|
|
137
|
+
service.use((turndownPluginGfm as { gfm: unknown }).gfm);
|
|
138
|
+
service.addRule("anchor tags", {
|
|
139
|
+
filter: ["a"],
|
|
140
|
+
replacement(content, node) {
|
|
141
|
+
if (content === "") return "";
|
|
142
|
+
const prefix = content[0] === " " ? " " : "";
|
|
143
|
+
const suffix = content.at(-1) === " " ? " " : "";
|
|
144
|
+
const text = content.trim().replace(/\n\n.*/g, "");
|
|
145
|
+
if (text === "") return "";
|
|
146
|
+
const href = node.getAttribute("href");
|
|
147
|
+
let title = node.title;
|
|
148
|
+
if (href) {
|
|
149
|
+
try {
|
|
150
|
+
const parsed = new URL(href);
|
|
151
|
+
if (!["https:", "http:", "file:"].includes(parsed.protocol)) {
|
|
152
|
+
return `${prefix}${text}${suffix}`;
|
|
153
|
+
}
|
|
154
|
+
} catch {
|
|
155
|
+
if (!/^https?:|^file:/.test(href)) {
|
|
156
|
+
return `${prefix}[${text}](${href} "${title}")${suffix}`;
|
|
157
|
+
}
|
|
158
|
+
return `${prefix}${text}${suffix}`;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if (text.replace(/\\_/g, "_") === href && !title) return `<${href}>`;
|
|
162
|
+
if (!title && href) title = href;
|
|
163
|
+
const titlePart = title ? ` "${title}"` : "";
|
|
164
|
+
return `${prefix}[${text}](${href}${titlePart})${suffix}`;
|
|
165
|
+
},
|
|
166
|
+
});
|
|
167
|
+
return service;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Joplin GFM `cell()`: trim, `<br>` for newlines, escape pipes, pad. */
|
|
171
|
+
const formatCell = (content: string, first: boolean, colspan: number) => {
|
|
172
|
+
let value = content
|
|
173
|
+
.trim()
|
|
174
|
+
.replace(/\n\r/g, "<br>")
|
|
175
|
+
.replace(/\n/g, "<br>")
|
|
176
|
+
.replace(/\|+/g, "\\|");
|
|
177
|
+
while (value.length < 3) value += " ";
|
|
178
|
+
for (let extra = 1; extra < colspan; extra += 1) value += " | ";
|
|
179
|
+
return `${first ? "| " : " "}${value} |`;
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
/** SheetJS error text for non-finite numbers (`make_html_row`). */
|
|
183
|
+
const nonFiniteText = (value: number): string =>
|
|
184
|
+
Number.isNaN(value) ? "#NUM!" : "#DIV/0!";
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Cell Markdown content (before `formatCell`), from the HTML `make_html_row`
|
|
188
|
+
* (xlsx 0.20.3) writes for it: `h` if present, else the escaped formatted
|
|
189
|
+
* text, wrapped in a link for an external hyperlink. HTML that is exactly
|
|
190
|
+
* the escaped text is rendered from the text; any other HTML goes through
|
|
191
|
+
* turndown.
|
|
192
|
+
*/
|
|
193
|
+
const cellContent = (cell: CellObject | undefined): string => {
|
|
194
|
+
if (!cell || cell.v == null) return "";
|
|
195
|
+
let text: string;
|
|
196
|
+
let html: string;
|
|
197
|
+
if (cell.t === "n" && !Number.isFinite(cell.v as number)) {
|
|
198
|
+
// SheetJS swaps in an error cell, which has no `h` and no hyperlink.
|
|
199
|
+
text = nonFiniteText(cell.v as number);
|
|
200
|
+
return text ? plainCellContent(text) : "";
|
|
201
|
+
}
|
|
202
|
+
if (cell.h) {
|
|
203
|
+
html = cell.h;
|
|
204
|
+
text = typeof cell.v === "string" ? cell.v : "";
|
|
205
|
+
} else {
|
|
206
|
+
text =
|
|
207
|
+
cell.w ||
|
|
208
|
+
(XLSX.utils.format_cell(cell), cell.w as string | undefined) ||
|
|
209
|
+
"";
|
|
210
|
+
html = escapeHtml(text);
|
|
211
|
+
}
|
|
212
|
+
if (!html) return "";
|
|
213
|
+
const target = cell.l?.Target;
|
|
214
|
+
if (target && target.charAt(0) !== "#") {
|
|
215
|
+
return markupCellContent(`<a href="${escapeHtml(target)}">${html}</a>`);
|
|
216
|
+
}
|
|
217
|
+
return escapeHtml(text) === html
|
|
218
|
+
? plainCellContent(text)
|
|
219
|
+
: markupCellContent(html);
|
|
220
|
+
};
|
|
221
|
+
|
|
222
|
+
interface MergeSpan {
|
|
223
|
+
rowspan: number;
|
|
224
|
+
colspan: number;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Merge lookup as `make_html_row` resolves it: the first merge (in sheet
|
|
229
|
+
* order) containing a cell decides; its top-left cell carries the span and
|
|
230
|
+
* the other covered cells are skipped.
|
|
231
|
+
*/
|
|
232
|
+
const mergeLookup = (
|
|
233
|
+
merges: XLSX.Range[]
|
|
234
|
+
): ((row: number, col: number) => MergeSpan | "covered" | undefined) => {
|
|
235
|
+
if (merges.length === 0) return () => undefined;
|
|
236
|
+
const byCell = new Map<string, MergeSpan | "covered">();
|
|
237
|
+
for (let index = merges.length - 1; index >= 0; index -= 1) {
|
|
238
|
+
const merge = merges[index];
|
|
239
|
+
if (!merge) continue;
|
|
240
|
+
for (let row = merge.s.r; row <= merge.e.r; row += 1) {
|
|
241
|
+
for (let col = merge.s.c; col <= merge.e.c; col += 1) {
|
|
242
|
+
byCell.set(
|
|
243
|
+
`${row}:${col}`,
|
|
244
|
+
row === merge.s.r && col === merge.s.c
|
|
245
|
+
? {
|
|
246
|
+
rowspan: merge.e.r - merge.s.r + 1,
|
|
247
|
+
colspan: merge.e.c - merge.s.c + 1,
|
|
248
|
+
}
|
|
249
|
+
: "covered"
|
|
250
|
+
);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
return (row, col) => byCell.get(`${row}:${col}`);
|
|
255
|
+
};
|
|
256
|
+
|
|
257
|
+
const readCell = (
|
|
258
|
+
sheet: WorkSheet,
|
|
259
|
+
row: number,
|
|
260
|
+
col: number
|
|
261
|
+
): CellObject | undefined => {
|
|
262
|
+
const dense = (sheet as { "!data"?: CellObject[][] })["!data"];
|
|
263
|
+
if (dense) return dense[row]?.[col];
|
|
264
|
+
return sheet[XLSX.utils.encode_cell({ r: row, c: col })] as
|
|
265
|
+
| CellObject
|
|
266
|
+
| undefined;
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Markdown for one worksheet with a `!ref`, trimmed as markitdown trims it.
|
|
271
|
+
* Output matches `turndown(sheet_to_html(sheet))` for the same sheet.
|
|
272
|
+
*/
|
|
273
|
+
export function sheetToMarkdown(sheet: WorkSheet): string {
|
|
274
|
+
// Memoized markup is per sheet: the file processor reuses one worker across
|
|
275
|
+
// files, so a process-wide cache would retain every earlier workbook.
|
|
276
|
+
markupCache.clear();
|
|
277
|
+
const range = XLSX.utils.decode_range(sheet["!ref"] ?? "A1");
|
|
278
|
+
const mergeAt = mergeLookup(sheet["!merges"] ?? []);
|
|
279
|
+
const rows: string[] = [];
|
|
280
|
+
let columnCount = 0;
|
|
281
|
+
let onlyCell: string | null = null;
|
|
282
|
+
let firstRowCells = 0;
|
|
283
|
+
for (let row = range.s.r; row <= range.e.r; row += 1) {
|
|
284
|
+
const cells: string[] = [];
|
|
285
|
+
let cellCount = 0;
|
|
286
|
+
for (let col = range.s.c; col <= range.e.c; col += 1) {
|
|
287
|
+
const merge = mergeAt(row, col);
|
|
288
|
+
if (merge === "covered") continue;
|
|
289
|
+
const content = cellContent(readCell(sheet, row, col));
|
|
290
|
+
cells.push(formatCell(content, cellCount === 0, merge?.colspan ?? 1));
|
|
291
|
+
cellCount += 1;
|
|
292
|
+
onlyCell = content;
|
|
293
|
+
}
|
|
294
|
+
columnCount = Math.max(columnCount, cellCount);
|
|
295
|
+
if (row === range.s.r) firstRowCells = cellCount;
|
|
296
|
+
// A row whose cells are all covered by merges renders as an empty line,
|
|
297
|
+
// which the GFM plugin collapses away.
|
|
298
|
+
if (cells.length > 0) rows.push(cells.join(""));
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// A one-row table with at most one cell is left as plain content.
|
|
302
|
+
if (range.e.r === range.s.r && firstRowCells <= 1) {
|
|
303
|
+
return (onlyCell ?? "").trim();
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
const secondLine = rows[1] ?? "";
|
|
307
|
+
const header =
|
|
308
|
+
columnCount && !/\| :?---/.test(secondLine)
|
|
309
|
+
? `|${" |".repeat(columnCount)}\n|${" --- |".repeat(columnCount)}\n`
|
|
310
|
+
: "";
|
|
311
|
+
return `${header}${rows.join("\n")}`.trim();
|
|
312
|
+
}
|
|
@@ -8,7 +8,9 @@ import type { Converter, ConvertInput, ConvertResult } from "../types";
|
|
|
8
8
|
import { basenameWithoutExt } from "../path";
|
|
9
9
|
import { NATIVE_VERSIONS } from "../versions";
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
/** Converter id of Markdown sources, the only documents with link edges. */
|
|
12
|
+
export const MARKDOWN_CONVERTER_ID = "native/markdown" as const;
|
|
13
|
+
const CONVERTER_ID = MARKDOWN_CONVERTER_ID;
|
|
12
14
|
const CONVERTER_VERSION = NATIVE_VERSIONS.markdown;
|
|
13
15
|
|
|
14
16
|
/** UTF-8 BOM character */
|
|
@@ -10,7 +10,7 @@ import type {
|
|
|
10
10
|
RecordAdapter,
|
|
11
11
|
} from "./types";
|
|
12
12
|
|
|
13
|
-
import { unsupportedError } from "./errors";
|
|
13
|
+
import { adapterError, unsupportedError } from "./errors";
|
|
14
14
|
|
|
15
15
|
export class ConverterRegistry {
|
|
16
16
|
private readonly converters: Converter[] = [];
|
|
@@ -71,13 +71,48 @@ export class ConverterRegistry {
|
|
|
71
71
|
}
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
+
/**
|
|
75
|
+
* Load an adapter module; if it cannot load in this runtime, stand in a
|
|
76
|
+
* converter that fails its file types with ADAPTER_FAILURE and the reason.
|
|
77
|
+
*/
|
|
78
|
+
export async function loadAdapter(
|
|
79
|
+
load: () => Promise<Converter>,
|
|
80
|
+
id: string,
|
|
81
|
+
extensions: readonly string[],
|
|
82
|
+
mimes: readonly string[]
|
|
83
|
+
): Promise<Converter> {
|
|
84
|
+
try {
|
|
85
|
+
return await load();
|
|
86
|
+
} catch (cause) {
|
|
87
|
+
const reason = cause instanceof Error ? cause.message : String(cause);
|
|
88
|
+
return {
|
|
89
|
+
id,
|
|
90
|
+
version: "unavailable",
|
|
91
|
+
// Same matching as the real adapter, so a sniffed MIME without an
|
|
92
|
+
// extension still reports ADAPTER_FAILURE, not UNSUPPORTED.
|
|
93
|
+
canHandle: (mime, ext) =>
|
|
94
|
+
extensions.includes(ext) || mimes.includes(mime),
|
|
95
|
+
convert: (input) =>
|
|
96
|
+
Promise.resolve({
|
|
97
|
+
ok: false,
|
|
98
|
+
error: adapterError(
|
|
99
|
+
input,
|
|
100
|
+
id,
|
|
101
|
+
`${id} is unavailable in this build: ${reason}`
|
|
102
|
+
),
|
|
103
|
+
}),
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
74
108
|
/**
|
|
75
109
|
* Create the default registry with all MVP converters.
|
|
76
110
|
* Priority order per PRD §8.6:
|
|
77
111
|
* 1. native/markdown - handles .md
|
|
78
112
|
* 2. native/plaintext - handles .txt
|
|
79
|
-
* 3. adapter/
|
|
80
|
-
* 4. adapter/
|
|
113
|
+
* 3. adapter/xlsx - handles .xlsx (linear SheetJS -> Markdown tables)
|
|
114
|
+
* 4. adapter/markitdown-ts - handles .pdf, .docx
|
|
115
|
+
* 5. adapter/officeparser - handles .pptx
|
|
81
116
|
*/
|
|
82
117
|
export async function createDefaultRegistry(): Promise<ConverterRegistry> {
|
|
83
118
|
const registry = new ConverterRegistry();
|
|
@@ -85,9 +120,29 @@ export async function createDefaultRegistry(): Promise<ConverterRegistry> {
|
|
|
85
120
|
// Import converters dynamically to avoid circular deps
|
|
86
121
|
const { markdownConverter } = await import("./native/markdown");
|
|
87
122
|
const { plaintextConverter } = await import("./native/plaintext");
|
|
88
|
-
const {
|
|
89
|
-
|
|
90
|
-
|
|
123
|
+
const { xlsxAdapter } = await import("./adapters/xlsx/adapter");
|
|
124
|
+
// The PDF/Office adapters load pdf.js, which cannot initialize where its
|
|
125
|
+
// native canvas binding is missing (a standalone compiled executable). A
|
|
126
|
+
// failed adapter only fails its own file types.
|
|
127
|
+
const markitdownAdapter = await loadAdapter(
|
|
128
|
+
async () =>
|
|
129
|
+
(await import("./adapters/markitdownTs/adapter")).markitdownAdapter,
|
|
130
|
+
"adapter/markitdown-ts",
|
|
131
|
+
[".pdf", ".docx"],
|
|
132
|
+
[
|
|
133
|
+
"application/pdf",
|
|
134
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
135
|
+
]
|
|
136
|
+
);
|
|
137
|
+
const officeparserAdapter = await loadAdapter(
|
|
138
|
+
async () =>
|
|
139
|
+
(await import("./adapters/officeparser/adapter")).officeparserAdapter,
|
|
140
|
+
"adapter/officeparser",
|
|
141
|
+
[".pptx"],
|
|
142
|
+
[
|
|
143
|
+
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
|
144
|
+
]
|
|
145
|
+
);
|
|
91
146
|
const { jsonlAdapter } = await import("./adapters/jsonl/adapter");
|
|
92
147
|
const { transcriptAdapter } = await import("./adapters/transcript/adapter");
|
|
93
148
|
const { emailRecordAdapter } = await import("./adapters/email/adapter");
|
|
@@ -98,6 +153,7 @@ export async function createDefaultRegistry(): Promise<ConverterRegistry> {
|
|
|
98
153
|
// Register in priority order
|
|
99
154
|
registry.register(markdownConverter);
|
|
100
155
|
registry.register(plaintextConverter);
|
|
156
|
+
registry.register(xlsxAdapter);
|
|
101
157
|
registry.register(markitdownAdapter);
|
|
102
158
|
registry.register(officeparserAdapter);
|
|
103
159
|
registry.registerRecordAdapter(jsonlAdapter);
|
|
@@ -18,5 +18,7 @@ export const NATIVE_VERSIONS = {
|
|
|
18
18
|
*/
|
|
19
19
|
export const ADAPTER_VERSIONS = {
|
|
20
20
|
"markitdown-ts": "0.0.10+xlsx.0.20.3",
|
|
21
|
+
/** Direct SheetJS -> Markdown tables (fn-198); output matches markitdown. */
|
|
22
|
+
xlsx: "sheetjs-0.20.3+md.1",
|
|
21
23
|
officeparser: "7.8.0+pdfjs.6.3.289",
|
|
22
24
|
} as const;
|
|
@@ -144,7 +144,7 @@ export const evaluateFreshnessAudit = (
|
|
|
144
144
|
document,
|
|
145
145
|
kind: "stale-indexed-revision",
|
|
146
146
|
message: document.lastErrorCode
|
|
147
|
-
?
|
|
147
|
+
? `The latest indexing attempt recorded an error (${document.lastErrorCode})`
|
|
148
148
|
: "The document has no indexed-at revision evidence",
|
|
149
149
|
detail: {
|
|
150
150
|
indexedAt: document.indexedAt,
|
package/src/core/audit-links.ts
CHANGED
|
@@ -16,8 +16,9 @@ import {
|
|
|
16
16
|
/**
|
|
17
17
|
* 1.1: workspace-wide resolution and reference/resolution/scope evidence.
|
|
18
18
|
* 1.2: `links.outside-index` for workspace files that are not indexed.
|
|
19
|
+
* 1.3: relative Markdown links to such files are `outside-index` too.
|
|
19
20
|
*/
|
|
20
|
-
export const LINK_AUDIT_RULE_VERSION = "1.
|
|
21
|
+
export const LINK_AUDIT_RULE_VERSION = "1.3" as const;
|
|
21
22
|
/** Default per-rule cap; runs pass the effective `maxFindings` instead. */
|
|
22
23
|
export const LINK_AUDIT_MAX_FINDINGS_PER_RULE = 1000;
|
|
23
24
|
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Existence-only classification of unresolved workspace wiki links
|
|
3
|
-
* target is a file inside the link workspace
|
|
4
|
-
* (an attachment,
|
|
2
|
+
* Existence-only classification of unresolved workspace wiki links and
|
|
3
|
+
* relative Markdown links whose target is a file inside the link workspace
|
|
4
|
+
* that is not an indexed document (an attachment, a script beside a note, or
|
|
5
|
+
* a note in an unindexed or excluded folder). Obsidian
|
|
5
6
|
* resolves such links, so the link audit reports them as `outside-index`
|
|
6
7
|
* instead of unresolved. Only file names are listed: no file is opened,
|
|
7
8
|
* indexed, or returned, and no graph edge is created.
|
|
@@ -18,7 +19,10 @@ import { readdir, realpath, stat } from "node:fs/promises";
|
|
|
18
19
|
// node:path join: platform path algebra; no Bun equivalent.
|
|
19
20
|
import { join } from "node:path";
|
|
20
21
|
|
|
21
|
-
import type {
|
|
22
|
+
import type {
|
|
23
|
+
AuditLinkSnapshot,
|
|
24
|
+
AuditLinkSnapshotLink,
|
|
25
|
+
} from "../store/sqlite/graph-link-resolver";
|
|
22
26
|
|
|
23
27
|
import {
|
|
24
28
|
createWorkspaceFileMatcher,
|
|
@@ -26,6 +30,30 @@ import {
|
|
|
26
30
|
} from "../store/sqlite/workspace-link-resolver";
|
|
27
31
|
import { pathContains, placeDocument } from "./link-workspace";
|
|
28
32
|
|
|
33
|
+
/** Path identity for exact-path matching: NFC, case-insensitive. */
|
|
34
|
+
const pathKey = (path: string): string => path.normalize("NFC").toLowerCase();
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* A Markdown link names the file's exact path; as with wiki links, `.md` is
|
|
38
|
+
* optional, so `[x](Note)` matches an unindexed `Note.md`. An indexed
|
|
39
|
+
* `Note.md` is not outside the index, so that link keeps its status. Indexed
|
|
40
|
+
* paths are compared under the same key as listed files (NFC, Unicode
|
|
41
|
+
* lowercase), never SQLite's ASCII-only NOCASE.
|
|
42
|
+
*/
|
|
43
|
+
const markdownTargetExists = (
|
|
44
|
+
link: AuditLinkSnapshotLink,
|
|
45
|
+
targetPath: string,
|
|
46
|
+
paths: ReadonlySet<string> | undefined,
|
|
47
|
+
indexedKeys: (collection: string) => ReadonlySet<string>
|
|
48
|
+
): boolean => {
|
|
49
|
+
const key = pathKey(targetPath);
|
|
50
|
+
if (paths?.has(key) === true) return true;
|
|
51
|
+
if (key.endsWith(".md") || paths?.has(`${key}.md`) !== true) return false;
|
|
52
|
+
return !indexedKeys(link.targetCollection).has(
|
|
53
|
+
pathKey(`${link.targetRefNorm}.md`)
|
|
54
|
+
);
|
|
55
|
+
};
|
|
56
|
+
|
|
29
57
|
/** Upper bound of files listed per workspace; beyond it the listing is partial. */
|
|
30
58
|
export const WORKSPACE_FILE_LISTING_MAX_FILES = 200_000;
|
|
31
59
|
|
|
@@ -145,8 +173,10 @@ export const listWorkspaceFiles = async (
|
|
|
145
173
|
};
|
|
146
174
|
|
|
147
175
|
/**
|
|
148
|
-
* Mark unresolved plain wiki links
|
|
149
|
-
* workspace whose target exists as a workspace file.
|
|
176
|
+
* Mark unresolved plain wiki links and relative Markdown links of audited
|
|
177
|
+
* documents inside a link workspace whose target exists as a workspace file.
|
|
178
|
+
* A wiki link matches under workspace resolution rules; a Markdown link must
|
|
179
|
+
* name the file's exact path (NFC, case-insensitive), `.md` optional. Each involved workspace
|
|
150
180
|
* is listed once per call. A failed or partial listing never marks a link
|
|
151
181
|
* that is not in it; the snapshot then carries one diagnostic.
|
|
152
182
|
*/
|
|
@@ -163,8 +193,8 @@ export const markOutsideIndexLinks = async (
|
|
|
163
193
|
.filter(
|
|
164
194
|
({ link }) =>
|
|
165
195
|
link.resolved === null &&
|
|
166
|
-
link.linkType === "
|
|
167
|
-
|
|
196
|
+
(link.linkType === "markdown" ||
|
|
197
|
+
(link.linkType === "wiki" && link.explicitCollection !== true)) &&
|
|
168
198
|
(audited === null || audited.has(link.sourceId))
|
|
169
199
|
);
|
|
170
200
|
if (candidates.length === 0) return snapshot;
|
|
@@ -172,19 +202,34 @@ export const markOutsideIndexLinks = async (
|
|
|
172
202
|
const documentRelPaths = new Map(
|
|
173
203
|
snapshot.documents.map((document) => [document.id, document.relPath])
|
|
174
204
|
);
|
|
175
|
-
|
|
205
|
+
type PlacedLink = {
|
|
206
|
+
link: AuditLinkSnapshot["links"][number];
|
|
207
|
+
index: number;
|
|
208
|
+
key: string;
|
|
209
|
+
} & ({ sourcePath: string } | { targetPath: string });
|
|
210
|
+
const placed = candidates.flatMap(({ link, index }): PlacedLink[] => {
|
|
211
|
+
const membership = memberships.get(link.sourceCollection);
|
|
176
212
|
const placement = placeDocument(
|
|
177
|
-
|
|
213
|
+
membership,
|
|
178
214
|
documentRelPaths.get(link.sourceId) ?? link.sourceRelPath
|
|
179
215
|
);
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
216
|
+
if (placement.key === null) return [];
|
|
217
|
+
if (link.linkType === "wiki") {
|
|
218
|
+
return [{ link, index, key: placement.key, sourcePath: placement.path }];
|
|
219
|
+
}
|
|
220
|
+
// A Markdown target is stored collection-relative; place it the same way.
|
|
221
|
+
const target = placeDocument(membership, link.targetRefNorm);
|
|
222
|
+
return target.key === placement.key
|
|
223
|
+
? [{ link, index, key: placement.key, targetPath: target.path }]
|
|
224
|
+
: [];
|
|
183
225
|
});
|
|
184
226
|
if (placed.length === 0) return snapshot;
|
|
185
|
-
const
|
|
227
|
+
const listings = new Map<
|
|
186
228
|
string,
|
|
187
|
-
|
|
229
|
+
{
|
|
230
|
+
matches: ReturnType<typeof createWorkspaceFileMatcher>;
|
|
231
|
+
paths: Set<string>;
|
|
232
|
+
}
|
|
188
233
|
>();
|
|
189
234
|
const maxFiles = options.maxFiles ?? WORKSPACE_FILE_LISTING_MAX_FILES;
|
|
190
235
|
let incomplete = 0;
|
|
@@ -194,12 +239,40 @@ export const markOutsideIndexLinks = async (
|
|
|
194
239
|
signal: options.signal,
|
|
195
240
|
});
|
|
196
241
|
if (!listing.complete) incomplete += 1;
|
|
197
|
-
|
|
242
|
+
listings.set(key, {
|
|
243
|
+
matches: createWorkspaceFileMatcher(listing.files),
|
|
244
|
+
paths: new Set(listing.files.map(pathKey)),
|
|
245
|
+
});
|
|
198
246
|
}
|
|
247
|
+
const indexedByCollection = new Map<string, Set<string>>();
|
|
248
|
+
const indexedKeys = (collection: string): ReadonlySet<string> => {
|
|
249
|
+
let keys = indexedByCollection.get(collection);
|
|
250
|
+
if (!keys) {
|
|
251
|
+
keys = new Set(
|
|
252
|
+
db
|
|
253
|
+
.query<{ relPath: string }, [string]>(
|
|
254
|
+
"SELECT rel_path AS relPath FROM documents WHERE active = 1 AND collection = ?"
|
|
255
|
+
)
|
|
256
|
+
.all(collection)
|
|
257
|
+
.map(({ relPath }) => pathKey(relPath))
|
|
258
|
+
);
|
|
259
|
+
indexedByCollection.set(collection, keys);
|
|
260
|
+
}
|
|
261
|
+
return keys;
|
|
262
|
+
};
|
|
199
263
|
const links = [...snapshot.links];
|
|
200
264
|
for (const entry of placed) {
|
|
201
|
-
const
|
|
202
|
-
|
|
265
|
+
const listing = listings.get(entry.key);
|
|
266
|
+
const exists =
|
|
267
|
+
"targetPath" in entry
|
|
268
|
+
? markdownTargetExists(
|
|
269
|
+
entry.link,
|
|
270
|
+
entry.targetPath,
|
|
271
|
+
listing?.paths,
|
|
272
|
+
indexedKeys
|
|
273
|
+
)
|
|
274
|
+
: listing?.matches(entry.link.targetRefNorm, entry.sourcePath) === true;
|
|
275
|
+
if (exists) {
|
|
203
276
|
links[entry.index] = { ...entry.link, outsideIndex: true };
|
|
204
277
|
}
|
|
205
278
|
}
|
package/src/core/change-diff.ts
CHANGED
|
@@ -12,7 +12,7 @@ import type {
|
|
|
12
12
|
|
|
13
13
|
import { parseFrontmatter } from "../ingestion/frontmatter";
|
|
14
14
|
import { buildLineOffsets } from "../ingestion/position";
|
|
15
|
-
import { getExcludedRanges } from "../ingestion/strip";
|
|
15
|
+
import { type ExcludedRange, getExcludedRanges } from "../ingestion/strip";
|
|
16
16
|
import { normalizeMarkdownPath, normalizeWikiName, parseLinks } from "./links";
|
|
17
17
|
import { extractSections } from "./sections";
|
|
18
18
|
|
|
@@ -104,15 +104,27 @@ const extractTypedEdges = (markdown: string): string[] => {
|
|
|
104
104
|
return [...new Set(values)].sort();
|
|
105
105
|
};
|
|
106
106
|
|
|
107
|
+
export interface DocumentStructureOptions {
|
|
108
|
+
/** Only Markdown sources carry links (matches link extraction at sync). */
|
|
109
|
+
markdownSource: boolean;
|
|
110
|
+
excludedRanges?: ExcludedRange[];
|
|
111
|
+
}
|
|
112
|
+
|
|
107
113
|
export const extractDocumentStructure = (
|
|
108
114
|
markdown: string,
|
|
109
115
|
relPath: string,
|
|
110
|
-
dateFields: Readonly<Record<string, string>> | null | undefined
|
|
116
|
+
dateFields: Readonly<Record<string, string>> | null | undefined,
|
|
117
|
+
{ markdownSource, excludedRanges }: DocumentStructureOptions
|
|
111
118
|
): DocumentStructureSnapshot => {
|
|
112
|
-
const
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
119
|
+
const links = markdownSource
|
|
120
|
+
? parseLinks(
|
|
121
|
+
markdown,
|
|
122
|
+
buildLineOffsets(markdown),
|
|
123
|
+
excludedRanges ?? getExcludedRanges(markdown)
|
|
124
|
+
)
|
|
125
|
+
.map((link) => linkSummary(link, relPath))
|
|
126
|
+
.filter((value): value is string => value !== null)
|
|
127
|
+
: [];
|
|
116
128
|
|
|
117
129
|
return {
|
|
118
130
|
headings: withOccurrences(
|