@gmickel/gno 2.8.2 → 2.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v2.8.2.zip → gno-browser-clipper-v2.8.4.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.8.4.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +9 -1
- package/spec/cli.md +77 -6
- package/spec/mcp.md +5 -6
- package/spec/output-schemas/doctor.schema.json +10 -4
- package/spec/output-schemas/status.schema.json +4 -2
- package/src/cli/commands/daemon.ts +7 -2
- package/src/cli/commands/doctor-activation.ts +38 -11
- package/src/cli/commands/doctor.ts +46 -25
- package/src/cli/commands/index-cmd.ts +10 -2
- package/src/cli/commands/links.ts +5 -22
- package/src/cli/commands/shared.ts +26 -1
- package/src/cli/commands/status.ts +76 -4
- package/src/cli/commands/update.ts +6 -1
- package/src/cli/commands/vec.ts +42 -2
- package/src/cli/program.ts +2 -2
- package/src/config/types.ts +19 -0
- package/src/converters/adapters/markitdownTs/adapter.ts +1 -52
- package/src/converters/adapters/shared/ooxml-protection.ts +58 -0
- package/src/converters/adapters/xlsx/adapter.ts +132 -0
- package/src/converters/adapters/xlsx/sheet-markdown.ts +312 -0
- package/src/converters/registry.ts +62 -6
- package/src/converters/versions.ts +2 -0
- package/src/core/activation-empty.ts +16 -0
- package/src/core/activation-status.ts +23 -5
- package/src/core/audit-freshness.ts +1 -1
- package/src/core/audit-outside-index.ts +48 -3
- package/src/core/change-diff.ts +16 -4
- package/src/core/link-destination-parse.ts +21 -4
- package/src/core/link-inventory-markdown.ts +23 -3
- package/src/core/links.ts +127 -31
- package/src/core/network-boundary-inventory.ts +18 -0
- package/src/index.ts +10 -0
- package/src/ingestion/document-metadata.ts +256 -0
- package/src/ingestion/file-child-env.ts +9 -0
- package/src/ingestion/file-child.ts +106 -0
- package/src/ingestion/file-processor.ts +546 -0
- package/src/ingestion/frontmatter.ts +6 -2
- package/src/ingestion/index.ts +1 -0
- package/src/ingestion/prepare-convert.ts +32 -0
- package/src/ingestion/prepare-file.ts +272 -0
- package/src/ingestion/prepare-worker.ts +43 -0
- package/src/ingestion/record-container.ts +7 -2
- package/src/ingestion/strip.ts +136 -31
- package/src/ingestion/sync-options.ts +7 -1
- package/src/ingestion/sync.ts +207 -400
- package/src/ingestion/types.ts +18 -0
- package/src/llm/native-worker/embedding-identity.ts +15 -3
- package/src/mcp/http-transport.ts +11 -0
- package/src/mcp/tools/links.ts +10 -38
- package/src/serve/activation-health.ts +25 -7
- package/src/serve/public/components/BootstrapStatus.tsx +9 -4
- package/src/serve/resident-request.ts +19 -0
- package/src/serve/resident-runtime.ts +84 -8
- package/src/serve/routes/links.ts +6 -24
- package/src/serve/server.ts +3 -2
- package/src/serve/status.ts +27 -8
- package/src/sessions/config-refresh.ts +3 -2
- package/src/store/sqlite/graph-similarity.ts +5 -13
- package/src/store/vector/partition-index.ts +100 -0
- package/src/store/vector/stored-vectors.ts +20 -15
- package/browser-extension/artifacts/gno-browser-clipper-v2.8.2.zip.sha256 +0 -1
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worksheet to Markdown table, linear in cells (fn-198).
|
|
3
|
+
*
|
|
4
|
+
* Reproduces the Markdown the markitdown-ts chain produces for a worksheet
|
|
5
|
+
* (SheetJS `sheet_to_html` -> jsdom -> turndown + Joplin GFM tables) without
|
|
6
|
+
* building a DOM. That chain is O(columns x rows^2) and uses about 20 KB per
|
|
7
|
+
* cell: the GFM plugin's column-alignment scan re-reads `table.rows.length`
|
|
8
|
+
* through a jsdom proxy for every row of every column. Here each cell is
|
|
9
|
+
* rendered once from the sheet data:
|
|
10
|
+
*
|
|
11
|
+
* - cell text is chosen as `sheet_to_html` does (`h`, or escaped `w`),
|
|
12
|
+
* - plain text goes through turndown's whitespace collapsing and Markdown
|
|
13
|
+
* escaping, per line (a line break starts a new text node),
|
|
14
|
+
* - cells whose HTML carries markup (rich-text runs, hyperlinks) are
|
|
15
|
+
* converted by turndown itself with the same rules, one cell at a time,
|
|
16
|
+
* - rows, the empty heading row, `---` borders (SheetJS never sets an
|
|
17
|
+
* alignment), merges (`colspan` padding, covered cells skipped) and the
|
|
18
|
+
* single-cell-table case follow the GFM plugin.
|
|
19
|
+
*
|
|
20
|
+
* @module src/converters/adapters/xlsx/sheet-markdown
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import type { CellObject, WorkSheet } from "xlsx";
|
|
24
|
+
|
|
25
|
+
// @ts-expect-error -- @joplin/turndown-plugin-gfm ships no types
|
|
26
|
+
import turndownPluginGfm from "@joplin/turndown-plugin-gfm";
|
|
27
|
+
// @ts-expect-error -- turndown ships no types
|
|
28
|
+
import TurndownService from "turndown";
|
|
29
|
+
import * as XLSX from "xlsx";
|
|
30
|
+
|
|
31
|
+
/** The part of turndown's API this module uses. */
|
|
32
|
+
interface Turndown {
|
|
33
|
+
turndown(html: string): string;
|
|
34
|
+
use(plugin: unknown): Turndown;
|
|
35
|
+
addRule(
|
|
36
|
+
key: string,
|
|
37
|
+
rule: {
|
|
38
|
+
filter: string[];
|
|
39
|
+
replacement: (
|
|
40
|
+
content: string,
|
|
41
|
+
node: { getAttribute(name: string): string | null; title: string }
|
|
42
|
+
) => string;
|
|
43
|
+
}
|
|
44
|
+
): Turndown;
|
|
45
|
+
}
|
|
46
|
+
const TurndownConstructor = TurndownService as new (options: {
|
|
47
|
+
headingStyle: "atx";
|
|
48
|
+
}) => Turndown;
|
|
49
|
+
|
|
50
|
+
/** SheetJS `escapehtml` (xlsx 0.20.3), used to recognise plain cell HTML. */
|
|
51
|
+
const HTML_ENCODINGS: Record<string, string> = {
|
|
52
|
+
'"': """,
|
|
53
|
+
"'": "'",
|
|
54
|
+
">": ">",
|
|
55
|
+
"<": "<",
|
|
56
|
+
"&": "&",
|
|
57
|
+
};
|
|
58
|
+
const escapeHtml = (text: string): string =>
|
|
59
|
+
text
|
|
60
|
+
.replace(/[&<>'"]/g, (char) => HTML_ENCODINGS[char] ?? char)
|
|
61
|
+
.replace(/\n/g, "<br/>")
|
|
62
|
+
.replace(
|
|
63
|
+
// oxlint-disable-next-line no-control-regex -- SheetJS htmlcharegex: C0 controls become numeric references
|
|
64
|
+
/[\u0000-\u001f]/g,
|
|
65
|
+
(char) => `&#x${`000${char.charCodeAt(0).toString(16)}`.slice(-4)};`
|
|
66
|
+
);
|
|
67
|
+
|
|
68
|
+
/** turndown 7.2 `escapeMarkdown`, applied to each text node. */
|
|
69
|
+
const MARKDOWN_ESCAPES: Array<[RegExp, string]> = [
|
|
70
|
+
[/\\/g, "\\\\"],
|
|
71
|
+
[/\*/g, "\\*"],
|
|
72
|
+
[/^-/g, "\\-"],
|
|
73
|
+
[/^\+ /g, "\\+ "],
|
|
74
|
+
[/^(=+)/g, "\\$1"],
|
|
75
|
+
[/^(#{1,6}) /g, "\\$1 "],
|
|
76
|
+
[/`/g, "\\`"],
|
|
77
|
+
[/^~~~/g, "\\~~~"],
|
|
78
|
+
[/\[/g, "\\["],
|
|
79
|
+
[/\]/g, "\\]"],
|
|
80
|
+
[/^>/g, "\\>"],
|
|
81
|
+
[/_/g, "\\_"],
|
|
82
|
+
[/^(\d+)\. /g, "$1\\. "],
|
|
83
|
+
];
|
|
84
|
+
/** Cheap pre-test: text that no escape rule can touch skips the chain. */
|
|
85
|
+
const MAY_NEED_ESCAPE = /[\\*`[\]_]|^[-+=#~>\d]/;
|
|
86
|
+
|
|
87
|
+
const escapeMarkdown = (text: string): string =>
|
|
88
|
+
MAY_NEED_ESCAPE.test(text)
|
|
89
|
+
? MARKDOWN_ESCAPES.reduce(
|
|
90
|
+
(value, [pattern, replacement]) => value.replace(pattern, replacement),
|
|
91
|
+
text
|
|
92
|
+
)
|
|
93
|
+
: text;
|
|
94
|
+
|
|
95
|
+
/** turndown's `<br>` replacement with the default `br: " "` option. */
|
|
96
|
+
const LINE_BREAK = " \n";
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Plain cell text as turndown renders it inside a table cell: each line is a
|
|
100
|
+
* text node whose whitespace runs collapse to one space, with the leading
|
|
101
|
+
* and trailing space dropped (cells and line breaks are block boundaries),
|
|
102
|
+
* escaped on its own; every line break renders, empty lines included.
|
|
103
|
+
*/
|
|
104
|
+
const plainCellContent = (text: string): string => {
|
|
105
|
+
const lines: string[] = [];
|
|
106
|
+
for (const raw of text.split("\n")) {
|
|
107
|
+
// The HTML parser turns the escaped NUL (�) into U+FFFD.
|
|
108
|
+
const line = raw
|
|
109
|
+
.replaceAll("\u0000", "\uFFFD")
|
|
110
|
+
.replace(/[ \r\n\t]+/g, " ")
|
|
111
|
+
.replace(/^ | $/g, "");
|
|
112
|
+
lines.push(line ? escapeMarkdown(line) : "");
|
|
113
|
+
}
|
|
114
|
+
return lines.join(LINE_BREAK);
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
let markupTurndown: Turndown | null = null;
|
|
118
|
+
const markupCache = new Map<string, string>();
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Cell HTML with markup (rich-text runs, a hyperlink), converted by turndown
|
|
122
|
+
* with markitdown's rules. A paragraph gives the same block context as a
|
|
123
|
+
* table cell.
|
|
124
|
+
*/
|
|
125
|
+
const markupCellContent = (html: string): string => {
|
|
126
|
+
const cached = markupCache.get(html);
|
|
127
|
+
if (cached !== undefined) return cached;
|
|
128
|
+
markupTurndown ??= createMarkitdownTurndown();
|
|
129
|
+
const content = markupTurndown.turndown(`<p>${html}</p>`);
|
|
130
|
+
markupCache.set(html, content);
|
|
131
|
+
return content;
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
/** markitdown-ts `CustomTurnDown` configuration (vendored dist). */
|
|
135
|
+
function createMarkitdownTurndown(): Turndown {
|
|
136
|
+
const service = new TurndownConstructor({ headingStyle: "atx" });
|
|
137
|
+
service.use((turndownPluginGfm as { gfm: unknown }).gfm);
|
|
138
|
+
service.addRule("anchor tags", {
|
|
139
|
+
filter: ["a"],
|
|
140
|
+
replacement(content, node) {
|
|
141
|
+
if (content === "") return "";
|
|
142
|
+
const prefix = content[0] === " " ? " " : "";
|
|
143
|
+
const suffix = content.at(-1) === " " ? " " : "";
|
|
144
|
+
const text = content.trim().replace(/\n\n.*/g, "");
|
|
145
|
+
if (text === "") return "";
|
|
146
|
+
const href = node.getAttribute("href");
|
|
147
|
+
let title = node.title;
|
|
148
|
+
if (href) {
|
|
149
|
+
try {
|
|
150
|
+
const parsed = new URL(href);
|
|
151
|
+
if (!["https:", "http:", "file:"].includes(parsed.protocol)) {
|
|
152
|
+
return `${prefix}${text}${suffix}`;
|
|
153
|
+
}
|
|
154
|
+
} catch {
|
|
155
|
+
if (!/^https?:|^file:/.test(href)) {
|
|
156
|
+
return `${prefix}[${text}](${href} "${title}")${suffix}`;
|
|
157
|
+
}
|
|
158
|
+
return `${prefix}${text}${suffix}`;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if (text.replace(/\\_/g, "_") === href && !title) return `<${href}>`;
|
|
162
|
+
if (!title && href) title = href;
|
|
163
|
+
const titlePart = title ? ` "${title}"` : "";
|
|
164
|
+
return `${prefix}[${text}](${href}${titlePart})${suffix}`;
|
|
165
|
+
},
|
|
166
|
+
});
|
|
167
|
+
return service;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Joplin GFM `cell()`: trim, `<br>` for newlines, escape pipes, pad. */
|
|
171
|
+
const formatCell = (content: string, first: boolean, colspan: number) => {
|
|
172
|
+
let value = content
|
|
173
|
+
.trim()
|
|
174
|
+
.replace(/\n\r/g, "<br>")
|
|
175
|
+
.replace(/\n/g, "<br>")
|
|
176
|
+
.replace(/\|+/g, "\\|");
|
|
177
|
+
while (value.length < 3) value += " ";
|
|
178
|
+
for (let extra = 1; extra < colspan; extra += 1) value += " | ";
|
|
179
|
+
return `${first ? "| " : " "}${value} |`;
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
/** SheetJS error text for non-finite numbers (`make_html_row`). */
|
|
183
|
+
const nonFiniteText = (value: number): string =>
|
|
184
|
+
Number.isNaN(value) ? "#NUM!" : "#DIV/0!";
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Cell Markdown content (before `formatCell`), from the HTML `make_html_row`
|
|
188
|
+
* (xlsx 0.20.3) writes for it: `h` if present, else the escaped formatted
|
|
189
|
+
* text, wrapped in a link for an external hyperlink. HTML that is exactly
|
|
190
|
+
* the escaped text is rendered from the text; any other HTML goes through
|
|
191
|
+
* turndown.
|
|
192
|
+
*/
|
|
193
|
+
const cellContent = (cell: CellObject | undefined): string => {
|
|
194
|
+
if (!cell || cell.v == null) return "";
|
|
195
|
+
let text: string;
|
|
196
|
+
let html: string;
|
|
197
|
+
if (cell.t === "n" && !Number.isFinite(cell.v as number)) {
|
|
198
|
+
// SheetJS swaps in an error cell, which has no `h` and no hyperlink.
|
|
199
|
+
text = nonFiniteText(cell.v as number);
|
|
200
|
+
return text ? plainCellContent(text) : "";
|
|
201
|
+
}
|
|
202
|
+
if (cell.h) {
|
|
203
|
+
html = cell.h;
|
|
204
|
+
text = typeof cell.v === "string" ? cell.v : "";
|
|
205
|
+
} else {
|
|
206
|
+
text =
|
|
207
|
+
cell.w ||
|
|
208
|
+
(XLSX.utils.format_cell(cell), cell.w as string | undefined) ||
|
|
209
|
+
"";
|
|
210
|
+
html = escapeHtml(text);
|
|
211
|
+
}
|
|
212
|
+
if (!html) return "";
|
|
213
|
+
const target = cell.l?.Target;
|
|
214
|
+
if (target && target.charAt(0) !== "#") {
|
|
215
|
+
return markupCellContent(`<a href="${escapeHtml(target)}">${html}</a>`);
|
|
216
|
+
}
|
|
217
|
+
return escapeHtml(text) === html
|
|
218
|
+
? plainCellContent(text)
|
|
219
|
+
: markupCellContent(html);
|
|
220
|
+
};
|
|
221
|
+
|
|
222
|
+
interface MergeSpan {
|
|
223
|
+
rowspan: number;
|
|
224
|
+
colspan: number;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Merge lookup as `make_html_row` resolves it: the first merge (in sheet
|
|
229
|
+
* order) containing a cell decides; its top-left cell carries the span and
|
|
230
|
+
* the other covered cells are skipped.
|
|
231
|
+
*/
|
|
232
|
+
const mergeLookup = (
|
|
233
|
+
merges: XLSX.Range[]
|
|
234
|
+
): ((row: number, col: number) => MergeSpan | "covered" | undefined) => {
|
|
235
|
+
if (merges.length === 0) return () => undefined;
|
|
236
|
+
const byCell = new Map<string, MergeSpan | "covered">();
|
|
237
|
+
for (let index = merges.length - 1; index >= 0; index -= 1) {
|
|
238
|
+
const merge = merges[index];
|
|
239
|
+
if (!merge) continue;
|
|
240
|
+
for (let row = merge.s.r; row <= merge.e.r; row += 1) {
|
|
241
|
+
for (let col = merge.s.c; col <= merge.e.c; col += 1) {
|
|
242
|
+
byCell.set(
|
|
243
|
+
`${row}:${col}`,
|
|
244
|
+
row === merge.s.r && col === merge.s.c
|
|
245
|
+
? {
|
|
246
|
+
rowspan: merge.e.r - merge.s.r + 1,
|
|
247
|
+
colspan: merge.e.c - merge.s.c + 1,
|
|
248
|
+
}
|
|
249
|
+
: "covered"
|
|
250
|
+
);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
return (row, col) => byCell.get(`${row}:${col}`);
|
|
255
|
+
};
|
|
256
|
+
|
|
257
|
+
const readCell = (
|
|
258
|
+
sheet: WorkSheet,
|
|
259
|
+
row: number,
|
|
260
|
+
col: number
|
|
261
|
+
): CellObject | undefined => {
|
|
262
|
+
const dense = (sheet as { "!data"?: CellObject[][] })["!data"];
|
|
263
|
+
if (dense) return dense[row]?.[col];
|
|
264
|
+
return sheet[XLSX.utils.encode_cell({ r: row, c: col })] as
|
|
265
|
+
| CellObject
|
|
266
|
+
| undefined;
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Markdown for one worksheet with a `!ref`, trimmed as markitdown trims it.
|
|
271
|
+
* Output matches `turndown(sheet_to_html(sheet))` for the same sheet.
|
|
272
|
+
*/
|
|
273
|
+
export function sheetToMarkdown(sheet: WorkSheet): string {
|
|
274
|
+
// Memoized markup is per sheet: the file processor reuses one worker across
|
|
275
|
+
// files, so a process-wide cache would retain every earlier workbook.
|
|
276
|
+
markupCache.clear();
|
|
277
|
+
const range = XLSX.utils.decode_range(sheet["!ref"] ?? "A1");
|
|
278
|
+
const mergeAt = mergeLookup(sheet["!merges"] ?? []);
|
|
279
|
+
const rows: string[] = [];
|
|
280
|
+
let columnCount = 0;
|
|
281
|
+
let onlyCell: string | null = null;
|
|
282
|
+
let firstRowCells = 0;
|
|
283
|
+
for (let row = range.s.r; row <= range.e.r; row += 1) {
|
|
284
|
+
const cells: string[] = [];
|
|
285
|
+
let cellCount = 0;
|
|
286
|
+
for (let col = range.s.c; col <= range.e.c; col += 1) {
|
|
287
|
+
const merge = mergeAt(row, col);
|
|
288
|
+
if (merge === "covered") continue;
|
|
289
|
+
const content = cellContent(readCell(sheet, row, col));
|
|
290
|
+
cells.push(formatCell(content, cellCount === 0, merge?.colspan ?? 1));
|
|
291
|
+
cellCount += 1;
|
|
292
|
+
onlyCell = content;
|
|
293
|
+
}
|
|
294
|
+
columnCount = Math.max(columnCount, cellCount);
|
|
295
|
+
if (row === range.s.r) firstRowCells = cellCount;
|
|
296
|
+
// A row whose cells are all covered by merges renders as an empty line,
|
|
297
|
+
// which the GFM plugin collapses away.
|
|
298
|
+
if (cells.length > 0) rows.push(cells.join(""));
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// A one-row table with at most one cell is left as plain content.
|
|
302
|
+
if (range.e.r === range.s.r && firstRowCells <= 1) {
|
|
303
|
+
return (onlyCell ?? "").trim();
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
const secondLine = rows[1] ?? "";
|
|
307
|
+
const header =
|
|
308
|
+
columnCount && !/\| :?---/.test(secondLine)
|
|
309
|
+
? `|${" |".repeat(columnCount)}\n|${" --- |".repeat(columnCount)}\n`
|
|
310
|
+
: "";
|
|
311
|
+
return `${header}${rows.join("\n")}`.trim();
|
|
312
|
+
}
|
|
@@ -10,7 +10,7 @@ import type {
|
|
|
10
10
|
RecordAdapter,
|
|
11
11
|
} from "./types";
|
|
12
12
|
|
|
13
|
-
import { unsupportedError } from "./errors";
|
|
13
|
+
import { adapterError, unsupportedError } from "./errors";
|
|
14
14
|
|
|
15
15
|
export class ConverterRegistry {
|
|
16
16
|
private readonly converters: Converter[] = [];
|
|
@@ -71,13 +71,48 @@ export class ConverterRegistry {
|
|
|
71
71
|
}
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
+
/**
|
|
75
|
+
* Load an adapter module; if it cannot load in this runtime, stand in a
|
|
76
|
+
* converter that fails its file types with ADAPTER_FAILURE and the reason.
|
|
77
|
+
*/
|
|
78
|
+
export async function loadAdapter(
|
|
79
|
+
load: () => Promise<Converter>,
|
|
80
|
+
id: string,
|
|
81
|
+
extensions: readonly string[],
|
|
82
|
+
mimes: readonly string[]
|
|
83
|
+
): Promise<Converter> {
|
|
84
|
+
try {
|
|
85
|
+
return await load();
|
|
86
|
+
} catch (cause) {
|
|
87
|
+
const reason = cause instanceof Error ? cause.message : String(cause);
|
|
88
|
+
return {
|
|
89
|
+
id,
|
|
90
|
+
version: "unavailable",
|
|
91
|
+
// Same matching as the real adapter, so a sniffed MIME without an
|
|
92
|
+
// extension still reports ADAPTER_FAILURE, not UNSUPPORTED.
|
|
93
|
+
canHandle: (mime, ext) =>
|
|
94
|
+
extensions.includes(ext) || mimes.includes(mime),
|
|
95
|
+
convert: (input) =>
|
|
96
|
+
Promise.resolve({
|
|
97
|
+
ok: false,
|
|
98
|
+
error: adapterError(
|
|
99
|
+
input,
|
|
100
|
+
id,
|
|
101
|
+
`${id} is unavailable in this build: ${reason}`
|
|
102
|
+
),
|
|
103
|
+
}),
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
74
108
|
/**
|
|
75
109
|
* Create the default registry with all MVP converters.
|
|
76
110
|
* Priority order per PRD §8.6:
|
|
77
111
|
* 1. native/markdown - handles .md
|
|
78
112
|
* 2. native/plaintext - handles .txt
|
|
79
|
-
* 3. adapter/
|
|
80
|
-
* 4. adapter/
|
|
113
|
+
* 3. adapter/xlsx - handles .xlsx (linear SheetJS -> Markdown tables)
|
|
114
|
+
* 4. adapter/markitdown-ts - handles .pdf, .docx
|
|
115
|
+
* 5. adapter/officeparser - handles .pptx
|
|
81
116
|
*/
|
|
82
117
|
export async function createDefaultRegistry(): Promise<ConverterRegistry> {
|
|
83
118
|
const registry = new ConverterRegistry();
|
|
@@ -85,9 +120,29 @@ export async function createDefaultRegistry(): Promise<ConverterRegistry> {
|
|
|
85
120
|
// Import converters dynamically to avoid circular deps
|
|
86
121
|
const { markdownConverter } = await import("./native/markdown");
|
|
87
122
|
const { plaintextConverter } = await import("./native/plaintext");
|
|
88
|
-
const {
|
|
89
|
-
|
|
90
|
-
|
|
123
|
+
const { xlsxAdapter } = await import("./adapters/xlsx/adapter");
|
|
124
|
+
// The PDF/Office adapters load pdf.js, which cannot initialize where its
|
|
125
|
+
// native canvas binding is missing (a standalone compiled executable). A
|
|
126
|
+
// failed adapter only fails its own file types.
|
|
127
|
+
const markitdownAdapter = await loadAdapter(
|
|
128
|
+
async () =>
|
|
129
|
+
(await import("./adapters/markitdownTs/adapter")).markitdownAdapter,
|
|
130
|
+
"adapter/markitdown-ts",
|
|
131
|
+
[".pdf", ".docx"],
|
|
132
|
+
[
|
|
133
|
+
"application/pdf",
|
|
134
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
135
|
+
]
|
|
136
|
+
);
|
|
137
|
+
const officeparserAdapter = await loadAdapter(
|
|
138
|
+
async () =>
|
|
139
|
+
(await import("./adapters/officeparser/adapter")).officeparserAdapter,
|
|
140
|
+
"adapter/officeparser",
|
|
141
|
+
[".pptx"],
|
|
142
|
+
[
|
|
143
|
+
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
|
144
|
+
]
|
|
145
|
+
);
|
|
91
146
|
const { jsonlAdapter } = await import("./adapters/jsonl/adapter");
|
|
92
147
|
const { transcriptAdapter } = await import("./adapters/transcript/adapter");
|
|
93
148
|
const { emailRecordAdapter } = await import("./adapters/email/adapter");
|
|
@@ -98,6 +153,7 @@ export async function createDefaultRegistry(): Promise<ConverterRegistry> {
|
|
|
98
153
|
// Register in priority order
|
|
99
154
|
registry.register(markdownConverter);
|
|
100
155
|
registry.register(plaintextConverter);
|
|
156
|
+
registry.register(xlsxAdapter);
|
|
101
157
|
registry.register(markitdownAdapter);
|
|
102
158
|
registry.register(officeparserAdapter);
|
|
103
159
|
registry.registerRecordAdapter(jsonlAdapter);
|
|
@@ -18,5 +18,7 @@ export const NATIVE_VERSIONS = {
|
|
|
18
18
|
*/
|
|
19
19
|
export const ADAPTER_VERSIONS = {
|
|
20
20
|
"markitdown-ts": "0.0.10+xlsx.0.20.3",
|
|
21
|
+
/** Direct SheetJS -> Markdown tables (fn-198); output matches markitdown. */
|
|
22
|
+
xlsx: "sheetjs-0.20.3+md.1",
|
|
21
23
|
officeparser: "7.8.0+pdfjs.6.3.289",
|
|
22
24
|
} as const;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Empty-collection rule shared by server surfaces and the Web UI bundle, so
|
|
3
|
+
* it carries type imports only.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type { ActivationCollectionStatus } from "./activation-status";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* A collection with nothing indexed yet: its only activation problem is
|
|
10
|
+
* `no_documents`. Surfaces show it as informational, never as a failure.
|
|
11
|
+
*/
|
|
12
|
+
export function isEmptyActivationCollection(
|
|
13
|
+
collection: Pick<ActivationCollectionStatus, "ready" | "remediation">
|
|
14
|
+
): boolean {
|
|
15
|
+
return !collection.ready && collection.remediation?.code === "no_documents";
|
|
16
|
+
}
|
|
@@ -14,6 +14,7 @@ import type {
|
|
|
14
14
|
ConnectorVerificationTarget,
|
|
15
15
|
} from "./connector-verifier";
|
|
16
16
|
|
|
17
|
+
import { isEmptyActivationCollection } from "./activation-empty";
|
|
17
18
|
import { createEphemeralActivationProbePlan } from "./activation-probe-plan";
|
|
18
19
|
import { verifyLexicalActivation } from "./activation-verifier";
|
|
19
20
|
import {
|
|
@@ -73,7 +74,13 @@ export interface ActivationConnectorStatus {
|
|
|
73
74
|
|
|
74
75
|
export interface ActivationStatus {
|
|
75
76
|
schemaVersion: "1.0";
|
|
77
|
+
/** At least one collection passed the lexical proof. */
|
|
76
78
|
usable: boolean;
|
|
79
|
+
/**
|
|
80
|
+
* At least one collection is configured and none failed its proof. A
|
|
81
|
+
* collection with no documents yet is not a failure (see
|
|
82
|
+
* `isEmptyActivationCollection`), so it does not clear this flag.
|
|
83
|
+
*/
|
|
77
84
|
healthy: boolean;
|
|
78
85
|
collections: ActivationCollectionStatus[];
|
|
79
86
|
/** Only fingerprint-current persisted connector receipts may appear here. */
|
|
@@ -178,10 +185,13 @@ function remediationFor(
|
|
|
178
185
|
stage: ActivationStageName,
|
|
179
186
|
code: ActivationVerificationCode
|
|
180
187
|
): ActivationRemediation {
|
|
181
|
-
const command =
|
|
188
|
+
const command =
|
|
189
|
+
code === "no_documents"
|
|
190
|
+
? "gno update"
|
|
191
|
+
: `gno index ${collection} --no-embed`;
|
|
182
192
|
const messages: Partial<Record<ActivationVerificationCode, string>> = {
|
|
183
193
|
no_documents:
|
|
184
|
-
"
|
|
194
|
+
"No documents are indexed yet. Add supported files to this collection's folder, then run gno update.",
|
|
185
195
|
no_probe_term:
|
|
186
196
|
"Add searchable text or adjust the collection filters, then reindex.",
|
|
187
197
|
index_query_failed:
|
|
@@ -322,7 +332,11 @@ async function buildConnectorStatuses(
|
|
|
322
332
|
): Promise<{ items: ActivationConnectorStatus[]; total: number }> {
|
|
323
333
|
const sortedTargets = [...targets].sort((a, b) => a.id.localeCompare(b.id));
|
|
324
334
|
const boundedTargets = sortedTargets.slice(0, MAX_CONNECTOR_TARGETS);
|
|
325
|
-
|
|
335
|
+
// A collection with no documents gives a connector nothing to prove.
|
|
336
|
+
const provable = collections.filter(
|
|
337
|
+
({ projected }) => !isEmptyActivationCollection(projected)
|
|
338
|
+
);
|
|
339
|
+
const allPairs = provable.flatMap((collection) =>
|
|
326
340
|
boundedTargets.map((target) => ({ collection, target }))
|
|
327
341
|
);
|
|
328
342
|
const items = await mapBounded(
|
|
@@ -363,7 +377,7 @@ async function buildConnectorStatuses(
|
|
|
363
377
|
};
|
|
364
378
|
}
|
|
365
379
|
);
|
|
366
|
-
return { items, total:
|
|
380
|
+
return { items, total: provable.length * sortedTargets.length };
|
|
367
381
|
}
|
|
368
382
|
|
|
369
383
|
async function verifyCoalesced(
|
|
@@ -501,7 +515,11 @@ export async function buildActivationStatus(
|
|
|
501
515
|
schemaVersion: "1.0",
|
|
502
516
|
usable: projected.some((collection) => collection.ready),
|
|
503
517
|
healthy:
|
|
504
|
-
projected.length > 0 &&
|
|
518
|
+
projected.length > 0 &&
|
|
519
|
+
projected.every(
|
|
520
|
+
(collection) =>
|
|
521
|
+
collection.ready || isEmptyActivationCollection(collection)
|
|
522
|
+
),
|
|
505
523
|
collections: projected,
|
|
506
524
|
connectors: connectorStatuses.items,
|
|
507
525
|
connectorProjection: {
|
|
@@ -144,7 +144,7 @@ export const evaluateFreshnessAudit = (
|
|
|
144
144
|
document,
|
|
145
145
|
kind: "stale-indexed-revision",
|
|
146
146
|
message: document.lastErrorCode
|
|
147
|
-
?
|
|
147
|
+
? `The latest indexing attempt recorded an error (${document.lastErrorCode})`
|
|
148
148
|
: "The document has no indexed-at revision evidence",
|
|
149
149
|
detail: {
|
|
150
150
|
indexedAt: document.indexedAt,
|
|
@@ -19,7 +19,10 @@ import { readdir, realpath, stat } from "node:fs/promises";
|
|
|
19
19
|
// node:path join: platform path algebra; no Bun equivalent.
|
|
20
20
|
import { join } from "node:path";
|
|
21
21
|
|
|
22
|
-
import type {
|
|
22
|
+
import type {
|
|
23
|
+
AuditLinkSnapshot,
|
|
24
|
+
AuditLinkSnapshotLink,
|
|
25
|
+
} from "../store/sqlite/graph-link-resolver";
|
|
23
26
|
|
|
24
27
|
import {
|
|
25
28
|
createWorkspaceFileMatcher,
|
|
@@ -30,6 +33,27 @@ import { pathContains, placeDocument } from "./link-workspace";
|
|
|
30
33
|
/** Path identity for exact-path matching: NFC, case-insensitive. */
|
|
31
34
|
const pathKey = (path: string): string => path.normalize("NFC").toLowerCase();
|
|
32
35
|
|
|
36
|
+
/**
|
|
37
|
+
* A Markdown link names the file's exact path; as with wiki links, `.md` is
|
|
38
|
+
* optional, so `[x](Note)` matches an unindexed `Note.md`. An indexed
|
|
39
|
+
* `Note.md` is not outside the index, so that link keeps its status. Indexed
|
|
40
|
+
* paths are compared under the same key as listed files (NFC, Unicode
|
|
41
|
+
* lowercase), never SQLite's ASCII-only NOCASE.
|
|
42
|
+
*/
|
|
43
|
+
const markdownTargetExists = (
|
|
44
|
+
link: AuditLinkSnapshotLink,
|
|
45
|
+
targetPath: string,
|
|
46
|
+
paths: ReadonlySet<string> | undefined,
|
|
47
|
+
indexedKeys: (collection: string) => ReadonlySet<string>
|
|
48
|
+
): boolean => {
|
|
49
|
+
const key = pathKey(targetPath);
|
|
50
|
+
if (paths?.has(key) === true) return true;
|
|
51
|
+
if (key.endsWith(".md") || paths?.has(`${key}.md`) !== true) return false;
|
|
52
|
+
return !indexedKeys(link.targetCollection).has(
|
|
53
|
+
pathKey(`${link.targetRefNorm}.md`)
|
|
54
|
+
);
|
|
55
|
+
};
|
|
56
|
+
|
|
33
57
|
/** Upper bound of files listed per workspace; beyond it the listing is partial. */
|
|
34
58
|
export const WORKSPACE_FILE_LISTING_MAX_FILES = 200_000;
|
|
35
59
|
|
|
@@ -152,7 +176,7 @@ export const listWorkspaceFiles = async (
|
|
|
152
176
|
* Mark unresolved plain wiki links and relative Markdown links of audited
|
|
153
177
|
* documents inside a link workspace whose target exists as a workspace file.
|
|
154
178
|
* A wiki link matches under workspace resolution rules; a Markdown link must
|
|
155
|
-
* name the file's exact path (NFC, case-insensitive). Each involved workspace
|
|
179
|
+
* name the file's exact path (NFC, case-insensitive), `.md` optional. Each involved workspace
|
|
156
180
|
* is listed once per call. A failed or partial listing never marks a link
|
|
157
181
|
* that is not in it; the snapshot then carries one diagnostic.
|
|
158
182
|
*/
|
|
@@ -220,12 +244,33 @@ export const markOutsideIndexLinks = async (
|
|
|
220
244
|
paths: new Set(listing.files.map(pathKey)),
|
|
221
245
|
});
|
|
222
246
|
}
|
|
247
|
+
const indexedByCollection = new Map<string, Set<string>>();
|
|
248
|
+
const indexedKeys = (collection: string): ReadonlySet<string> => {
|
|
249
|
+
let keys = indexedByCollection.get(collection);
|
|
250
|
+
if (!keys) {
|
|
251
|
+
keys = new Set(
|
|
252
|
+
db
|
|
253
|
+
.query<{ relPath: string }, [string]>(
|
|
254
|
+
"SELECT rel_path AS relPath FROM documents WHERE active = 1 AND collection = ?"
|
|
255
|
+
)
|
|
256
|
+
.all(collection)
|
|
257
|
+
.map(({ relPath }) => pathKey(relPath))
|
|
258
|
+
);
|
|
259
|
+
indexedByCollection.set(collection, keys);
|
|
260
|
+
}
|
|
261
|
+
return keys;
|
|
262
|
+
};
|
|
223
263
|
const links = [...snapshot.links];
|
|
224
264
|
for (const entry of placed) {
|
|
225
265
|
const listing = listings.get(entry.key);
|
|
226
266
|
const exists =
|
|
227
267
|
"targetPath" in entry
|
|
228
|
-
?
|
|
268
|
+
? markdownTargetExists(
|
|
269
|
+
entry.link,
|
|
270
|
+
entry.targetPath,
|
|
271
|
+
listing?.paths,
|
|
272
|
+
indexedKeys
|
|
273
|
+
)
|
|
229
274
|
: listing?.matches(entry.link.targetRefNorm, entry.sourcePath) === true;
|
|
230
275
|
if (exists) {
|
|
231
276
|
links[entry.index] = { ...entry.link, outsideIndex: true };
|
package/src/core/change-diff.ts
CHANGED
|
@@ -104,15 +104,27 @@ const extractTypedEdges = (markdown: string): string[] => {
|
|
|
104
104
|
return [...new Set(values)].sort();
|
|
105
105
|
};
|
|
106
106
|
|
|
107
|
+
export interface DocumentStructureOptions {
|
|
108
|
+
/** Only Markdown sources carry links (matches link extraction at sync). */
|
|
109
|
+
markdownSource: boolean;
|
|
110
|
+
excludedRanges?: ExcludedRange[];
|
|
111
|
+
}
|
|
112
|
+
|
|
107
113
|
export const extractDocumentStructure = (
|
|
108
114
|
markdown: string,
|
|
109
115
|
relPath: string,
|
|
110
116
|
dateFields: Readonly<Record<string, string>> | null | undefined,
|
|
111
|
-
excludedRanges:
|
|
117
|
+
{ markdownSource, excludedRanges }: DocumentStructureOptions
|
|
112
118
|
): DocumentStructureSnapshot => {
|
|
113
|
-
const links =
|
|
114
|
-
|
|
115
|
-
|
|
119
|
+
const links = markdownSource
|
|
120
|
+
? parseLinks(
|
|
121
|
+
markdown,
|
|
122
|
+
buildLineOffsets(markdown),
|
|
123
|
+
excludedRanges ?? getExcludedRanges(markdown)
|
|
124
|
+
)
|
|
125
|
+
.map((link) => linkSummary(link, relPath))
|
|
126
|
+
.filter((value): value is string => value !== null)
|
|
127
|
+
: [];
|
|
116
128
|
|
|
117
129
|
return {
|
|
118
130
|
headings: withOccurrences(
|
|
@@ -42,6 +42,19 @@ const COMMONMARK_ESCAPABLE = new Set([
|
|
|
42
42
|
" ",
|
|
43
43
|
]);
|
|
44
44
|
|
|
45
|
+
/**
|
|
46
|
+
* True when the character at `index` is backslash-escaped: preceded by an odd
|
|
47
|
+
* number of backslashes. An escaped `[` is literal text in CommonMark and
|
|
48
|
+
* cannot open a link.
|
|
49
|
+
*/
|
|
50
|
+
export function isBackslashEscaped(text: string, index: number): boolean {
|
|
51
|
+
let backslashes = 0;
|
|
52
|
+
for (let i = index - 1; i >= 0 && text[i] === "\\"; i -= 1) {
|
|
53
|
+
backslashes += 1;
|
|
54
|
+
}
|
|
55
|
+
return backslashes % 2 === 1;
|
|
56
|
+
}
|
|
57
|
+
|
|
45
58
|
export interface LinkEncodingStyle {
|
|
46
59
|
spaces: "percent" | "backslash" | "raw";
|
|
47
60
|
parens: "percent" | "backslash" | "raw";
|
|
@@ -220,10 +233,10 @@ export function parseParenthesizedDestination(
|
|
|
220
233
|
i += 1;
|
|
221
234
|
continue;
|
|
222
235
|
}
|
|
223
|
-
if (
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
236
|
+
if (ch === " " || ch === "\t" || ch === "\n" || ch === "\r") {
|
|
237
|
+
// CommonMark: a bare destination has no unescaped whitespace, inside
|
|
238
|
+
// nested parentheses too (`(foo(bar baz).md)` is not a link).
|
|
239
|
+
if (depth > 0) return null;
|
|
227
240
|
destEnd = i;
|
|
228
241
|
break;
|
|
229
242
|
}
|
|
@@ -262,6 +275,10 @@ export function parseParenthesizedDestination(
|
|
|
262
275
|
close = j;
|
|
263
276
|
}
|
|
264
277
|
|
|
278
|
+
// The destination and optional title must end at a real `)`:
|
|
279
|
+
// `[x](<note.md>` or `[x](<note.md>.` is not a link.
|
|
280
|
+
if (markdown[close] !== ")") return null;
|
|
281
|
+
|
|
265
282
|
const destinationRaw = markdown.slice(destStart, destEnd);
|
|
266
283
|
if (!destinationRaw) return null;
|
|
267
284
|
return {
|