@gmickel/gno 2.8.1 → 2.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/skill/SKILL.md +4 -2
- package/browser-extension/artifacts/{gno-browser-clipper-v2.8.1.zip → gno-browser-clipper-v2.8.2.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.8.2.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +1 -1
- package/spec/cli.md +16 -5
- package/src/converters/native/markdown.ts +3 -1
- package/src/core/audit-links.ts +2 -1
- package/src/core/audit-outside-index.ts +45 -17
- package/src/core/change-diff.ts +3 -3
- package/src/core/link-inventory-opaque.ts +3 -1
- package/src/ingestion/strip.ts +89 -136
- package/src/ingestion/sync.ts +30 -14
- package/browser-extension/artifacts/gno-browser-clipper-v2.8.1.zip.sha256 +0 -1
package/README.md
CHANGED
|
@@ -139,7 +139,7 @@ See the [guide](docs/COMPILED-CONTEXT.md).
|
|
|
139
139
|
|
|
140
140
|
<!-- public-truth:current-version -->
|
|
141
141
|
|
|
142
|
-
> Current source version: **v2.8.
|
|
142
|
+
> Current source version: **v2.8.2**. See [CHANGELOG.md](./CHANGELOG.md).
|
|
143
143
|
|
|
144
144
|
<!-- /public-truth -->
|
|
145
145
|
|
package/assets/skill/SKILL.md
CHANGED
|
@@ -356,8 +356,10 @@ rewrite, persist findings, judge factual truth, or replace retrieval.
|
|
|
356
356
|
finding. Link findings carry `referenceKind`, `resolutionStatus`, and
|
|
357
357
|
`resolvedScope` in their evidence detail; ambiguous vault links list the tied
|
|
358
358
|
`candidates`. `links.outside-index` `info` findings are link targets that
|
|
359
|
-
exist in the vault but are not indexed (attachments,
|
|
360
|
-
folders); Obsidian resolves them, so do
|
|
359
|
+
exist in the vault but are not indexed (attachments, files a Markdown link
|
|
360
|
+
names by path, excluded or unindexed folders); Obsidian resolves them, so do
|
|
361
|
+
not report them as broken links. Links are read only from Markdown notes,
|
|
362
|
+
never from code spans, code blocks, or non-Markdown files.
|
|
361
363
|
Report `truncation.snapshotTruncated` as "totals cover the bounded snapshot,
|
|
362
364
|
not the whole index".
|
|
363
365
|
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
7bc830276cb923a482b662b980208b0978bf8ddbfbade0873e67d9757cf9c989 gno-browser-clipper-v2.8.2.zip
|
package/package.json
CHANGED
package/spec/cli.md
CHANGED
|
@@ -965,18 +965,29 @@ Orphans stay "no incoming or outgoing resolved links", with connectivity drawn
|
|
|
965
965
|
from the whole index even when the audited documents are scoped; a tied link
|
|
966
966
|
connects nothing.
|
|
967
967
|
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
968
|
+
Links are parsed only from documents converted by the Markdown converter
|
|
969
|
+
(`native/markdown`); plain text, source code, data files and converted
|
|
970
|
+
documents store no links and produce no link findings, while staying
|
|
971
|
+
searchable. The parser skips frontmatter, HTML comments, and code as a
|
|
972
|
+
CommonMark + GFM parser reads it: inline code spans, fenced code blocks and
|
|
973
|
+
indented code blocks, inside blockquotes and list items too (an indented list
|
|
974
|
+
paragraph is not code). A link outside code on the same line still counts.
|
|
975
|
+
|
|
976
|
+
A plain wiki link, embed, or relative Markdown link from a document in a link
|
|
977
|
+
workspace whose target is not an indexed document but exists as a file inside
|
|
978
|
+
the workspace root (a non-Markdown attachment, a script beside the note, a
|
|
979
|
+
note in an unindexed folder, a note excluded by a collection pattern) is not
|
|
980
|
+
reported by `links.local-targets`. It is an `info`
|
|
972
981
|
finding of `links.outside-index` (evidence kind `outside-index-target`,
|
|
973
982
|
`resolutionStatus: "outside-index"`). That rule stays `pass`, so these
|
|
974
983
|
findings never change the exit code. The check is existence-only: each
|
|
975
984
|
involved workspace root is listed once per run (hidden files and folders
|
|
976
985
|
skipped, symlinked folders not descended, a symlink counted only when it
|
|
977
986
|
resolves to a regular file inside the workspace, at most 200,000 files), no
|
|
978
|
-
file is opened, and no graph edge is created.
|
|
987
|
+
file is opened, and no graph edge is created. Wiki targets match files by the workspace
|
|
979
988
|
link rules; a non-Markdown target needs its extension, `.md` is optional. A
|
|
989
|
+
Markdown link matches only the file at its resolved path (NFC,
|
|
990
|
+
case-insensitive). A
|
|
980
991
|
target missing from the listing stays unresolved; an unreadable folder or the
|
|
981
992
|
file bound makes the listing incomplete, which the `links.outside-index` rule
|
|
982
993
|
message states once.
|
|
@@ -8,7 +8,9 @@ import type { Converter, ConvertInput, ConvertResult } from "../types";
|
|
|
8
8
|
import { basenameWithoutExt } from "../path";
|
|
9
9
|
import { NATIVE_VERSIONS } from "../versions";
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
/** Converter id of Markdown sources, the only documents with link edges. */
|
|
12
|
+
export const MARKDOWN_CONVERTER_ID = "native/markdown" as const;
|
|
13
|
+
const CONVERTER_ID = MARKDOWN_CONVERTER_ID;
|
|
12
14
|
const CONVERTER_VERSION = NATIVE_VERSIONS.markdown;
|
|
13
15
|
|
|
14
16
|
/** UTF-8 BOM character */
|
package/src/core/audit-links.ts
CHANGED
|
@@ -16,8 +16,9 @@ import {
|
|
|
16
16
|
/**
|
|
17
17
|
* 1.1: workspace-wide resolution and reference/resolution/scope evidence.
|
|
18
18
|
* 1.2: `links.outside-index` for workspace files that are not indexed.
|
|
19
|
+
* 1.3: relative Markdown links to such files are `outside-index` too.
|
|
19
20
|
*/
|
|
20
|
-
export const LINK_AUDIT_RULE_VERSION = "1.
|
|
21
|
+
export const LINK_AUDIT_RULE_VERSION = "1.3" as const;
|
|
21
22
|
/** Default per-rule cap; runs pass the effective `maxFindings` instead. */
|
|
22
23
|
export const LINK_AUDIT_MAX_FINDINGS_PER_RULE = 1000;
|
|
23
24
|
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Existence-only classification of unresolved workspace wiki links
|
|
3
|
-
* target is a file inside the link workspace
|
|
4
|
-
* (an attachment,
|
|
2
|
+
* Existence-only classification of unresolved workspace wiki links and
|
|
3
|
+
* relative Markdown links whose target is a file inside the link workspace
|
|
4
|
+
* that is not an indexed document (an attachment, a script beside a note, or
|
|
5
|
+
* a note in an unindexed or excluded folder). Obsidian
|
|
5
6
|
* resolves such links, so the link audit reports them as `outside-index`
|
|
6
7
|
* instead of unresolved. Only file names are listed: no file is opened,
|
|
7
8
|
* indexed, or returned, and no graph edge is created.
|
|
@@ -26,6 +27,9 @@ import {
|
|
|
26
27
|
} from "../store/sqlite/workspace-link-resolver";
|
|
27
28
|
import { pathContains, placeDocument } from "./link-workspace";
|
|
28
29
|
|
|
30
|
+
/** Path identity for exact-path matching: NFC, case-insensitive. */
|
|
31
|
+
const pathKey = (path: string): string => path.normalize("NFC").toLowerCase();
|
|
32
|
+
|
|
29
33
|
/** Upper bound of files listed per workspace; beyond it the listing is partial. */
|
|
30
34
|
export const WORKSPACE_FILE_LISTING_MAX_FILES = 200_000;
|
|
31
35
|
|
|
@@ -145,8 +149,10 @@ export const listWorkspaceFiles = async (
|
|
|
145
149
|
};
|
|
146
150
|
|
|
147
151
|
/**
|
|
148
|
-
* Mark unresolved plain wiki links
|
|
149
|
-
* workspace whose target exists as a workspace file.
|
|
152
|
+
* Mark unresolved plain wiki links and relative Markdown links of audited
|
|
153
|
+
* documents inside a link workspace whose target exists as a workspace file.
|
|
154
|
+
* A wiki link matches under workspace resolution rules; a Markdown link must
|
|
155
|
+
* name the file's exact path (NFC, case-insensitive). Each involved workspace
|
|
150
156
|
* is listed once per call. A failed or partial listing never marks a link
|
|
151
157
|
* that is not in it; the snapshot then carries one diagnostic.
|
|
152
158
|
*/
|
|
@@ -163,8 +169,8 @@ export const markOutsideIndexLinks = async (
|
|
|
163
169
|
.filter(
|
|
164
170
|
({ link }) =>
|
|
165
171
|
link.resolved === null &&
|
|
166
|
-
link.linkType === "
|
|
167
|
-
|
|
172
|
+
(link.linkType === "markdown" ||
|
|
173
|
+
(link.linkType === "wiki" && link.explicitCollection !== true)) &&
|
|
168
174
|
(audited === null || audited.has(link.sourceId))
|
|
169
175
|
);
|
|
170
176
|
if (candidates.length === 0) return snapshot;
|
|
@@ -172,19 +178,34 @@ export const markOutsideIndexLinks = async (
|
|
|
172
178
|
const documentRelPaths = new Map(
|
|
173
179
|
snapshot.documents.map((document) => [document.id, document.relPath])
|
|
174
180
|
);
|
|
175
|
-
|
|
181
|
+
type PlacedLink = {
|
|
182
|
+
link: AuditLinkSnapshot["links"][number];
|
|
183
|
+
index: number;
|
|
184
|
+
key: string;
|
|
185
|
+
} & ({ sourcePath: string } | { targetPath: string });
|
|
186
|
+
const placed = candidates.flatMap(({ link, index }): PlacedLink[] => {
|
|
187
|
+
const membership = memberships.get(link.sourceCollection);
|
|
176
188
|
const placement = placeDocument(
|
|
177
|
-
|
|
189
|
+
membership,
|
|
178
190
|
documentRelPaths.get(link.sourceId) ?? link.sourceRelPath
|
|
179
191
|
);
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
192
|
+
if (placement.key === null) return [];
|
|
193
|
+
if (link.linkType === "wiki") {
|
|
194
|
+
return [{ link, index, key: placement.key, sourcePath: placement.path }];
|
|
195
|
+
}
|
|
196
|
+
// A Markdown target is stored collection-relative; place it the same way.
|
|
197
|
+
const target = placeDocument(membership, link.targetRefNorm);
|
|
198
|
+
return target.key === placement.key
|
|
199
|
+
? [{ link, index, key: placement.key, targetPath: target.path }]
|
|
200
|
+
: [];
|
|
183
201
|
});
|
|
184
202
|
if (placed.length === 0) return snapshot;
|
|
185
|
-
const
|
|
203
|
+
const listings = new Map<
|
|
186
204
|
string,
|
|
187
|
-
|
|
205
|
+
{
|
|
206
|
+
matches: ReturnType<typeof createWorkspaceFileMatcher>;
|
|
207
|
+
paths: Set<string>;
|
|
208
|
+
}
|
|
188
209
|
>();
|
|
189
210
|
const maxFiles = options.maxFiles ?? WORKSPACE_FILE_LISTING_MAX_FILES;
|
|
190
211
|
let incomplete = 0;
|
|
@@ -194,12 +215,19 @@ export const markOutsideIndexLinks = async (
|
|
|
194
215
|
signal: options.signal,
|
|
195
216
|
});
|
|
196
217
|
if (!listing.complete) incomplete += 1;
|
|
197
|
-
|
|
218
|
+
listings.set(key, {
|
|
219
|
+
matches: createWorkspaceFileMatcher(listing.files),
|
|
220
|
+
paths: new Set(listing.files.map(pathKey)),
|
|
221
|
+
});
|
|
198
222
|
}
|
|
199
223
|
const links = [...snapshot.links];
|
|
200
224
|
for (const entry of placed) {
|
|
201
|
-
const
|
|
202
|
-
|
|
225
|
+
const listing = listings.get(entry.key);
|
|
226
|
+
const exists =
|
|
227
|
+
"targetPath" in entry
|
|
228
|
+
? listing?.paths.has(pathKey(entry.targetPath)) === true
|
|
229
|
+
: listing?.matches(entry.link.targetRefNorm, entry.sourcePath) === true;
|
|
230
|
+
if (exists) {
|
|
203
231
|
links[entry.index] = { ...entry.link, outsideIndex: true };
|
|
204
232
|
}
|
|
205
233
|
}
|
package/src/core/change-diff.ts
CHANGED
|
@@ -12,7 +12,7 @@ import type {
|
|
|
12
12
|
|
|
13
13
|
import { parseFrontmatter } from "../ingestion/frontmatter";
|
|
14
14
|
import { buildLineOffsets } from "../ingestion/position";
|
|
15
|
-
import { getExcludedRanges } from "../ingestion/strip";
|
|
15
|
+
import { type ExcludedRange, getExcludedRanges } from "../ingestion/strip";
|
|
16
16
|
import { normalizeMarkdownPath, normalizeWikiName, parseLinks } from "./links";
|
|
17
17
|
import { extractSections } from "./sections";
|
|
18
18
|
|
|
@@ -107,9 +107,9 @@ const extractTypedEdges = (markdown: string): string[] => {
|
|
|
107
107
|
export const extractDocumentStructure = (
|
|
108
108
|
markdown: string,
|
|
109
109
|
relPath: string,
|
|
110
|
-
dateFields: Readonly<Record<string, string>> | null | undefined
|
|
110
|
+
dateFields: Readonly<Record<string, string>> | null | undefined,
|
|
111
|
+
excludedRanges: ExcludedRange[] = getExcludedRanges(markdown)
|
|
111
112
|
): DocumentStructureSnapshot => {
|
|
112
|
-
const excludedRanges = getExcludedRanges(markdown);
|
|
113
113
|
const links = parseLinks(markdown, buildLineOffsets(markdown), excludedRanges)
|
|
114
114
|
.map((link) => linkSummary(link, relPath))
|
|
115
115
|
.filter((value): value is string => value !== null);
|
|
@@ -81,7 +81,9 @@ export function findExcludedKind(
|
|
|
81
81
|
export function codeContextReason(
|
|
82
82
|
kind: ExcludedRange["kind"] | null
|
|
83
83
|
): "code_fence_context" | "inline_code_context" | null {
|
|
84
|
-
if (kind === "fenced_code"
|
|
84
|
+
if (kind === "fenced_code" || kind === "indented_code") {
|
|
85
|
+
return "code_fence_context";
|
|
86
|
+
}
|
|
85
87
|
if (kind === "inline_code") return "inline_code_context";
|
|
86
88
|
return null;
|
|
87
89
|
}
|
package/src/ingestion/strip.ts
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* Identifies regions to exclude from link/tag extraction:
|
|
5
5
|
* - YAML frontmatter
|
|
6
6
|
* - Fenced code blocks (CommonMark backtick and tilde fences)
|
|
7
|
+
* - Indented code blocks
|
|
7
8
|
* - Inline code
|
|
8
9
|
* - HTML comments
|
|
9
10
|
*
|
|
@@ -13,6 +14,10 @@
|
|
|
13
14
|
* @module src/ingestion/strip
|
|
14
15
|
*/
|
|
15
16
|
|
|
17
|
+
import { fromMarkdown } from "mdast-util-from-markdown";
|
|
18
|
+
import { gfmFromMarkdown } from "mdast-util-gfm";
|
|
19
|
+
import { gfm } from "micromark-extension-gfm";
|
|
20
|
+
|
|
16
21
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
17
22
|
// Types
|
|
18
23
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -20,6 +25,7 @@
|
|
|
20
25
|
export type ExcludedRangeKind =
|
|
21
26
|
| "frontmatter"
|
|
22
27
|
| "fenced_code"
|
|
28
|
+
| "indented_code"
|
|
23
29
|
| "inline_code"
|
|
24
30
|
| "html_comment";
|
|
25
31
|
|
|
@@ -39,150 +45,100 @@ export interface ExcludedRange {
|
|
|
39
45
|
/** Frontmatter at start of file (YAML between --- delimiters) */
|
|
40
46
|
const FRONTMATTER_REGEX = /^---\r?\n[\s\S]*?(?:\r?\n)?---(?:\r?\n|$)/;
|
|
41
47
|
|
|
42
|
-
/** CommonMark fence opener: 0–3 spaces, then 3+ backticks or tildes + info. */
|
|
43
|
-
const FENCE_OPEN_REGEX = /^ {0,3}(`{3,}|~{3,})(.*)$/u;
|
|
44
|
-
|
|
45
|
-
/** CommonMark fence closer: matching character, length ≥ opener, trailing space/tabs only. */
|
|
46
|
-
const FENCE_CLOSE_REGEX = /^ {0,3}(`{3,}|~{3,})[\t ]*$/u;
|
|
47
|
-
|
|
48
48
|
/** HTML comments */
|
|
49
49
|
const HTML_COMMENT_REGEX = /<!--[\s\S]*?-->/g;
|
|
50
50
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
51
|
+
/** Backtick (code span or fence) or tilde fence anywhere in the text. */
|
|
52
|
+
const INLINE_OR_FENCE_TRIGGER_REGEX = /`|~~~/;
|
|
53
|
+
/** Blockquote markers and indentation before a line's content. */
|
|
54
|
+
const CONTAINER_PREFIX_REGEX = /^[\t >]*/;
|
|
55
|
+
/** Four columns of indentation after any blockquote markers. */
|
|
56
|
+
const INDENTED_LINE_REGEX = /^(?:>[\t ]?)*(?:\t| {4})/;
|
|
57
57
|
/**
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
58
|
+
* Lines after which an indented line may start an indented code block: an
|
|
59
|
+
* ATX heading, table row, HTML or definition line, or a thematic break or
|
|
60
|
+
* setext underline. After prose or a list item it continues the paragraph.
|
|
61
61
|
*/
|
|
62
|
-
const
|
|
63
|
-
|
|
64
|
-
let offset = 0;
|
|
65
|
-
let open: OpenFence | null = null;
|
|
66
|
-
|
|
67
|
-
while (offset <= markdown.length) {
|
|
68
|
-
const nextNl = markdown.indexOf("\n", offset);
|
|
69
|
-
const lineEnd = nextNl === -1 ? markdown.length : nextNl;
|
|
70
|
-
const rawLine = markdown.slice(offset, lineEnd);
|
|
71
|
-
const logical = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine;
|
|
62
|
+
const BLOCK_END_LINE_REGEX =
|
|
63
|
+
/^(?:[#|<[]|([-*_=])[\t ]*(?:\1[\t ]*){2,}$)|>[\t ]*$/u;
|
|
72
64
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
length: run.length,
|
|
93
|
-
start: offset,
|
|
94
|
-
};
|
|
95
|
-
}
|
|
65
|
+
/**
|
|
66
|
+
* Cheap superset test for code: a backtick, a tilde fence, or an indented
|
|
67
|
+
* line that can open an indented code block (not a list continuation). When
|
|
68
|
+
* it fails the note has no code and the parser is skipped.
|
|
69
|
+
*/
|
|
70
|
+
const mayContainCode = (markdown: string): boolean => {
|
|
71
|
+
if (INLINE_OR_FENCE_TRIGGER_REGEX.test(markdown)) return true;
|
|
72
|
+
let previous: string | null = null;
|
|
73
|
+
for (const rawLine of markdown.split("\n")) {
|
|
74
|
+
const line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine;
|
|
75
|
+
const content = line.replace(CONTAINER_PREFIX_REGEX, "");
|
|
76
|
+
if (
|
|
77
|
+
content !== "" &&
|
|
78
|
+
INDENTED_LINE_REGEX.test(line) &&
|
|
79
|
+
(previous === null ||
|
|
80
|
+
previous === "" ||
|
|
81
|
+
BLOCK_END_LINE_REGEX.test(previous))
|
|
82
|
+
) {
|
|
83
|
+
return true;
|
|
96
84
|
}
|
|
97
|
-
|
|
98
|
-
if (nextNl === -1) break;
|
|
99
|
-
offset = nextNl + 1;
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
if (open) {
|
|
103
|
-
ranges.push({
|
|
104
|
-
start: open.start,
|
|
105
|
-
end: markdown.length,
|
|
106
|
-
kind: "fenced_code",
|
|
107
|
-
});
|
|
85
|
+
previous = content;
|
|
108
86
|
}
|
|
109
|
-
|
|
110
|
-
return ranges;
|
|
87
|
+
return false;
|
|
111
88
|
};
|
|
112
89
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
90
|
+
/** Opening fence at the start of a code node: backtick or tilde fence. */
|
|
91
|
+
const FENCE_START_REGEX = /^[\t ]*(?:`{3,}|~{3,})/;
|
|
92
|
+
|
|
93
|
+
interface CodeNode {
|
|
94
|
+
children?: CodeNode[];
|
|
95
|
+
position?: { start: { offset?: number }; end: { offset?: number } };
|
|
96
|
+
type: string;
|
|
117
97
|
}
|
|
118
98
|
|
|
119
|
-
/**
|
|
120
|
-
|
|
99
|
+
/**
|
|
100
|
+
* Code regions as the CommonMark + GFM parser sees them: inline code spans,
|
|
101
|
+
* fenced code blocks and indented code blocks, with their block context
|
|
102
|
+
* (paragraphs, blockquotes, list items) handled by the parser. Frontmatter is
|
|
103
|
+
* blanked first (same length, newlines kept) so it cannot open a fence.
|
|
104
|
+
*/
|
|
105
|
+
const collectCodeRanges = (
|
|
121
106
|
markdown: string,
|
|
122
|
-
|
|
107
|
+
frontmatterEnd: number
|
|
123
108
|
): ExcludedRange[] => {
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
)
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
const excluded = excludedRanges[excludedIndex];
|
|
135
|
-
if (excluded && cursor >= excluded.start && cursor < excluded.end) {
|
|
136
|
-
cursor = excluded.end;
|
|
137
|
-
continue;
|
|
138
|
-
}
|
|
139
|
-
if (markdown[cursor] !== "`") {
|
|
140
|
-
cursor += 1;
|
|
141
|
-
continue;
|
|
142
|
-
}
|
|
143
|
-
const start = cursor;
|
|
144
|
-
while (markdown[cursor] === "`") cursor += 1;
|
|
145
|
-
let backslashes = 0;
|
|
146
|
-
for (let i = start - 1; i >= 0 && markdown[i] === "\\"; i -= 1) {
|
|
147
|
-
backslashes += 1;
|
|
148
|
-
}
|
|
149
|
-
if (backslashes % 2 === 0) {
|
|
150
|
-
runs.push({ start, end: cursor, length: cursor - start });
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
const nextMatchingRun = Array.from<number | undefined>({
|
|
155
|
-
length: runs.length,
|
|
156
|
-
});
|
|
157
|
-
const latestByLength = new Map<number, number>();
|
|
158
|
-
for (let index = runs.length - 1; index >= 0; index -= 1) {
|
|
159
|
-
const run = runs[index];
|
|
160
|
-
if (!run) continue;
|
|
161
|
-
nextMatchingRun[index] = latestByLength.get(run.length);
|
|
162
|
-
latestByLength.set(run.length, index);
|
|
163
|
-
}
|
|
164
|
-
|
|
109
|
+
if (!mayContainCode(markdown.slice(frontmatterEnd))) return [];
|
|
110
|
+
const source =
|
|
111
|
+
frontmatterEnd > 0
|
|
112
|
+
? markdown.slice(0, frontmatterEnd).replace(/[^\r\n]/gu, " ") +
|
|
113
|
+
markdown.slice(frontmatterEnd)
|
|
114
|
+
: markdown;
|
|
115
|
+
const root = fromMarkdown(source, {
|
|
116
|
+
extensions: [gfm()],
|
|
117
|
+
mdastExtensions: [gfmFromMarkdown()],
|
|
118
|
+
}) as CodeNode;
|
|
165
119
|
const ranges: ExcludedRange[] = [];
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
const
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
120
|
+
const visit = (node: CodeNode): void => {
|
|
121
|
+
const start = node.position?.start.offset;
|
|
122
|
+
const end = node.position?.end.offset;
|
|
123
|
+
if (start !== undefined && end !== undefined && end > start) {
|
|
124
|
+
if (node.type === "inlineCode") {
|
|
125
|
+
ranges.push({ start, end, kind: "inline_code" });
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
if (node.type === "code") {
|
|
129
|
+
ranges.push({
|
|
130
|
+
start,
|
|
131
|
+
end,
|
|
132
|
+
kind: FENCE_START_REGEX.test(source.slice(start, end))
|
|
133
|
+
? "fenced_code"
|
|
134
|
+
: "indented_code",
|
|
135
|
+
});
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
178
138
|
}
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
kind: "inline_code",
|
|
183
|
-
});
|
|
184
|
-
index = closeIndex + 1;
|
|
185
|
-
}
|
|
139
|
+
for (const child of node.children ?? []) visit(child);
|
|
140
|
+
};
|
|
141
|
+
visit(root);
|
|
186
142
|
return ranges;
|
|
187
143
|
};
|
|
188
144
|
|
|
@@ -208,10 +164,7 @@ export function getExcludedRanges(markdown: string): ExcludedRange[] {
|
|
|
208
164
|
});
|
|
209
165
|
}
|
|
210
166
|
|
|
211
|
-
// 2.
|
|
212
|
-
ranges.push(...collectFencedCodeRanges(markdown));
|
|
213
|
-
|
|
214
|
-
// 3. HTML comments
|
|
167
|
+
// 2. HTML comments
|
|
215
168
|
HTML_COMMENT_REGEX.lastIndex = 0;
|
|
216
169
|
let match: RegExpExecArray | null;
|
|
217
170
|
while ((match = HTML_COMMENT_REGEX.exec(markdown)) !== null) {
|
|
@@ -222,10 +175,10 @@ export function getExcludedRanges(markdown: string): ExcludedRange[] {
|
|
|
222
175
|
});
|
|
223
176
|
}
|
|
224
177
|
|
|
225
|
-
//
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
178
|
+
// 3. Code spans and fenced or indented code blocks, from the parser.
|
|
179
|
+
ranges.push(
|
|
180
|
+
...collectCodeRanges(markdown, frontmatterMatch?.[0].length ?? 0)
|
|
181
|
+
);
|
|
229
182
|
|
|
230
183
|
// Sort by start position for efficient lookup
|
|
231
184
|
ranges.sort((a, b) => a.start - b.start);
|
package/src/ingestion/sync.ts
CHANGED
|
@@ -40,6 +40,7 @@ import {
|
|
|
40
40
|
import { createJsonlAdapter } from "../converters/adapters/jsonl/adapter";
|
|
41
41
|
import { createTranscriptAdapter } from "../converters/adapters/transcript/adapter";
|
|
42
42
|
import { getDefaultMimeDetector, type MimeDetector } from "../converters/mime";
|
|
43
|
+
import { MARKDOWN_CONVERTER_ID } from "../converters/native/markdown";
|
|
43
44
|
import {
|
|
44
45
|
type ConversionPipeline,
|
|
45
46
|
getDefaultPipeline,
|
|
@@ -84,7 +85,7 @@ import {
|
|
|
84
85
|
type SourceContentReaderPort,
|
|
85
86
|
type SourceReadFailure,
|
|
86
87
|
} from "./source-availability";
|
|
87
|
-
import { getExcludedRanges } from "./strip";
|
|
88
|
+
import { type ExcludedRange, getExcludedRanges } from "./strip";
|
|
88
89
|
import { extractTypedMetadata } from "./typed-metadata";
|
|
89
90
|
import { collectionToWalkConfig, DEFAULT_CHUNK_PARAMS } from "./types";
|
|
90
91
|
import { defaultWalker } from "./walker";
|
|
@@ -105,8 +106,10 @@ const MAX_CONCURRENCY = 16;
|
|
|
105
106
|
* will be re-processed. Must stay >= TYPED_METADATA_INGEST_VERSION (tested).
|
|
106
107
|
* 8: wiki links with a table-escaped alias (`[[Note\|Alias]]`) and Markdown
|
|
107
108
|
* link text with square brackets parse the way Obsidian renders them.
|
|
109
|
+
* 9: links are extracted from Markdown sources only, and never from code
|
|
110
|
+
* spans or indented code blocks.
|
|
108
111
|
*/
|
|
109
|
-
export const INGEST_VERSION =
|
|
112
|
+
export const INGEST_VERSION = 9;
|
|
110
113
|
const EMPTY_CONTENT_TYPE_RULES_FINGERPRINT =
|
|
111
114
|
fingerprintContentTypeMetadataRules([]);
|
|
112
115
|
const NON_RETRYABLE_CONVERSION_ERROR_CODES = new Set([
|
|
@@ -901,14 +904,19 @@ export class SyncService {
|
|
|
901
904
|
mime.ext,
|
|
902
905
|
contentTypeRules
|
|
903
906
|
);
|
|
907
|
+
// Code ranges come from a full Markdown parse: compute them once and
|
|
908
|
+
// share them with the change journal and link extraction.
|
|
909
|
+
const excludedRanges = getExcludedRanges(artifact.markdown);
|
|
904
910
|
const previousStructure = await this.readPreviousStructure(
|
|
905
911
|
store,
|
|
906
|
-
existing
|
|
912
|
+
existing,
|
|
913
|
+
{ markdown: artifact.markdown, excludedRanges }
|
|
907
914
|
);
|
|
908
915
|
const nextStructure = extractDocumentStructure(
|
|
909
916
|
artifact.markdown,
|
|
910
917
|
entry.relPath,
|
|
911
|
-
extractedMetadata.dateFields
|
|
918
|
+
extractedMetadata.dateFields,
|
|
919
|
+
excludedRanges
|
|
912
920
|
);
|
|
913
921
|
const structureDelta = diffDocumentStructure(
|
|
914
922
|
previousStructure,
|
|
@@ -1019,14 +1027,17 @@ export class SyncService {
|
|
|
1019
1027
|
});
|
|
1020
1028
|
}
|
|
1021
1029
|
|
|
1022
|
-
// 14. Extract and store links (wiki and markdown links)
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
const parsedLinks =
|
|
1026
|
-
artifact.
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
+
// 14. Extract and store links (wiki and markdown links). Only
|
|
1031
|
+
// Markdown sources carry links, as in Obsidian: link-shaped text in
|
|
1032
|
+
// code, plain text or converted documents stays searchable prose.
|
|
1033
|
+
const parsedLinks =
|
|
1034
|
+
artifact.meta.converterId === MARKDOWN_CONVERTER_ID
|
|
1035
|
+
? parseLinks(
|
|
1036
|
+
artifact.markdown,
|
|
1037
|
+
buildLineOffsets(artifact.markdown),
|
|
1038
|
+
excludedRanges
|
|
1039
|
+
)
|
|
1040
|
+
: [];
|
|
1030
1041
|
|
|
1031
1042
|
const linkInputs: DocLinkInput[] = [];
|
|
1032
1043
|
for (const link of parsedLinks) {
|
|
@@ -1183,7 +1194,8 @@ export class SyncService {
|
|
|
1183
1194
|
|
|
1184
1195
|
private async readPreviousStructure(
|
|
1185
1196
|
store: StorePort,
|
|
1186
|
-
existing: DocumentRow | null
|
|
1197
|
+
existing: DocumentRow | null,
|
|
1198
|
+
current?: { markdown: string; excludedRanges: ExcludedRange[] }
|
|
1187
1199
|
): Promise<ReturnType<typeof extractDocumentStructure> | null | undefined> {
|
|
1188
1200
|
if (!existing) return null;
|
|
1189
1201
|
if (!existing.mirrorHash) return undefined;
|
|
@@ -1198,7 +1210,11 @@ export class SyncService {
|
|
|
1198
1210
|
return extractDocumentStructure(
|
|
1199
1211
|
content.value,
|
|
1200
1212
|
existing.relPath,
|
|
1201
|
-
existing.dateFields
|
|
1213
|
+
existing.dateFields,
|
|
1214
|
+
// Unchanged content (a re-ingest) reuses the current parse.
|
|
1215
|
+
current !== undefined && content.value === current.markdown
|
|
1216
|
+
? current.excludedRanges
|
|
1217
|
+
: getExcludedRanges(content.value)
|
|
1202
1218
|
);
|
|
1203
1219
|
}
|
|
1204
1220
|
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
3f23832d1ed8b19e24eef29d6ea888a8c84009e39c61981d0b665c95cd7d792e gno-browser-clipper-v2.8.1.zip
|