@wienerberliner/pi-markdown-reader 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +75 -0
- package/package.json +48 -0
- package/src/index.ts +13 -0
- package/src/markdown/frontmatter.ts +25 -0
- package/src/markdown/parse.ts +131 -0
- package/src/markdown/path-slugs.ts +13 -0
- package/src/markdown/sections.ts +94 -0
- package/src/markdown/types.ts +34 -0
- package/src/tools/fs.ts +136 -0
- package/src/tools/markdown-index.ts +60 -0
- package/src/tools/markdown-outline.ts +49 -0
- package/src/tools/markdown-read.ts +49 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Daniel
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# pi-markdown-reader
|
|
2
|
+
|
|
3
|
+
Pi extension tools for deterministic, structure-aware Markdown reading.
|
|
4
|
+
|
|
5
|
+
Instead of sampling long Markdown files with brittle line ranges, agents can inspect a compact outline first, then read complete sections by exact `pathSlug`.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pi install npm:@wienerberliner/pi-markdown-reader
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Then restart Pi or run `/reload`.
|
|
14
|
+
|
|
15
|
+
## Tools
|
|
16
|
+
|
|
17
|
+
### `markdown_outline`
|
|
18
|
+
|
|
19
|
+
Returns a flat, source-ordered table of contents for a Markdown file:
|
|
20
|
+
|
|
21
|
+
- heading level and title
|
|
22
|
+
- hierarchical `pathSlug`
|
|
23
|
+
- `startLine`, `endLine`, and `lineCount`
|
|
24
|
+
- optional frontmatter metadata (`pathSlug`, keys, line span)
|
|
25
|
+
|
|
26
|
+
Example:
|
|
27
|
+
|
|
28
|
+
```json
|
|
29
|
+
{
|
|
30
|
+
"path": "report.md",
|
|
31
|
+
"includeFrontmatter": true
|
|
32
|
+
}
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
### `markdown_read`
|
|
36
|
+
|
|
37
|
+
Reads one or more complete Markdown sections by exact `pathSlug`.
|
|
38
|
+
|
|
39
|
+
The LLM-facing result is plain Markdown text. Metadata stays in tool details.
|
|
40
|
+
|
|
41
|
+
Example:
|
|
42
|
+
|
|
43
|
+
```json
|
|
44
|
+
{
|
|
45
|
+
"path": "report.md",
|
|
46
|
+
"sections": [
|
|
47
|
+
{ "pathSlug": "abstract" },
|
|
48
|
+
{ "pathSlug": "results/evidence" }
|
|
49
|
+
]
|
|
50
|
+
}
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
If a document has YAML frontmatter, it can be read with:
|
|
54
|
+
|
|
55
|
+
```json
|
|
56
|
+
{ "pathSlug": "frontmatter" }
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### `markdown_index`
|
|
60
|
+
|
|
61
|
+
Indexes a directory of Markdown files without reading full bodies. Useful for folders of reports, notes, and generated artifacts.
|
|
62
|
+
|
|
63
|
+
## Development
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
npm install
|
|
67
|
+
npm test
|
|
68
|
+
npm run typecheck
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
For local Pi development in this repository, trust the project and reload Pi. The project `.pi/settings.json` shadows the published npm package and loads the local checkout.
|
|
72
|
+
|
|
73
|
+
## License
|
|
74
|
+
|
|
75
|
+
MIT
|
package/package.json
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@wienerberliner/pi-markdown-reader",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Pi extension tools for deterministic, structure-aware Markdown outline and section reading.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"author": "Daniel",
|
|
8
|
+
"keywords": [
|
|
9
|
+
"pi-package",
|
|
10
|
+
"pi-extension",
|
|
11
|
+
"markdown",
|
|
12
|
+
"markdown-reader"
|
|
13
|
+
],
|
|
14
|
+
"repository": {
|
|
15
|
+
"type": "git",
|
|
16
|
+
"url": "git+https://github.com/dasomji/pi-markdown-reader.git"
|
|
17
|
+
},
|
|
18
|
+
"bugs": {
|
|
19
|
+
"url": "https://github.com/dasomji/pi-markdown-reader/issues"
|
|
20
|
+
},
|
|
21
|
+
"homepage": "https://github.com/dasomji/pi-markdown-reader#readme",
|
|
22
|
+
"files": [
|
|
23
|
+
"src",
|
|
24
|
+
"README.md",
|
|
25
|
+
"LICENSE"
|
|
26
|
+
],
|
|
27
|
+
"pi": {
|
|
28
|
+
"extensions": [
|
|
29
|
+
"./src/index.ts"
|
|
30
|
+
]
|
|
31
|
+
},
|
|
32
|
+
"publishConfig": {
|
|
33
|
+
"access": "public"
|
|
34
|
+
},
|
|
35
|
+
"scripts": {
|
|
36
|
+
"test": "vitest run",
|
|
37
|
+
"typecheck": "tsc --noEmit"
|
|
38
|
+
},
|
|
39
|
+
"peerDependencies": {
|
|
40
|
+
"typebox": "*"
|
|
41
|
+
},
|
|
42
|
+
"devDependencies": {
|
|
43
|
+
"@types/node": "^26.1.1",
|
|
44
|
+
"typebox": "latest",
|
|
45
|
+
"typescript": "latest",
|
|
46
|
+
"vitest": "latest"
|
|
47
|
+
}
|
|
48
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { createMarkdownIndexTool } from "./tools/markdown-index.js";
|
|
2
|
+
import { createMarkdownOutlineTool } from "./tools/markdown-outline.js";
|
|
3
|
+
import { createMarkdownReadTool } from "./tools/markdown-read.js";
|
|
4
|
+
|
|
5
|
+
interface ExtensionAPI {
|
|
6
|
+
registerTool(tool: unknown): void;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export default function markdownReaderExtension(pi: ExtensionAPI) {
|
|
10
|
+
pi.registerTool(createMarkdownOutlineTool());
|
|
11
|
+
pi.registerTool(createMarkdownReadTool());
|
|
12
|
+
pi.registerTool(createMarkdownIndexTool());
|
|
13
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { FrontmatterSummary } from "./types.js";
|
|
2
|
+
|
|
3
|
+
export function findFrontmatter(lines: string[]): FrontmatterSummary | undefined {
|
|
4
|
+
if (lines.length === 0 || !/^---\s*$/.test(lines[0] ?? "")) return undefined;
|
|
5
|
+
|
|
6
|
+
for (let index = 1; index < lines.length; index += 1) {
|
|
7
|
+
if (/^(---|\.\.\.)\s*$/.test(lines[index] ?? "")) {
|
|
8
|
+
const rawLines = lines.slice(1, index);
|
|
9
|
+
const keys = rawLines
|
|
10
|
+
.map((line) => line.match(/^([A-Za-z0-9_.-]+)\s*:/)?.[1])
|
|
11
|
+
.filter((key): key is string => Boolean(key));
|
|
12
|
+
const endLine = index + 1;
|
|
13
|
+
|
|
14
|
+
return {
|
|
15
|
+
pathSlug: "frontmatter",
|
|
16
|
+
keys: [...new Set(keys)],
|
|
17
|
+
startLine: 1,
|
|
18
|
+
endLine,
|
|
19
|
+
lineCount: endLine,
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
return undefined;
|
|
25
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { findFrontmatter } from "./frontmatter.js";
|
|
2
|
+
import { slugSegment } from "./path-slugs.js";
|
|
3
|
+
import type { Heading, ParsedMarkdown } from "./types.js";
|
|
4
|
+
|
|
5
|
+
interface RawHeading {
|
|
6
|
+
level: number;
|
|
7
|
+
title: string;
|
|
8
|
+
startLine: number;
|
|
9
|
+
pathSlug: string;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
interface ParseOptions {
|
|
13
|
+
maxDepth?: number;
|
|
14
|
+
includeFrontmatter?: boolean;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function splitMarkdownLines(text: string): string[] {
|
|
18
|
+
const normalized = text.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
|
|
19
|
+
if (normalized.length === 0) return [];
|
|
20
|
+
const withoutFinalNewline = normalized.endsWith("\n") ? normalized.slice(0, -1) : normalized;
|
|
21
|
+
return withoutFinalNewline.split("\n");
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function parseMarkdown(text: string, options: ParseOptions = {}): ParsedMarkdown {
|
|
25
|
+
const maxDepth = options.maxDepth ?? 6;
|
|
26
|
+
const includeFrontmatter = options.includeFrontmatter ?? true;
|
|
27
|
+
const lines = splitMarkdownLines(text);
|
|
28
|
+
const frontmatter = findFrontmatter(lines);
|
|
29
|
+
const rawHeadings = scanHeadings(lines, frontmatter?.endLine ?? 0, Boolean(frontmatter));
|
|
30
|
+
const headings = computeHeadingSpans(rawHeadings, lines.length).filter((heading) => heading.level <= maxDepth);
|
|
31
|
+
const firstH1 = rawHeadings.find((heading) => heading.level === 1);
|
|
32
|
+
|
|
33
|
+
return {
|
|
34
|
+
title: firstH1?.title,
|
|
35
|
+
frontmatter: includeFrontmatter ? frontmatter : undefined,
|
|
36
|
+
frontmatterKeys: frontmatter?.keys,
|
|
37
|
+
totalLines: lines.length,
|
|
38
|
+
lines,
|
|
39
|
+
headings,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function scanHeadings(lines: string[], frontmatterEndLine: number, hasFrontmatter: boolean): RawHeading[] {
|
|
44
|
+
const headings: Omit<RawHeading, "pathSlug">[] = [];
|
|
45
|
+
let fenceMarker: "`" | "~" | undefined;
|
|
46
|
+
let fenceLength = 0;
|
|
47
|
+
|
|
48
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
49
|
+
const lineNumber = index + 1;
|
|
50
|
+
const line = lines[index] ?? "";
|
|
51
|
+
|
|
52
|
+
if (lineNumber <= frontmatterEndLine) continue;
|
|
53
|
+
|
|
54
|
+
const fenceMatch = line.match(/^ {0,3}(`{3,}|~{3,})/);
|
|
55
|
+
if (fenceMatch) {
|
|
56
|
+
const markerRun = fenceMatch[1] ?? "";
|
|
57
|
+
const marker = markerRun[0] as "`" | "~";
|
|
58
|
+
if (!fenceMarker) {
|
|
59
|
+
fenceMarker = marker;
|
|
60
|
+
fenceLength = markerRun.length;
|
|
61
|
+
} else if (marker === fenceMarker && markerRun.length >= fenceLength) {
|
|
62
|
+
fenceMarker = undefined;
|
|
63
|
+
fenceLength = 0;
|
|
64
|
+
}
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (fenceMarker) continue;
|
|
69
|
+
|
|
70
|
+
const headingMatch = line.match(/^ {0,3}(#{1,6})[ \t]+(.+?)\s*$/);
|
|
71
|
+
if (!headingMatch) continue;
|
|
72
|
+
|
|
73
|
+
const level = (headingMatch[1] ?? "").length;
|
|
74
|
+
const title = (headingMatch[2] ?? "").replace(/[ \t]+#+[ \t]*$/, "").trim();
|
|
75
|
+
if (!title) continue;
|
|
76
|
+
|
|
77
|
+
headings.push({ level, title, startLine: lineNumber });
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return assignPathSlugs(headings, hasFrontmatter);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function assignPathSlugs(headings: Omit<RawHeading, "pathSlug">[], hasFrontmatter: boolean): RawHeading[] {
|
|
84
|
+
const stack: RawHeading[] = [];
|
|
85
|
+
const siblingCounts = new Map<string, number>();
|
|
86
|
+
if (hasFrontmatter) siblingCounts.set("\u0000frontmatter", 1);
|
|
87
|
+
const result: RawHeading[] = [];
|
|
88
|
+
|
|
89
|
+
for (const heading of headings) {
|
|
90
|
+
while (stack.length > 0 && stack[stack.length - 1]!.level >= heading.level) {
|
|
91
|
+
stack.pop();
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const parentPathSlug = stack[stack.length - 1]?.pathSlug;
|
|
95
|
+
const baseSegment = slugSegment(heading.title);
|
|
96
|
+
const counterKey = `${parentPathSlug ?? ""}\u0000${baseSegment}`;
|
|
97
|
+
const previousCount = siblingCounts.get(counterKey) ?? 0;
|
|
98
|
+
siblingCounts.set(counterKey, previousCount + 1);
|
|
99
|
+
|
|
100
|
+
const segment = previousCount === 0 ? baseSegment : `${baseSegment}-${previousCount}`;
|
|
101
|
+
const rawHeading = {
|
|
102
|
+
...heading,
|
|
103
|
+
pathSlug: parentPathSlug ? `${parentPathSlug}/${segment}` : segment,
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
result.push(rawHeading);
|
|
107
|
+
stack.push(rawHeading);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
return result;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function computeHeadingSpans(rawHeadings: RawHeading[], totalLines: number): Heading[] {
|
|
114
|
+
return rawHeadings.map((heading, index) => {
|
|
115
|
+
let endLine = totalLines;
|
|
116
|
+
|
|
117
|
+
for (let nextIndex = index + 1; nextIndex < rawHeadings.length; nextIndex += 1) {
|
|
118
|
+
const nextHeading = rawHeadings[nextIndex]!;
|
|
119
|
+
if (nextHeading.level <= heading.level) {
|
|
120
|
+
endLine = nextHeading.startLine - 1;
|
|
121
|
+
break;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
return {
|
|
126
|
+
...heading,
|
|
127
|
+
endLine,
|
|
128
|
+
lineCount: Math.max(0, endLine - heading.startLine + 1),
|
|
129
|
+
};
|
|
130
|
+
});
|
|
131
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export function slugSegment(title: string): string {
|
|
2
|
+
const normalized = title
|
|
3
|
+
.normalize("NFKD")
|
|
4
|
+
.replace(/[\u0300-\u036f]/g, "")
|
|
5
|
+
.toLowerCase()
|
|
6
|
+
.trim()
|
|
7
|
+
.replace(/[^\p{Letter}\p{Number}\s_-]+/gu, "")
|
|
8
|
+
.replace(/[\s_]+/g, "-")
|
|
9
|
+
.replace(/-+/g, "-")
|
|
10
|
+
.replace(/^-|-$/g, "");
|
|
11
|
+
|
|
12
|
+
return normalized || "section";
|
|
13
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import type { Heading, ParsedMarkdown, SectionRead } from "./types.js";
|
|
2
|
+
|
|
3
|
+
export const DEFAULT_MAX_SECTION_LINES = 2_000;
|
|
4
|
+
export const DEFAULT_MAX_SECTION_BYTES = 50 * 1024;
|
|
5
|
+
|
|
6
|
+
const encoder = new TextEncoder();
|
|
7
|
+
|
|
8
|
+
export function readSectionByPathSlug(parsed: ParsedMarkdown, pathSlug: string): SectionRead {
|
|
9
|
+
if (parsed.frontmatter?.pathSlug === pathSlug) {
|
|
10
|
+
return readSection(parsed, {
|
|
11
|
+
level: 0,
|
|
12
|
+
title: "Frontmatter",
|
|
13
|
+
pathSlug: parsed.frontmatter.pathSlug,
|
|
14
|
+
startLine: parsed.frontmatter.startLine,
|
|
15
|
+
endLine: parsed.frontmatter.endLine,
|
|
16
|
+
lineCount: parsed.frontmatter.lineCount,
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const heading = parsed.headings.find((candidate) => candidate.pathSlug === pathSlug);
|
|
21
|
+
if (!heading) {
|
|
22
|
+
const available = [parsed.frontmatter?.pathSlug, ...parsed.headings.map((candidate) => candidate.pathSlug)]
|
|
23
|
+
.filter(Boolean)
|
|
24
|
+
.join(", ");
|
|
25
|
+
throw new Error(`No Markdown heading found for pathSlug "${pathSlug}".${available ? ` Available pathSlugs: ${available}` : ""}`);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
return readSection(parsed, heading);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function readSection(parsed: ParsedMarkdown, heading: Heading): SectionRead {
|
|
32
|
+
const sectionLines = parsed.lines.slice(heading.startLine - 1, heading.endLine);
|
|
33
|
+
const truncated = truncateLines(sectionLines);
|
|
34
|
+
const nextLine = truncated.truncated ? heading.startLine + truncated.returnedLines : undefined;
|
|
35
|
+
|
|
36
|
+
return {
|
|
37
|
+
heading,
|
|
38
|
+
content: truncated.content,
|
|
39
|
+
truncated: truncated.truncated,
|
|
40
|
+
totalLines: heading.lineCount,
|
|
41
|
+
returnedLines: truncated.returnedLines,
|
|
42
|
+
nextLine: nextLine && nextLine <= heading.endLine ? nextLine : undefined,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function truncateLines(lines: string[]): { content: string; truncated: boolean; returnedLines: number } {
|
|
47
|
+
const selected: string[] = [];
|
|
48
|
+
let bytes = 0;
|
|
49
|
+
let truncated = false;
|
|
50
|
+
|
|
51
|
+
for (const line of lines) {
|
|
52
|
+
if (selected.length >= DEFAULT_MAX_SECTION_LINES) {
|
|
53
|
+
truncated = true;
|
|
54
|
+
break;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const separatorBytes = selected.length > 0 ? 1 : 0;
|
|
58
|
+
const lineBytes = encoder.encode(line).length;
|
|
59
|
+
|
|
60
|
+
if (bytes + separatorBytes + lineBytes > DEFAULT_MAX_SECTION_BYTES) {
|
|
61
|
+
const remainingBytes = DEFAULT_MAX_SECTION_BYTES - bytes - separatorBytes;
|
|
62
|
+
if (remainingBytes > 0) {
|
|
63
|
+
selected.push(truncateStringToBytes(line, remainingBytes));
|
|
64
|
+
}
|
|
65
|
+
truncated = true;
|
|
66
|
+
break;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
selected.push(line);
|
|
70
|
+
bytes += separatorBytes + lineBytes;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
if (selected.length < lines.length) truncated = true;
|
|
74
|
+
|
|
75
|
+
return {
|
|
76
|
+
content: selected.join("\n"),
|
|
77
|
+
truncated,
|
|
78
|
+
returnedLines: selected.length,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function truncateStringToBytes(value: string, maxBytes: number): string {
|
|
83
|
+
let result = "";
|
|
84
|
+
let bytes = 0;
|
|
85
|
+
|
|
86
|
+
for (const char of value) {
|
|
87
|
+
const charBytes = encoder.encode(char).length;
|
|
88
|
+
if (bytes + charBytes > maxBytes) break;
|
|
89
|
+
result += char;
|
|
90
|
+
bytes += charBytes;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
return result;
|
|
94
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
export interface Heading {
|
|
2
|
+
level: number;
|
|
3
|
+
title: string;
|
|
4
|
+
pathSlug: string;
|
|
5
|
+
startLine: number;
|
|
6
|
+
endLine: number;
|
|
7
|
+
lineCount: number;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export interface FrontmatterSummary {
|
|
11
|
+
pathSlug: "frontmatter";
|
|
12
|
+
keys: string[];
|
|
13
|
+
startLine: number;
|
|
14
|
+
endLine: number;
|
|
15
|
+
lineCount: number;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface ParsedMarkdown {
|
|
19
|
+
title?: string;
|
|
20
|
+
frontmatter?: FrontmatterSummary;
|
|
21
|
+
frontmatterKeys?: string[];
|
|
22
|
+
totalLines: number;
|
|
23
|
+
lines: string[];
|
|
24
|
+
headings: Heading[];
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface SectionRead {
|
|
28
|
+
heading: Heading;
|
|
29
|
+
content: string;
|
|
30
|
+
truncated: boolean;
|
|
31
|
+
totalLines: number;
|
|
32
|
+
returnedLines: number;
|
|
33
|
+
nextLine?: number;
|
|
34
|
+
}
|
package/src/tools/fs.ts
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { readdir, readFile, stat } from "node:fs/promises";
|
|
2
|
+
import { isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
3
|
+
import { TextDecoder } from "node:util";
|
|
4
|
+
|
|
5
|
+
export interface MarkdownFile {
|
|
6
|
+
absolutePath: string;
|
|
7
|
+
displayPath: string;
|
|
8
|
+
text: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function normalizeInputPath(inputPath: string, cwd: string): string {
|
|
12
|
+
const withoutAt = inputPath.startsWith("@") ? inputPath.slice(1) : inputPath;
|
|
13
|
+
return isAbsolute(withoutAt) ? resolve(withoutAt) : resolve(cwd, withoutAt);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export async function readMarkdownText(inputPath: string, cwd: string): Promise<MarkdownFile> {
|
|
17
|
+
const absolutePath = normalizeInputPath(inputPath, cwd);
|
|
18
|
+
const stats = await stat(absolutePath).catch((error: unknown) => {
|
|
19
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
20
|
+
throw new Error(`Cannot read Markdown file ${inputPath}: ${message}`);
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
if (stats.isDirectory()) {
|
|
24
|
+
throw new Error(`Cannot read Markdown file ${inputPath}: path is a directory`);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
if (!stats.isFile()) {
|
|
28
|
+
throw new Error(`Cannot read Markdown file ${inputPath}: path is not a regular file`);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const buffer = await readFile(absolutePath);
|
|
32
|
+
if (buffer.includes(0)) {
|
|
33
|
+
throw new Error(`Cannot read Markdown file ${inputPath}: file appears to be binary`);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
let text: string;
|
|
37
|
+
try {
|
|
38
|
+
text = new TextDecoder("utf-8", { fatal: true }).decode(buffer);
|
|
39
|
+
} catch {
|
|
40
|
+
throw new Error(`Cannot read Markdown file ${inputPath}: file is not valid UTF-8`);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
return {
|
|
44
|
+
absolutePath,
|
|
45
|
+
displayPath: displayPathFor(absolutePath, cwd),
|
|
46
|
+
text,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export async function listMarkdownFiles(inputPath: string, cwd: string, glob: string, maxFiles: number): Promise<{ root: string; files: string[]; truncated: boolean }> {
|
|
51
|
+
const root = normalizeInputPath(inputPath, cwd);
|
|
52
|
+
const stats = await stat(root).catch((error: unknown) => {
|
|
53
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
54
|
+
throw new Error(`Cannot index Markdown path ${inputPath}: ${message}`);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
if (stats.isFile()) {
|
|
58
|
+
return { root, files: [root], truncated: false };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
if (!stats.isDirectory()) {
|
|
62
|
+
throw new Error(`Cannot index Markdown path ${inputPath}: path is not a file or directory`);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const matcher = globMatcher(glob);
|
|
66
|
+
const files: string[] = [];
|
|
67
|
+
let truncated = false;
|
|
68
|
+
|
|
69
|
+
async function visit(directory: string): Promise<void> {
|
|
70
|
+
if (files.length >= maxFiles) {
|
|
71
|
+
truncated = true;
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const entries = await readdir(directory, { withFileTypes: true });
|
|
76
|
+
entries.sort((a, b) => a.name.localeCompare(b.name));
|
|
77
|
+
|
|
78
|
+
for (const entry of entries) {
|
|
79
|
+
if (files.length >= maxFiles) {
|
|
80
|
+
truncated = true;
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
if (entry.name === ".git" || entry.name === "node_modules" || entry.name === ".pi") continue;
|
|
85
|
+
|
|
86
|
+
const absolute = join(directory, entry.name);
|
|
87
|
+
if (entry.isDirectory()) {
|
|
88
|
+
await visit(absolute);
|
|
89
|
+
} else if (entry.isFile()) {
|
|
90
|
+
const rel = relative(root, absolute).split(sep).join("/");
|
|
91
|
+
if (matcher(rel)) files.push(absolute);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
await visit(root);
|
|
97
|
+
return { root, files, truncated };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function displayPathFor(absolutePath: string, cwd: string): string {
|
|
101
|
+
const relativePath = relative(cwd, absolutePath);
|
|
102
|
+
if (!relativePath.startsWith("..") && !isAbsolute(relativePath)) return relativePath || ".";
|
|
103
|
+
return absolutePath;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function globMatcher(glob: string): (relativePath: string) => boolean {
|
|
107
|
+
if (glob === "**/*.md") return (relativePath) => relativePath.endsWith(".md");
|
|
108
|
+
if (glob === "**/*.{md,markdown}") return (relativePath) => /\.(md|markdown)$/i.test(relativePath);
|
|
109
|
+
|
|
110
|
+
const regex = new RegExp(`^${escapeGlob(glob)}$`);
|
|
111
|
+
return (relativePath) => regex.test(relativePath);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function escapeGlob(glob: string): string {
|
|
115
|
+
let output = "";
|
|
116
|
+
for (let index = 0; index < glob.length; index += 1) {
|
|
117
|
+
const char = glob[index];
|
|
118
|
+
const next = glob[index + 1];
|
|
119
|
+
|
|
120
|
+
if (char === "*" && next === "*") {
|
|
121
|
+
output += ".*";
|
|
122
|
+
index += 1;
|
|
123
|
+
} else if (char === "*") {
|
|
124
|
+
output += "[^/]*";
|
|
125
|
+
} else if (char === "?") {
|
|
126
|
+
output += "[^/]";
|
|
127
|
+
} else {
|
|
128
|
+
output += escapeRegex(char ?? "");
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return output;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function escapeRegex(value: string): string {
|
|
135
|
+
return value.replace(/[|\\{}()[\]^$+*?.]/g, "\\$&");
|
|
136
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import { parseMarkdown } from "../markdown/parse.js";
|
|
3
|
+
import { displayPathFor, listMarkdownFiles, readMarkdownText } from "./fs.js";
|
|
4
|
+
|
|
5
|
+
export const markdownIndexParameters = Type.Object({
|
|
6
|
+
path: Type.String({ description: "Directory or Markdown file to index. A leading @ is ignored." }),
|
|
7
|
+
glob: Type.Optional(Type.String({ description: "Glob to match Markdown files. Defaults to **/*.md." })),
|
|
8
|
+
maxFiles: Type.Optional(Type.Number({ minimum: 1, maximum: 500, description: "Maximum files to index. Defaults to 100." })),
|
|
9
|
+
maxDepth: Type.Optional(Type.Number({ minimum: 1, maximum: 6, description: "Maximum heading depth to include. Defaults to 6." })),
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
export interface MarkdownIndexParams {
|
|
13
|
+
path: string;
|
|
14
|
+
glob?: string;
|
|
15
|
+
maxFiles?: number;
|
|
16
|
+
maxDepth?: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function createMarkdownIndexTool() {
|
|
20
|
+
return {
|
|
21
|
+
name: "markdown_index",
|
|
22
|
+
label: "Markdown Index",
|
|
23
|
+
description: "Index a directory of Markdown files without reading full bodies. Returns file paths, line counts, frontmatter keys, and compact heading outlines.",
|
|
24
|
+
promptSnippet: "Index Markdown files in a directory without reading full bodies.",
|
|
25
|
+
promptGuidelines: [
|
|
26
|
+
"Use markdown_index to inspect folders of Markdown reports, notes, or generated artifacts before choosing files for markdown_outline and markdown_read.",
|
|
27
|
+
],
|
|
28
|
+
parameters: markdownIndexParameters,
|
|
29
|
+
|
|
30
|
+
async execute(_toolCallId: string, params: MarkdownIndexParams, _signal: AbortSignal | undefined, _onUpdate: unknown, ctx: { cwd: string }) {
|
|
31
|
+
const maxFiles = Math.min(Math.max(Math.floor(params.maxFiles ?? 100), 1), 500);
|
|
32
|
+
const listing = await listMarkdownFiles(params.path, ctx.cwd, params.glob ?? "**/*.md", maxFiles);
|
|
33
|
+
const files = [];
|
|
34
|
+
|
|
35
|
+
for (const absolutePath of listing.files) {
|
|
36
|
+
const file = await readMarkdownText(absolutePath, ctx.cwd);
|
|
37
|
+
const parsed = parseMarkdown(file.text, { maxDepth: params.maxDepth, includeFrontmatter: true });
|
|
38
|
+
files.push({
|
|
39
|
+
path: displayPathFor(absolutePath, ctx.cwd),
|
|
40
|
+
title: parsed.title,
|
|
41
|
+
lineCount: parsed.totalLines,
|
|
42
|
+
headings: parsed.headings,
|
|
43
|
+
frontmatter: parsed.frontmatter,
|
|
44
|
+
frontmatterKeys: parsed.frontmatterKeys,
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const output = {
|
|
49
|
+
root: displayPathFor(listing.root, ctx.cwd),
|
|
50
|
+
files,
|
|
51
|
+
truncated: listing.truncated,
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
return {
|
|
55
|
+
content: [{ type: "text", text: JSON.stringify(output, null, 2) }],
|
|
56
|
+
details: output,
|
|
57
|
+
};
|
|
58
|
+
},
|
|
59
|
+
};
|
|
60
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import { parseMarkdown } from "../markdown/parse.js";
|
|
3
|
+
import { readMarkdownText } from "./fs.js";
|
|
4
|
+
|
|
5
|
+
export const markdownOutlineParameters = Type.Object({
|
|
6
|
+
path: Type.String({ description: "Markdown file path. A leading @ is ignored." }),
|
|
7
|
+
maxDepth: Type.Optional(Type.Number({ minimum: 1, maximum: 6, description: "Maximum heading depth to include. Defaults to 6." })),
|
|
8
|
+
includeFrontmatter: Type.Optional(Type.Boolean({ description: "Include a bounded frontmatter summary. Defaults to true." })),
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
export interface MarkdownOutlineParams {
|
|
12
|
+
path: string;
|
|
13
|
+
maxDepth?: number;
|
|
14
|
+
includeFrontmatter?: boolean;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function createMarkdownOutlineTool() {
|
|
18
|
+
return {
|
|
19
|
+
name: "markdown_outline",
|
|
20
|
+
label: "Markdown Outline",
|
|
21
|
+
description: "Extract a deterministic Markdown table of contents with heading levels, path slugs, line spans, and total line count.",
|
|
22
|
+
promptSnippet: "Extract a Markdown table of contents with heading levels, path slugs, line spans, and total line count.",
|
|
23
|
+
promptGuidelines: [
|
|
24
|
+
"Use markdown_outline before reading long Markdown files or generated reports.",
|
|
25
|
+
],
|
|
26
|
+
parameters: markdownOutlineParameters,
|
|
27
|
+
|
|
28
|
+
async execute(_toolCallId: string, params: MarkdownOutlineParams, _signal: AbortSignal | undefined, _onUpdate: unknown, ctx: { cwd: string }) {
|
|
29
|
+
const file = await readMarkdownText(params.path, ctx.cwd);
|
|
30
|
+
const parsed = parseMarkdown(file.text, {
|
|
31
|
+
maxDepth: params.maxDepth,
|
|
32
|
+
includeFrontmatter: params.includeFrontmatter,
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
const output = {
|
|
36
|
+
path: file.displayPath,
|
|
37
|
+
title: parsed.title,
|
|
38
|
+
frontmatter: parsed.frontmatter,
|
|
39
|
+
totalLines: parsed.totalLines,
|
|
40
|
+
headings: parsed.headings,
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
return {
|
|
44
|
+
content: [{ type: "text", text: JSON.stringify(output, null, 2) }],
|
|
45
|
+
details: output,
|
|
46
|
+
};
|
|
47
|
+
},
|
|
48
|
+
};
|
|
49
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { Type } from "typebox";
|
|
2
|
+
import { parseMarkdown } from "../markdown/parse.js";
|
|
3
|
+
import { readSectionByPathSlug } from "../markdown/sections.js";
|
|
4
|
+
import { readMarkdownText } from "./fs.js";
|
|
5
|
+
|
|
6
|
+
export const markdownReadParameters = Type.Object({
|
|
7
|
+
path: Type.String({ description: "Markdown file path. A leading @ is ignored." }),
|
|
8
|
+
sections: Type.Array(
|
|
9
|
+
Type.Object({
|
|
10
|
+
pathSlug: Type.String({ description: "Exact pathSlug returned by markdown_outline." }),
|
|
11
|
+
}),
|
|
12
|
+
{ minItems: 1, description: "Sections to read by exact pathSlug." },
|
|
13
|
+
),
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
export interface MarkdownReadParams {
|
|
17
|
+
path: string;
|
|
18
|
+
sections: Array<{ pathSlug: string }>;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function createMarkdownReadTool() {
|
|
22
|
+
return {
|
|
23
|
+
name: "markdown_read",
|
|
24
|
+
label: "Markdown Read",
|
|
25
|
+
description: "Read one or more complete Markdown sections by exact pathSlug. Each section includes descendant subsections and is hard-truncated only for output safety.",
|
|
26
|
+
promptSnippet: "Read one or more Markdown sections by exact path slug.",
|
|
27
|
+
promptGuidelines: [
|
|
28
|
+
"Use markdown_read to inspect specific headings by pathSlug instead of guessing line ranges with read, head, tail, or sed. Always pass requested headings in a sections array, even for one section.",
|
|
29
|
+
"markdown_read only accepts exact pathSlug selectors from markdown_outline; call a subsection's own pathSlug when you only need that subsection.",
|
|
30
|
+
],
|
|
31
|
+
parameters: markdownReadParameters,
|
|
32
|
+
|
|
33
|
+
async execute(_toolCallId: string, params: MarkdownReadParams, _signal: AbortSignal | undefined, _onUpdate: unknown, ctx: { cwd: string }) {
|
|
34
|
+
const file = await readMarkdownText(params.path, ctx.cwd);
|
|
35
|
+
const parsed = parseMarkdown(file.text);
|
|
36
|
+
const sections = params.sections.map((section) => readSectionByPathSlug(parsed, section.pathSlug));
|
|
37
|
+
const output = {
|
|
38
|
+
path: file.displayPath,
|
|
39
|
+
totalLines: parsed.totalLines,
|
|
40
|
+
sections,
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
return {
|
|
44
|
+
content: [{ type: "text", text: sections.map((section) => section.content).join("\n\n") }],
|
|
45
|
+
details: output,
|
|
46
|
+
};
|
|
47
|
+
},
|
|
48
|
+
};
|
|
49
|
+
}
|