@dynamik-dev/refdocs 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +109 -0
- package/dist/src/chunker.d.ts +9 -0
- package/dist/src/chunker.d.ts.map +1 -0
- package/dist/src/chunker.js +205 -0
- package/dist/src/chunker.js.map +1 -0
- package/dist/src/config.d.ts +8 -0
- package/dist/src/config.d.ts.map +1 -0
- package/dist/src/config.js +77 -0
- package/dist/src/config.js.map +1 -0
- package/dist/src/index.d.ts +3 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +124 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/indexer.d.ts +9 -0
- package/dist/src/indexer.d.ts.map +1 -0
- package/dist/src/indexer.js +58 -0
- package/dist/src/indexer.js.map +1 -0
- package/dist/src/search.d.ts +11 -0
- package/dist/src/search.d.ts.map +1 -0
- package/dist/src/search.js +63 -0
- package/dist/src/search.js.map +1 -0
- package/dist/src/types.d.ts +39 -0
- package/dist/src/types.d.ts.map +1 -0
- package/dist/src/types.js +2 -0
- package/dist/src/types.js.map +1 -0
- package/dist/tests/chunker.test.d.ts +2 -0
- package/dist/tests/chunker.test.d.ts.map +1 -0
- package/dist/tests/chunker.test.js +245 -0
- package/dist/tests/chunker.test.js.map +1 -0
- package/dist/tests/cli.test.d.ts +2 -0
- package/dist/tests/cli.test.d.ts.map +1 -0
- package/dist/tests/cli.test.js +166 -0
- package/dist/tests/cli.test.js.map +1 -0
- package/dist/tests/config.test.d.ts +2 -0
- package/dist/tests/config.test.d.ts.map +1 -0
- package/dist/tests/config.test.js +94 -0
- package/dist/tests/config.test.js.map +1 -0
- package/dist/tests/indexer.test.d.ts +2 -0
- package/dist/tests/indexer.test.d.ts.map +1 -0
- package/dist/tests/indexer.test.js +109 -0
- package/dist/tests/indexer.test.js.map +1 -0
- package/dist/tests/search.test.d.ts +2 -0
- package/dist/tests/search.test.d.ts.map +1 -0
- package/dist/tests/search.test.js +160 -0
- package/dist/tests/search.test.js.map +1 -0
- package/package.json +39 -0
package/README.md
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# refdocs
|
|
2
|
+
|
|
3
|
+
Index your markdown docs. Search them fast. Get back only what matters.
|
|
4
|
+
|
|
5
|
+
Built for LLM coding agents that need token-conscious access to project documentation — no network calls, no API keys, no MCP servers. Just a single binary and a JSON index file.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
$ refdocs search "database connections"
|
|
9
|
+
|
|
10
|
+
# [1] config/database.md:12-34
|
|
11
|
+
# Configuration > Database > Connections
|
|
12
|
+
|
|
13
|
+
Connection pooling is configured via the `pool` key in your
|
|
14
|
+
database config. Each connection type supports `min`, `max`,
|
|
15
|
+
and `idle_timeout` options...
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
# [2] guides/troubleshooting.md:88-104
|
|
20
|
+
# Troubleshooting > Database > Connection Refused
|
|
21
|
+
|
|
22
|
+
If you see "ECONNREFUSED", check that your database server
|
|
23
|
+
is running and the host/port in your config matches...
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
refdocs chunks markdown at heading boundaries into 100-800 token pieces, indexes them with fuzzy search, and returns only the relevant chunks — not entire files.
|
|
27
|
+
|
|
28
|
+
## Install
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
npm install -g @dynamik-dev/refdocs
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Or build from source:
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
bun install && bun run build
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Produces a standalone `./refdocs` binary. Or run directly:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
bun src/index.ts <command>
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Usage
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
# Point at your docs directory
|
|
50
|
+
echo '{ "paths": ["docs"] }' > .refdocs.json
|
|
51
|
+
|
|
52
|
+
# Build the index
|
|
53
|
+
refdocs index
|
|
54
|
+
# Indexed 42 files -> 156 chunks (45.2 KB, 320ms)
|
|
55
|
+
|
|
56
|
+
# Search
|
|
57
|
+
refdocs search "authentication"
|
|
58
|
+
refdocs search "config" -n 5 # top 5 results
|
|
59
|
+
refdocs search "api" -f "api/**/*.md" # filter by file glob
|
|
60
|
+
refdocs search "hooks" --json # structured output
|
|
61
|
+
refdocs search "auth" --raw # body only, for piping
|
|
62
|
+
|
|
63
|
+
# Inspect the index
|
|
64
|
+
refdocs list # files and chunk counts
|
|
65
|
+
refdocs info "api/auth.md" # chunks in a specific file
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## How it works
|
|
69
|
+
|
|
70
|
+
1. **Index** — parses each `.md` file into an AST, splits at h1/h2/h3 boundaries, merges small sections, splits large ones at paragraph breaks. Each chunk keeps its full heading breadcrumb (`Config > Database > Connections`).
|
|
71
|
+
|
|
72
|
+
2. **Search** — fuzzy matching (20% edit tolerance) with prefix search and field boosting. Titles weighted 2x, headings 1.5x, body 1x. Results ranked by TF-IDF. File-level glob filtering via `-f`.
|
|
73
|
+
|
|
74
|
+
3. **Output** — human-readable by default, `--json` for structured consumption, `--raw` for piping. Each result includes source file, line range, and heading trail.
|
|
75
|
+
|
|
76
|
+
## Configuration
|
|
77
|
+
|
|
78
|
+
`.refdocs.json` at project root:
|
|
79
|
+
|
|
80
|
+
```json
|
|
81
|
+
{
|
|
82
|
+
"paths": ["docs"],
|
|
83
|
+
"index": ".refdocs-index.json",
|
|
84
|
+
"chunkMaxTokens": 800,
|
|
85
|
+
"chunkMinTokens": 100,
|
|
86
|
+
"boostFields": { "title": 2, "headings": 1.5, "body": 1 }
|
|
87
|
+
}
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
All fields optional. See [Configuration](docs/configuration.md) for details.
|
|
91
|
+
|
|
92
|
+
## Documentation
|
|
93
|
+
|
|
94
|
+
- [Getting Started](docs/getting-started.md) — installation, quick start, and overview
|
|
95
|
+
- [CLI Reference](docs/cli-reference.md) — commands, flags, output formats, and exit codes
|
|
96
|
+
- [Configuration](docs/configuration.md) — `.refdocs.json` options with defaults and examples
|
|
97
|
+
- [Chunking](docs/chunking.md) — the 3-pass splitting algorithm and chunk structure
|
|
98
|
+
- [Search](docs/search.md) — fuzzy matching, boosting, scoring, and index persistence
|
|
99
|
+
|
|
100
|
+
## Tech
|
|
101
|
+
|
|
102
|
+
| Dependency | Role |
|
|
103
|
+
|------------|------|
|
|
104
|
+
| [MiniSearch](https://github.com/lucaong/minisearch) | Full-text fuzzy search (~7kb, pure JS) |
|
|
105
|
+
| [Commander](https://github.com/tj/commander.js) | CLI framework |
|
|
106
|
+
| [mdast-util-from-markdown](https://github.com/syntax-tree/mdast-util-from-markdown) | Markdown AST parsing |
|
|
107
|
+
| [picomatch](https://github.com/micromatch/picomatch) | Glob pattern matching |
|
|
108
|
+
|
|
109
|
+
Zero external services. Works offline, in containers, on planes.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Chunk } from "./types.js";
|
|
2
|
+
interface ChunkOptions {
|
|
3
|
+
maxTokens: number;
|
|
4
|
+
minTokens: number;
|
|
5
|
+
}
|
|
6
|
+
export declare function estimateTokens(text: string): number;
|
|
7
|
+
export declare function chunkMarkdown(content: string, filePath: string, options: ChunkOptions): Chunk[];
|
|
8
|
+
export {};
|
|
9
|
+
//# sourceMappingURL=chunker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAGxC,UAAU,YAAY;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAYD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,YAAY,GACpB,KAAK,EAAE,CA6CT"}
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import { fromMarkdown } from "mdast-util-from-markdown";
|
|
2
|
+
const FRONTMATTER_RE = /^---\r?\n[\s\S]*?\r?\n---\r?\n?/;
|
|
3
|
+
export function estimateTokens(text) {
|
|
4
|
+
return Math.ceil(text.length / 4);
|
|
5
|
+
}
|
|
6
|
+
export function chunkMarkdown(content, filePath, options) {
|
|
7
|
+
if (!content.trim())
|
|
8
|
+
return [];
|
|
9
|
+
// Strip YAML front matter before parsing
|
|
10
|
+
const stripped = content.replace(FRONTMATTER_RE, "");
|
|
11
|
+
if (!stripped.trim())
|
|
12
|
+
return [];
|
|
13
|
+
const lines = content.split("\n");
|
|
14
|
+
const strippedLines = stripped.split("\n");
|
|
15
|
+
const lineOffset = lines.length - strippedLines.length;
|
|
16
|
+
const tree = fromMarkdown(stripped);
|
|
17
|
+
// Pass 1: Extract sections at heading boundaries
|
|
18
|
+
const sections = extractSections(tree.children, strippedLines, lineOffset);
|
|
19
|
+
if (sections.length === 0) {
|
|
20
|
+
// No headings found — treat entire content as a single chunk
|
|
21
|
+
const body = stripped.trim();
|
|
22
|
+
if (!body)
|
|
23
|
+
return [];
|
|
24
|
+
return [
|
|
25
|
+
makeChunk(filePath, fileTitle(filePath), [fileTitle(filePath)], body, lineOffset + 1, lines.length, 0),
|
|
26
|
+
];
|
|
27
|
+
}
|
|
28
|
+
// Pass 2: Merge small sections
|
|
29
|
+
const merged = mergeSections(sections, options.minTokens);
|
|
30
|
+
// Pass 3: Split oversized sections
|
|
31
|
+
const final = splitSections(merged, options.maxTokens);
|
|
32
|
+
// Filter out sections with empty bodies
|
|
33
|
+
const nonEmpty = final.filter((s) => s.body.trim().length > 0);
|
|
34
|
+
return nonEmpty.map((section, i) => makeChunk(filePath, section.headings[section.headings.length - 1] || fileTitle(filePath), section.headings, section.body, section.startLine, section.endLine, i));
|
|
35
|
+
}
|
|
36
|
+
function extractSections(children, lines, lineOffset) {
|
|
37
|
+
const sections = [];
|
|
38
|
+
const headingStack = [];
|
|
39
|
+
let currentStart = null;
|
|
40
|
+
let currentBody = "";
|
|
41
|
+
function pushCurrentSection(endLine) {
|
|
42
|
+
if (currentStart !== null) {
|
|
43
|
+
const body = currentBody.trim();
|
|
44
|
+
sections.push({
|
|
45
|
+
headings: headingStack.map((h) => h.text),
|
|
46
|
+
depth: headingStack.length > 0 ? headingStack[headingStack.length - 1].depth : 0,
|
|
47
|
+
body,
|
|
48
|
+
startLine: currentStart + lineOffset,
|
|
49
|
+
endLine: endLine + lineOffset,
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
// Check if there's content before the first heading
|
|
54
|
+
const firstHeading = children.find((n) => n.type === "heading" && n.depth <= 3);
|
|
55
|
+
if (firstHeading && firstHeading.position) {
|
|
56
|
+
const firstHeadingLine = firstHeading.position.start.line;
|
|
57
|
+
if (firstHeadingLine > 1) {
|
|
58
|
+
const preambleLines = lines.slice(0, firstHeadingLine - 1);
|
|
59
|
+
const preambleBody = preambleLines.join("\n").trim();
|
|
60
|
+
if (preambleBody) {
|
|
61
|
+
sections.push({
|
|
62
|
+
headings: [fileTitle( /* deferred — caller knows filePath */)],
|
|
63
|
+
depth: 0,
|
|
64
|
+
body: preambleBody,
|
|
65
|
+
startLine: 1 + lineOffset,
|
|
66
|
+
endLine: firstHeadingLine - 1 + lineOffset,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
for (const node of children) {
|
|
72
|
+
if (node.type === "heading" && node.depth <= 3 && node.position) {
|
|
73
|
+
const headingLine = node.position.start.line;
|
|
74
|
+
// Flush previous section
|
|
75
|
+
if (currentStart !== null) {
|
|
76
|
+
pushCurrentSection(headingLine - 1);
|
|
77
|
+
currentBody = "";
|
|
78
|
+
}
|
|
79
|
+
const headingText = extractText(node);
|
|
80
|
+
const depth = node.depth;
|
|
81
|
+
// Update heading stack: pop to depth < current, then push
|
|
82
|
+
while (headingStack.length > 0 && headingStack[headingStack.length - 1].depth >= depth) {
|
|
83
|
+
headingStack.pop();
|
|
84
|
+
}
|
|
85
|
+
headingStack.push({ text: headingText, depth });
|
|
86
|
+
currentStart = headingLine;
|
|
87
|
+
currentBody = "";
|
|
88
|
+
}
|
|
89
|
+
else if (node.position) {
|
|
90
|
+
// Accumulate body content (including h4-h6)
|
|
91
|
+
const nodeStart = node.position.start.line - 1;
|
|
92
|
+
const nodeEnd = node.position.end.line;
|
|
93
|
+
const text = lines.slice(nodeStart, nodeEnd).join("\n");
|
|
94
|
+
if (currentBody)
|
|
95
|
+
currentBody += "\n\n";
|
|
96
|
+
currentBody += text;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
// Flush last section
|
|
100
|
+
if (currentStart !== null) {
|
|
101
|
+
pushCurrentSection(lines.length);
|
|
102
|
+
}
|
|
103
|
+
return sections;
|
|
104
|
+
}
|
|
105
|
+
function mergeSections(sections, minTokens) {
|
|
106
|
+
if (sections.length <= 1)
|
|
107
|
+
return sections;
|
|
108
|
+
const result = [sections[0]];
|
|
109
|
+
for (let i = 1; i < sections.length; i++) {
|
|
110
|
+
const current = sections[i];
|
|
111
|
+
const prev = result[result.length - 1];
|
|
112
|
+
const tokens = estimateTokens(current.body);
|
|
113
|
+
// Only merge siblings (same depth) when the current section is small
|
|
114
|
+
if (tokens < minTokens && prev.depth === current.depth) {
|
|
115
|
+
const heading = formatHeadingLine(current);
|
|
116
|
+
const addition = heading ? heading + "\n\n" + current.body : current.body;
|
|
117
|
+
prev.body = prev.body ? prev.body + "\n\n" + addition : addition;
|
|
118
|
+
prev.endLine = current.endLine;
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
result.push({ ...current });
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return result;
|
|
125
|
+
}
|
|
126
|
+
function formatHeadingLine(section) {
|
|
127
|
+
if (section.headings.length === 0)
|
|
128
|
+
return "";
|
|
129
|
+
const last = section.headings[section.headings.length - 1];
|
|
130
|
+
const hashes = "#".repeat(section.depth || 1);
|
|
131
|
+
return `${hashes} ${last}`;
|
|
132
|
+
}
|
|
133
|
+
function splitSections(sections, maxTokens) {
|
|
134
|
+
const result = [];
|
|
135
|
+
for (const section of sections) {
|
|
136
|
+
const tokens = estimateTokens(section.body);
|
|
137
|
+
if (tokens <= maxTokens) {
|
|
138
|
+
result.push(section);
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
// Split at paragraph boundaries
|
|
142
|
+
const paragraphs = section.body.split(/\n\n+/);
|
|
143
|
+
let accumBody = "";
|
|
144
|
+
let subStart = section.startLine;
|
|
145
|
+
const totalLines = section.endLine - section.startLine + 1;
|
|
146
|
+
let linesConsumed = 0;
|
|
147
|
+
for (let i = 0; i < paragraphs.length; i++) {
|
|
148
|
+
const para = paragraphs[i];
|
|
149
|
+
const candidate = accumBody ? accumBody + "\n\n" + para : para;
|
|
150
|
+
if (estimateTokens(candidate) > maxTokens && accumBody) {
|
|
151
|
+
// Flush accumulated
|
|
152
|
+
const bodyLines = accumBody.split("\n").length;
|
|
153
|
+
result.push({
|
|
154
|
+
...section,
|
|
155
|
+
body: accumBody,
|
|
156
|
+
startLine: subStart,
|
|
157
|
+
endLine: subStart + bodyLines - 1,
|
|
158
|
+
});
|
|
159
|
+
linesConsumed += bodyLines + 1; // +1 for paragraph gap
|
|
160
|
+
subStart = section.startLine + linesConsumed;
|
|
161
|
+
accumBody = para;
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
164
|
+
accumBody = candidate;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
// Flush remainder
|
|
168
|
+
if (accumBody.trim()) {
|
|
169
|
+
result.push({
|
|
170
|
+
...section,
|
|
171
|
+
body: accumBody,
|
|
172
|
+
startLine: subStart,
|
|
173
|
+
endLine: section.endLine,
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return result;
|
|
178
|
+
}
|
|
179
|
+
function extractText(node) {
|
|
180
|
+
if ("value" in node && typeof node.value === "string")
|
|
181
|
+
return node.value;
|
|
182
|
+
if ("children" in node) {
|
|
183
|
+
return node.children.map(extractText).join("");
|
|
184
|
+
}
|
|
185
|
+
return "";
|
|
186
|
+
}
|
|
187
|
+
function fileTitle(filePath) {
|
|
188
|
+
if (!filePath)
|
|
189
|
+
return "Untitled";
|
|
190
|
+
const name = filePath.split("/").pop() || filePath;
|
|
191
|
+
return name.replace(/\.md$/i, "");
|
|
192
|
+
}
|
|
193
|
+
function makeChunk(file, title, headings, body, startLine, endLine, index) {
|
|
194
|
+
return {
|
|
195
|
+
id: `${file}:${index}`,
|
|
196
|
+
file,
|
|
197
|
+
title,
|
|
198
|
+
headings: headings.join(" > "),
|
|
199
|
+
body,
|
|
200
|
+
startLine,
|
|
201
|
+
endLine,
|
|
202
|
+
tokenEstimate: estimateTokens(body),
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
//# sourceMappingURL=chunker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAiBxD,MAAM,cAAc,GAAG,iCAAiC,CAAC;AAEzD,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,OAAe,EACf,QAAgB,EAChB,OAAqB;IAErB,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAE/B,yCAAyC;IACzC,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IACrD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAEhC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;IAEvD,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEpC,iDAAiD;IACjD,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,EAAE,UAAU,CAAC,CAAC;IAE3E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,6DAA6D;QAC7D,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QACrB,OAAO;YACL,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;SACvG,CAAC;IACJ,CAAC;IAED,+BAA+B;IAC/B,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAE1D,mCAAmC;IACnC,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAEvD,wCAAwC;IACxC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE/D,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CACjC,SAAS,CACP,QAAQ,EACR,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,EACpE,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,IAAI,EACZ,OAAO,CAAC,SAAS,EACjB,OAAO,CAAC,OAAO,EACf,CAAC,CACF,CACF,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CACtB,QAAmB,EACnB,KAAe,EACf,UAAkB;IAElB,MAAM,QAAQ,GAAiB,EAAE,CAAC;IAClC,MAAM,YAAY,GAAsC,EAAE,CAAC;IAC3D,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,IAAI,WAAW,GAAG,EAAE,CAAC;IAErB,SAAS,kBAAkB,CAAC,OAAe;QACzC,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;YAChC,QAAQ,CAAC,IAAI,CAAC;gBACZ,QAAQ,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;gBACzC,KAAK,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAChF,IAAI;gBACJ,SAAS,EAAE,YAAY,GAAG,UAAU;gBACpC,OAAO,EAAE,OAAO,GAAG,UAAU;aAC9B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,oDAAoD;IACpD,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAChC,CAAC,CAAC,EAAgB,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAC1D,CAAC;IAEF,IAAI,YAAY,IAAI,YAAY,CAAC,QAAQ,EAAE,CAAC;QAC1C,MAAM,gBAAgB,GAAG,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QAC1D,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,GAAG,CAAC,CAAC,CAAC;YAC3D,MAAM,YAAY,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YACrD,IAAI,YAAY,EAAE,CAAC;gBACjB,QAAQ,CAAC,IAAI,CAAC;oBACZ,QAAQ,EAAE,CAAC,SAAS,EAAC,sCAAsC,CAAC,CAAC;oBAC7D,KAAK,EAAE,CAAC;oBACR,IAAI,EAAE,YAAY;oBAClB,SAAS,EAAE,CAAC,GAAG,UAAU;oBACzB,OAAO,EAAE,gBAAgB,GAAG,CAAC,GAAG,UAAU;iBAC3C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChE,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;YAE7C,yBAAyB;YACzB,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;gBAC1B,kBAAkB,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;gBACpC,WAAW,GAAG,EAAE,CAAC;YACnB,CAAC;YAED,MAAM,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YACtC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;YAEzB,0DAA0D;YAC1D,OAAO,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC;gBACvF,YAAY,CAAC,GAAG,EAAE,CAAC;YACrB,CAAC;YACD,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YAEhD,YAAY,GAAG,WAAW,CAAC;YAC3B,WAAW,GAAG,EAAE,CAAC;QACnB,CAAC;aAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACzB,4CAA4C;YAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC;YAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;YACvC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxD,IAAI,WAAW;gBAAE,WAAW,IAAI,MAAM,CAAC;YACvC,WAAW,IAAI,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;QAC1B,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,aAAa,CAAC,QAAsB,EAAE,SAAiB;IAC9D,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,QAAQ,CAAC;IAE1C,MAAM,MAAM,GAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAE3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAE5C,qEAAqE;QACrE,IAAI,MAAM,GAAG,SAAS,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,CAAC,KAAK,EAAE,CAAC;YACvD,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;YAC1E,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;YACjE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAmB;IAC5C,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAC7C,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC3D,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAC9C,OAAO,GAAG,MAAM,IAAI,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,aAAa,CAAC,QAAsB,EAAE,SAAiB;IAC9D,MAAM,MAAM,GAAiB,EAAE,CAAC;IAEhC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,SAAS;QACX,CAAC;QAED,gCAAgC;QAChC,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/C,IAAI,SAAS,GAAG,EAAE,CAAC;QACnB,IAAI,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC;QACjC,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QAC3D,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAC3B,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,GAAG,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAE/D,IAAI,cAAc,CAAC,SAAS,CAAC,GAAG,SAAS,IAAI,SAAS,EAAE,CAAC;gBACvD,oBAAoB;gBACpB,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC;oBACV,GAAG,OAAO;oBACV,IAAI,EAAE,SAAS;oBACf,SAAS,EAAE,QAAQ;oBACnB,OAAO,EAAE,QAAQ,GAAG,SAAS,GAAG,CAAC;iBAClC,CAAC,CAAC;gBACH,aAAa,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC,uBAAuB;gBACvD,QAAQ,GAAG,OAAO,CAAC,SAAS,GAAG,aAAa,CAAC;gBAC7C,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACN,SAAS,GAAG,SAAS,CAAC;YACxB,CAAC;QACH,CAAC;QAED,kBAAkB;QAClB,IAAI,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC;gBACV,GAAG,OAAO;gBACV,IAAI,EAAE,SAAS;gBACf,SAAS,EAAE,QAAQ;gBACnB,OAAO,EAAE,OAAO,CAAC,OAAO;aACzB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,IAAa;IAChC,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC;IACzE,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;QACvB,OAAQ,IAAI,CAAC,QAAsB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,SAAS,CAAC,QAAiB;IAClC,IAAI,CAAC,QAAQ;QAAE,OAAO,UAAU,CAAC;IACjC,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,QAAQ,CAAC;IACnD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,SAAS,CAChB,IAAY,EACZ,KAAa,EACb,QAAkB,EAClB,IAAY,EACZ,SAAiB,EACjB,OAAe,EACf,KAAa;IAEb,OAAO;QACL,EAAE,EAAE,GAAG,IAAI,IAAI,KAAK,EAAE;QACtB,IAAI;QACJ,KAAK;QACL,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC;QAC9B,IAAI;QACJ,SAAS;QACT,OAAO;QACP,aAAa,EAAE,cAAc,CAAC,IAAI,CAAC;KACpC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { RefdocsConfig } from "./types.js";
|
|
2
|
+
export interface ConfigResult {
|
|
3
|
+
config: RefdocsConfig;
|
|
4
|
+
configDir: string;
|
|
5
|
+
}
|
|
6
|
+
export declare function loadConfig(cwd?: string): ConfigResult;
|
|
7
|
+
export declare function validateConfig(raw: unknown): string[];
|
|
8
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAgBhD,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,aAAa,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,UAAU,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,YAAY,CAyBrD;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,EAAE,CA4CrD"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { readFileSync, existsSync } from "node:fs";
|
|
2
|
+
import { join, dirname, resolve } from "node:path";
|
|
3
|
+
const CONFIG_FILENAME = ".refdocs.json";
|
|
4
|
+
const DEFAULT_CONFIG = {
|
|
5
|
+
paths: ["ref-docs"],
|
|
6
|
+
index: ".refdocs-index.json",
|
|
7
|
+
chunkMaxTokens: 800,
|
|
8
|
+
chunkMinTokens: 100,
|
|
9
|
+
boostFields: {
|
|
10
|
+
title: 2,
|
|
11
|
+
headings: 1.5,
|
|
12
|
+
body: 1,
|
|
13
|
+
},
|
|
14
|
+
};
|
|
15
|
+
export function loadConfig(cwd) {
|
|
16
|
+
const startDir = resolve(cwd ?? process.cwd());
|
|
17
|
+
let dir = startDir;
|
|
18
|
+
while (true) {
|
|
19
|
+
const configPath = join(dir, CONFIG_FILENAME);
|
|
20
|
+
if (existsSync(configPath)) {
|
|
21
|
+
const raw = JSON.parse(readFileSync(configPath, "utf-8"));
|
|
22
|
+
const errors = validateConfig(raw);
|
|
23
|
+
if (errors.length > 0) {
|
|
24
|
+
throw new Error(`Invalid ${CONFIG_FILENAME}: ${errors.join("; ")}`);
|
|
25
|
+
}
|
|
26
|
+
return {
|
|
27
|
+
config: { ...DEFAULT_CONFIG, ...raw, boostFields: { ...DEFAULT_CONFIG.boostFields, ...raw.boostFields } },
|
|
28
|
+
configDir: dir,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
const parent = dirname(dir);
|
|
32
|
+
if (parent === dir)
|
|
33
|
+
break;
|
|
34
|
+
dir = parent;
|
|
35
|
+
}
|
|
36
|
+
return { config: DEFAULT_CONFIG, configDir: startDir };
|
|
37
|
+
}
|
|
38
|
+
export function validateConfig(raw) {
|
|
39
|
+
const errors = [];
|
|
40
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
|
|
41
|
+
return ["Config must be a JSON object"];
|
|
42
|
+
}
|
|
43
|
+
const obj = raw;
|
|
44
|
+
if (obj.paths !== undefined) {
|
|
45
|
+
if (!Array.isArray(obj.paths) || !obj.paths.every((p) => typeof p === "string")) {
|
|
46
|
+
errors.push('"paths" must be an array of strings');
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
if (obj.index !== undefined && typeof obj.index !== "string") {
|
|
50
|
+
errors.push('"index" must be a string');
|
|
51
|
+
}
|
|
52
|
+
if (obj.chunkMaxTokens !== undefined) {
|
|
53
|
+
if (typeof obj.chunkMaxTokens !== "number" || obj.chunkMaxTokens <= 0) {
|
|
54
|
+
errors.push('"chunkMaxTokens" must be a positive number');
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
if (obj.chunkMinTokens !== undefined) {
|
|
58
|
+
if (typeof obj.chunkMinTokens !== "number" || obj.chunkMinTokens <= 0) {
|
|
59
|
+
errors.push('"chunkMinTokens" must be a positive number');
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
if (obj.boostFields !== undefined) {
|
|
63
|
+
if (typeof obj.boostFields !== "object" || obj.boostFields === null || Array.isArray(obj.boostFields)) {
|
|
64
|
+
errors.push('"boostFields" must be an object');
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
const bf = obj.boostFields;
|
|
68
|
+
for (const key of ["title", "headings", "body"]) {
|
|
69
|
+
if (bf[key] !== undefined && typeof bf[key] !== "number") {
|
|
70
|
+
errors.push(`"boostFields.${key}" must be a number`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return errors;
|
|
76
|
+
}
|
|
77
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGnD,MAAM,eAAe,GAAG,eAAe,CAAC;AAExC,MAAM,cAAc,GAAkB;IACpC,KAAK,EAAE,CAAC,UAAU,CAAC;IACnB,KAAK,EAAE,qBAAqB;IAC5B,cAAc,EAAE,GAAG;IACnB,cAAc,EAAE,GAAG;IACnB,WAAW,EAAE;QACX,KAAK,EAAE,CAAC;QACR,QAAQ,EAAE,GAAG;QACb,IAAI,EAAE,CAAC;KACR;CACF,CAAC;AAOF,MAAM,UAAU,UAAU,CAAC,GAAY;IACrC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC/C,IAAI,GAAG,GAAG,QAAQ,CAAC;IAEnB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;QAC9C,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;YAC1D,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CACb,WAAW,eAAe,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACnD,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,MAAM,EAAE,EAAE,GAAG,cAAc,EAAE,GAAG,GAAG,EAAE,WAAW,EAAE,EAAE,GAAG,cAAc,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,WAAW,EAAE,EAAE;gBACzG,SAAS,EAAE,GAAG;aACf,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,MAAM,KAAK,GAAG;YAAE,MAAM;QAC1B,GAAG,GAAG,MAAM,CAAC;IACf,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAY;IACzC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAClE,OAAO,CAAC,8BAA8B,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,GAAG,GAAG,GAA8B,CAAC;IAE3C,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,EAAE,CAAC;YAChF,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC7D,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,GAAG,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QACrC,IAAI,OAAO,GAAG,CAAC,cAAc,KAAK,QAAQ,IAAI,GAAG,CAAC,cAAc,IAAI,CAAC,EAAE,CAAC;YACtE,MAAM,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAED,IAAI,GAAG,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QACrC,IAAI,OAAO,GAAG,CAAC,cAAc,KAAK,QAAQ,IAAI,GAAG,CAAC,cAAc,IAAI,CAAC,EAAE,CAAC;YACtE,MAAM,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAED,IAAI,GAAG,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QAClC,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,IAAI,GAAG,CAAC,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YACtG,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,MAAM,EAAE,GAAG,GAAG,CAAC,WAAsC,CAAC;YACtD,KAAK,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC;gBAChD,IAAI,EAAE,CAAC,GAAG,CAAC,KAAK,SAAS,IAAI,OAAO,EAAE,CAAC,GAAG,CAAC,KAAK,QAAQ,EAAE,CAAC;oBACzD,MAAM,CAAC,IAAI,CAAC,gBAAgB,GAAG,oBAAoB,CAAC,CAAC;gBACvD,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { Command } from "commander";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { loadConfig } from "./config.js";
|
|
5
|
+
import { buildIndex, loadPersistedIndex } from "./indexer.js";
|
|
6
|
+
import { search } from "./search.js";
|
|
7
|
+
const program = new Command();
|
|
8
|
+
program
|
|
9
|
+
.name("refdocs")
|
|
10
|
+
.description("Local CLI tool for indexing and searching markdown documentation")
|
|
11
|
+
.version("0.1.0");
|
|
12
|
+
program
|
|
13
|
+
.command("index")
|
|
14
|
+
.description("Index all markdown files in configured paths")
|
|
15
|
+
.action(() => {
|
|
16
|
+
try {
|
|
17
|
+
const { config, configDir } = loadConfig();
|
|
18
|
+
const summary = buildIndex(config, configDir);
|
|
19
|
+
console.log(`Indexed ${summary.filesIndexed} files → ${summary.chunksCreated} chunks`);
|
|
20
|
+
console.log(`Index size: ${(summary.indexSizeBytes / 1024).toFixed(1)} KB`);
|
|
21
|
+
console.log(`Done in ${summary.elapsedMs}ms`);
|
|
22
|
+
}
|
|
23
|
+
catch (err) {
|
|
24
|
+
console.error(err.message);
|
|
25
|
+
process.exit(1);
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
program
|
|
29
|
+
.command("search <query>")
|
|
30
|
+
.description("Fuzzy search the index and return top chunks")
|
|
31
|
+
.option("-n, --results <count>", "number of results", "3")
|
|
32
|
+
.option("-f, --file <pattern>", "filter results to files matching glob")
|
|
33
|
+
.option("--json", "output as JSON")
|
|
34
|
+
.option("--raw", "output chunk body only, no metadata")
|
|
35
|
+
.action((query, opts) => {
|
|
36
|
+
try {
|
|
37
|
+
const { config, configDir } = loadConfig();
|
|
38
|
+
const indexPath = join(configDir, config.index);
|
|
39
|
+
const { index } = loadPersistedIndex(indexPath, config);
|
|
40
|
+
const maxResults = Math.min(Math.max(1, parseInt(opts.results, 10) || 3), 10);
|
|
41
|
+
const results = search(index, query, {
|
|
42
|
+
maxResults,
|
|
43
|
+
fileFilter: opts.file,
|
|
44
|
+
});
|
|
45
|
+
if (results.length === 0) {
|
|
46
|
+
console.log("No results found.");
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
if (opts.json) {
|
|
50
|
+
console.log(JSON.stringify(results, null, 2));
|
|
51
|
+
}
|
|
52
|
+
else if (opts.raw) {
|
|
53
|
+
for (const r of results) {
|
|
54
|
+
console.log(r.body);
|
|
55
|
+
console.log("");
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
formatResults(results);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
console.error(err.message);
|
|
64
|
+
process.exit(1);
|
|
65
|
+
}
|
|
66
|
+
});
|
|
67
|
+
program
|
|
68
|
+
.command("list")
|
|
69
|
+
.description("List all indexed files and their chunk counts")
|
|
70
|
+
.action(() => {
|
|
71
|
+
try {
|
|
72
|
+
const { config, configDir } = loadConfig();
|
|
73
|
+
const indexPath = join(configDir, config.index);
|
|
74
|
+
const { chunks } = loadPersistedIndex(indexPath, config);
|
|
75
|
+
const byFile = new Map();
|
|
76
|
+
for (const chunk of chunks) {
|
|
77
|
+
byFile.set(chunk.file, (byFile.get(chunk.file) || 0) + 1);
|
|
78
|
+
}
|
|
79
|
+
for (const [file, count] of [...byFile.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
|
|
80
|
+
console.log(`${file} (${count} chunk${count !== 1 ? "s" : ""})`);
|
|
81
|
+
}
|
|
82
|
+
console.log(`\n${byFile.size} files, ${chunks.length} chunks total`);
|
|
83
|
+
}
|
|
84
|
+
catch (err) {
|
|
85
|
+
console.error(err.message);
|
|
86
|
+
process.exit(1);
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
program
|
|
90
|
+
.command("info <file>")
|
|
91
|
+
.description("Show all chunks for a specific file")
|
|
92
|
+
.action((file) => {
|
|
93
|
+
try {
|
|
94
|
+
const { config, configDir } = loadConfig();
|
|
95
|
+
const indexPath = join(configDir, config.index);
|
|
96
|
+
const { chunks } = loadPersistedIndex(indexPath, config);
|
|
97
|
+
const fileChunks = chunks.filter((c) => c.file === file);
|
|
98
|
+
if (fileChunks.length === 0) {
|
|
99
|
+
console.error(`No chunks found for "${file}". Run \`refdocs list\` to see indexed files.`);
|
|
100
|
+
process.exit(1);
|
|
101
|
+
}
|
|
102
|
+
console.log(`${file}: ${fileChunks.length} chunk${fileChunks.length !== 1 ? "s" : ""}\n`);
|
|
103
|
+
for (const chunk of fileChunks) {
|
|
104
|
+
console.log(` [${chunk.startLine}-${chunk.endLine}] ${chunk.headings} (~${chunk.tokenEstimate} tokens)`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
console.error(err.message);
|
|
109
|
+
process.exit(1);
|
|
110
|
+
}
|
|
111
|
+
});
|
|
112
|
+
function formatResults(results) {
|
|
113
|
+
results.forEach((r, i) => {
|
|
114
|
+
console.log(`# [${i + 1}] ${r.file}:${r.lines[0]}-${r.lines[1]}`);
|
|
115
|
+
console.log(`# ${r.headings.join(" > ")}`);
|
|
116
|
+
console.log("");
|
|
117
|
+
console.log(r.body);
|
|
118
|
+
if (i < results.length - 1) {
|
|
119
|
+
console.log("\n---\n");
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
program.parse();
|
|
124
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,SAAS,CAAC;KACf,WAAW,CAAC,kEAAkE,CAAC;KAC/E,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,8CAA8C,CAAC;KAC3D,MAAM,CAAC,GAAG,EAAE;IACX,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,UAAU,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,YAAY,YAAY,OAAO,CAAC,aAAa,SAAS,CAAC,CAAC;QACvF,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC5E,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC;IAChD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAE,GAAa,CAAC,OAAO,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,8CAA8C,CAAC;KAC3D,MAAM,CAAC,uBAAuB,EAAE,mBAAmB,EAAE,GAAG,CAAC;KACzD,MAAM,CAAC,sBAAsB,EAAE,uCAAuC,CAAC;KACvE,MAAM,CAAC,QAAQ,EAAE,gBAAgB,CAAC;KAClC,MAAM,CAAC,OAAO,EAAE,qCAAqC,CAAC;KACtD,MAAM,CAAC,CAAC,KAAa,EAAE,IAAuE,EAAE,EAAE;IACjG,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,UAAU,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAChD,MAAM,EAAE,KAAK,EAAE,GAAG,kBAAkB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAExD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9E,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE;YACnC,UAAU;YACV,UAAU,EAAE,IAAI,CAAC,IAAI;SACtB,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;YACjC,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC;aAAM,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACpB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACxB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACpB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,aAAa,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAE,GAAa,CAAC,OAAO,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,+CAA+C,CAAC;KAC5D,MAAM,CAAC,GAAG,EAAE;IACX,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,UAAU,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAChD,MAAM,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAEzD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5D,CAAC;QAED,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3F,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,KAAK,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,IAAI,WAAW,MAAM,CAAC,MAAM,eAAe,CAAC,CAAC;IACvE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAE,GAAa,CAAC,OAAO,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,aAAa,CAAC;KACtB,WAAW,CAAC,qCAAqC,CAAC;KAClD,MAAM,CAAC,CAAC,IAAY,EAAE,EAAE;IACvB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,UAAU,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAChD,MAAM,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAEzD,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QACzD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,KAAK,CAAC,wBAAwB,IAAI,+CAA+C,CAAC,CAAC;YAC3F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,KAAK,UAAU,CAAC,MAAM,SAAS,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC1F,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,QAAQ,MAAM,KAAK,CAAC,aAAa,UAAU,CAAC,CAAC;QAC5G,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAE,GAAa,CAAC,OAAO,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,SAAS,aAAa,CAAC,OAAuB;IAC5C,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACvB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACpB,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACzB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,OAAO,CAAC,KAAK,EAAE,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Chunk, RefdocsConfig, IndexSummary } from "./types.js";
|
|
2
|
+
import type MiniSearch from "minisearch";
|
|
3
|
+
export declare function findMarkdownFiles(dirs: string[], baseDir: string): string[];
|
|
4
|
+
export declare function buildIndex(config: RefdocsConfig, configDir: string): IndexSummary;
|
|
5
|
+
export declare function loadPersistedIndex(indexPath: string, config: RefdocsConfig): {
|
|
6
|
+
index: MiniSearch<Chunk>;
|
|
7
|
+
chunks: Chunk[];
|
|
8
|
+
};
|
|
9
|
+
//# sourceMappingURL=indexer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/indexer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAGrE,OAAO,KAAK,UAAU,MAAM,YAAY,CAAC;AAEzC,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAsB3E;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,GAAG,YAAY,CA4BjF;AAED,wBAAgB,kBAAkB,CAChC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,aAAa,GACpB;IAAE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IAAC,MAAM,EAAE,KAAK,EAAE,CAAA;CAAE,CAM/C"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { readFileSync, writeFileSync, existsSync, readdirSync, statSync } from "node:fs";
|
|
2
|
+
import { join, relative } from "node:path";
|
|
3
|
+
import { chunkMarkdown } from "./chunker.js";
|
|
4
|
+
import { createSearchIndex, indexChunks, serializeIndex, loadIndex as loadSearchIndex } from "./search.js";
|
|
5
|
+
export function findMarkdownFiles(dirs, baseDir) {
|
|
6
|
+
const files = [];
|
|
7
|
+
function walk(dir) {
|
|
8
|
+
if (!existsSync(dir))
|
|
9
|
+
return;
|
|
10
|
+
const entries = readdirSync(dir);
|
|
11
|
+
for (const entry of entries) {
|
|
12
|
+
const fullPath = join(dir, entry);
|
|
13
|
+
const stat = statSync(fullPath);
|
|
14
|
+
if (stat.isDirectory()) {
|
|
15
|
+
walk(fullPath);
|
|
16
|
+
}
|
|
17
|
+
else if (entry.endsWith(".md")) {
|
|
18
|
+
files.push(relative(baseDir, fullPath));
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
for (const dir of dirs) {
|
|
23
|
+
walk(join(baseDir, dir));
|
|
24
|
+
}
|
|
25
|
+
return files.sort();
|
|
26
|
+
}
|
|
27
|
+
export function buildIndex(config, configDir) {
|
|
28
|
+
const start = Date.now();
|
|
29
|
+
const files = findMarkdownFiles(config.paths, configDir);
|
|
30
|
+
const allChunks = [];
|
|
31
|
+
for (const file of files) {
|
|
32
|
+
const content = readFileSync(join(configDir, file), "utf-8");
|
|
33
|
+
const chunks = chunkMarkdown(content, file, {
|
|
34
|
+
maxTokens: config.chunkMaxTokens,
|
|
35
|
+
minTokens: config.chunkMinTokens,
|
|
36
|
+
});
|
|
37
|
+
allChunks.push(...chunks);
|
|
38
|
+
}
|
|
39
|
+
const index = createSearchIndex(config);
|
|
40
|
+
indexChunks(index, allChunks);
|
|
41
|
+
const serialized = serializeIndex(index, allChunks);
|
|
42
|
+
const indexPath = join(configDir, config.index);
|
|
43
|
+
writeFileSync(indexPath, serialized, "utf-8");
|
|
44
|
+
return {
|
|
45
|
+
filesIndexed: files.length,
|
|
46
|
+
chunksCreated: allChunks.length,
|
|
47
|
+
indexSizeBytes: Buffer.byteLength(serialized, "utf-8"),
|
|
48
|
+
elapsedMs: Date.now() - start,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
export function loadPersistedIndex(indexPath, config) {
|
|
52
|
+
if (!existsSync(indexPath)) {
|
|
53
|
+
throw new Error("Index not found. Run `refdocs index` first.");
|
|
54
|
+
}
|
|
55
|
+
const json = readFileSync(indexPath, "utf-8");
|
|
56
|
+
return loadSearchIndex(json, config);
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=indexer.js.map
|