mdcontext 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.changeset/README.md +28 -0
- package/.changeset/config.json +11 -0
- package/.github/workflows/ci.yml +83 -0
- package/.github/workflows/release.yml +113 -0
- package/.tldrignore +112 -0
- package/AGENTS.md +46 -0
- package/BACKLOG.md +338 -0
- package/README.md +231 -11
- package/biome.json +36 -0
- package/cspell.config.yaml +14 -0
- package/dist/chunk-KRYIFLQR.js +92 -0
- package/dist/chunk-S7E6TFX6.js +742 -0
- package/dist/chunk-VVTGZNBT.js +1519 -0
- package/dist/cli/main.d.ts +1 -0
- package/dist/cli/main.js +2015 -0
- package/dist/index.d.ts +266 -0
- package/dist/index.js +86 -0
- package/dist/mcp/server.d.ts +1 -0
- package/dist/mcp/server.js +376 -0
- package/docs/019-USAGE.md +586 -0
- package/docs/020-current-implementation.md +364 -0
- package/docs/021-DOGFOODING-FINDINGS.md +175 -0
- package/docs/BACKLOG.md +80 -0
- package/docs/DESIGN.md +439 -0
- package/docs/PROJECT.md +88 -0
- package/docs/ROADMAP.md +407 -0
- package/docs/test-links.md +9 -0
- package/package.json +69 -10
- package/pnpm-workspace.yaml +5 -0
- package/research/config-analysis/01-current-implementation.md +470 -0
- package/research/config-analysis/02-strategy-recommendation.md +428 -0
- package/research/config-analysis/03-task-candidates.md +715 -0
- package/research/config-analysis/033-research-configuration-management.md +828 -0
- package/research/config-analysis/034-research-effect-cli-config.md +1504 -0
- package/research/config-analysis/04-consolidated-task-candidates.md +277 -0
- package/research/dogfood/consolidated-tool-evaluation.md +373 -0
- package/research/dogfood/strategy-a/a-synthesis.md +184 -0
- package/research/dogfood/strategy-a/a1-docs.md +226 -0
- package/research/dogfood/strategy-a/a2-amorphic.md +156 -0
- package/research/dogfood/strategy-a/a3-llm.md +164 -0
- package/research/dogfood/strategy-b/b-synthesis.md +228 -0
- package/research/dogfood/strategy-b/b1-architecture.md +207 -0
- package/research/dogfood/strategy-b/b2-gaps.md +258 -0
- package/research/dogfood/strategy-b/b3-workflows.md +250 -0
- package/research/dogfood/strategy-c/c-synthesis.md +451 -0
- package/research/dogfood/strategy-c/c1-explorer.md +192 -0
- package/research/dogfood/strategy-c/c2-diver-memory.md +145 -0
- package/research/dogfood/strategy-c/c3-diver-control.md +148 -0
- package/research/dogfood/strategy-c/c4-diver-failure.md +151 -0
- package/research/dogfood/strategy-c/c5-diver-execution.md +221 -0
- package/research/dogfood/strategy-c/c6-diver-org.md +221 -0
- package/research/effect-cli-error-handling.md +845 -0
- package/research/effect-errors-as-values.md +943 -0
- package/research/errors-task-analysis/00-consolidated-tasks.md +207 -0
- package/research/errors-task-analysis/cli-commands-analysis.md +909 -0
- package/research/errors-task-analysis/embeddings-analysis.md +709 -0
- package/research/errors-task-analysis/index-search-analysis.md +812 -0
- package/research/mdcontext-error-analysis.md +521 -0
- package/research/npm_publish/011-npm-workflow-research-agent2.md +792 -0
- package/research/npm_publish/012-npm-workflow-research-agent1.md +530 -0
- package/research/npm_publish/013-npm-workflow-research-agent3.md +722 -0
- package/research/npm_publish/014-npm-workflow-synthesis.md +556 -0
- package/research/npm_publish/031-npm-workflow-task-analysis.md +134 -0
- package/research/semantic-search/002-research-embedding-models.md +490 -0
- package/research/semantic-search/003-research-rag-alternatives.md +523 -0
- package/research/semantic-search/004-research-vector-search.md +841 -0
- package/research/semantic-search/032-research-semantic-search.md +427 -0
- package/research/task-management-2026/00-synthesis-recommendations.md +295 -0
- package/research/task-management-2026/01-ai-workflow-tools.md +416 -0
- package/research/task-management-2026/02-agent-framework-patterns.md +476 -0
- package/research/task-management-2026/03-lightweight-file-based.md +567 -0
- package/research/task-management-2026/04-established-tools-ai-features.md +541 -0
- package/research/task-management-2026/linear/01-core-features-workflow.md +771 -0
- package/research/task-management-2026/linear/02-api-integrations.md +930 -0
- package/research/task-management-2026/linear/03-ai-features.md +368 -0
- package/research/task-management-2026/linear/04-pricing-setup.md +205 -0
- package/research/task-management-2026/linear/05-usage-patterns-best-practices.md +605 -0
- package/scripts/rebuild-hnswlib.js +63 -0
- package/src/cli/argv-preprocessor.test.ts +210 -0
- package/src/cli/argv-preprocessor.ts +202 -0
- package/src/cli/cli.test.ts +430 -0
- package/src/cli/commands/backlinks.ts +54 -0
- package/src/cli/commands/context.ts +197 -0
- package/src/cli/commands/index-cmd.ts +300 -0
- package/src/cli/commands/index.ts +13 -0
- package/src/cli/commands/links.ts +52 -0
- package/src/cli/commands/search.ts +451 -0
- package/src/cli/commands/stats.ts +146 -0
- package/src/cli/commands/tree.ts +107 -0
- package/src/cli/flag-schemas.ts +275 -0
- package/src/cli/help.ts +386 -0
- package/src/cli/index.ts +9 -0
- package/src/cli/main.ts +145 -0
- package/src/cli/options.ts +31 -0
- package/src/cli/typo-suggester.test.ts +105 -0
- package/src/cli/typo-suggester.ts +130 -0
- package/src/cli/utils.ts +126 -0
- package/src/core/index.ts +1 -0
- package/src/core/types.ts +140 -0
- package/src/embeddings/index.ts +8 -0
- package/src/embeddings/openai-provider.ts +165 -0
- package/src/embeddings/semantic-search.ts +583 -0
- package/src/embeddings/types.ts +82 -0
- package/src/embeddings/vector-store.ts +299 -0
- package/src/index/index.ts +4 -0
- package/src/index/indexer.ts +446 -0
- package/src/index/storage.ts +196 -0
- package/src/index/types.ts +109 -0
- package/src/index/watcher.ts +131 -0
- package/src/index.ts +8 -0
- package/src/mcp/server.ts +483 -0
- package/src/parser/index.ts +1 -0
- package/src/parser/parser.test.ts +291 -0
- package/src/parser/parser.ts +395 -0
- package/src/parser/section-filter.ts +270 -0
- package/src/search/query-parser.test.ts +260 -0
- package/src/search/query-parser.ts +319 -0
- package/src/search/searcher.test.ts +182 -0
- package/src/search/searcher.ts +602 -0
- package/src/summarize/budget-bugs.test.ts +620 -0
- package/src/summarize/formatters.ts +419 -0
- package/src/summarize/index.ts +20 -0
- package/src/summarize/summarizer.test.ts +275 -0
- package/src/summarize/summarizer.ts +528 -0
- package/src/summarize/verify-bugs.test.ts +238 -0
- package/src/utils/index.ts +1 -0
- package/src/utils/tokens.test.ts +142 -0
- package/src/utils/tokens.ts +186 -0
- package/tests/fixtures/cli/.mdcontext/config.json +8 -0
- package/tests/fixtures/cli/.mdcontext/indexes/documents.json +33 -0
- package/tests/fixtures/cli/.mdcontext/indexes/links.json +12 -0
- package/tests/fixtures/cli/.mdcontext/indexes/sections.json +233 -0
- package/tests/fixtures/cli/.mdcontext/vectors.bin +0 -0
- package/tests/fixtures/cli/.mdcontext/vectors.meta.json +1264 -0
- package/tests/fixtures/cli/README.md +9 -0
- package/tests/fixtures/cli/api-reference.md +11 -0
- package/tests/fixtures/cli/getting-started.md +11 -0
- package/tsconfig.json +26 -0
- package/vitest.config.ts +21 -0
- package/vitest.setup.ts +12 -0
package/dist/cli/main.js
ADDED
|
@@ -0,0 +1,2015 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
watchDirectory
|
|
4
|
+
} from "../chunk-KRYIFLQR.js";
|
|
5
|
+
import {
|
|
6
|
+
assembleContext,
|
|
7
|
+
buildEmbeddings,
|
|
8
|
+
estimateEmbeddingCost,
|
|
9
|
+
formatAssembledContext,
|
|
10
|
+
formatSummary,
|
|
11
|
+
getEmbeddingStats,
|
|
12
|
+
handleApiKeyError,
|
|
13
|
+
isAdvancedQuery,
|
|
14
|
+
search,
|
|
15
|
+
searchContent,
|
|
16
|
+
semanticSearch,
|
|
17
|
+
summarizeFile
|
|
18
|
+
} from "../chunk-VVTGZNBT.js";
|
|
19
|
+
import {
|
|
20
|
+
buildIndex,
|
|
21
|
+
createStorage,
|
|
22
|
+
getIncomingLinks,
|
|
23
|
+
getOutgoingLinks,
|
|
24
|
+
loadDocumentIndex,
|
|
25
|
+
loadSectionIndex,
|
|
26
|
+
parseFile
|
|
27
|
+
} from "../chunk-S7E6TFX6.js";
|
|
28
|
+
|
|
29
|
+
// src/cli/main.ts
|
|
30
|
+
import { CliConfig, Command as Command8 } from "@effect/cli";
|
|
31
|
+
import { NodeContext, NodeRuntime } from "@effect/platform-node";
|
|
32
|
+
import { Effect as Effect8, Layer } from "effect";
|
|
33
|
+
|
|
34
|
+
// src/cli/flag-schemas.ts
|
|
35
|
+
var jsonFlag = {
|
|
36
|
+
name: "json",
|
|
37
|
+
type: "boolean",
|
|
38
|
+
description: "Output as JSON"
|
|
39
|
+
};
|
|
40
|
+
var prettyFlag = {
|
|
41
|
+
name: "pretty",
|
|
42
|
+
type: "boolean",
|
|
43
|
+
description: "Pretty-print JSON output"
|
|
44
|
+
};
|
|
45
|
+
var forceFlag = {
|
|
46
|
+
name: "force",
|
|
47
|
+
type: "boolean",
|
|
48
|
+
description: "Force full rebuild"
|
|
49
|
+
};
|
|
50
|
+
var rootFlag = {
|
|
51
|
+
name: "root",
|
|
52
|
+
type: "string",
|
|
53
|
+
alias: "r",
|
|
54
|
+
description: "Root directory"
|
|
55
|
+
};
|
|
56
|
+
var indexSchema = {
|
|
57
|
+
name: "index",
|
|
58
|
+
flags: [
|
|
59
|
+
{
|
|
60
|
+
name: "embed",
|
|
61
|
+
type: "boolean",
|
|
62
|
+
alias: "e",
|
|
63
|
+
description: "Build semantic embeddings"
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
name: "no-embed",
|
|
67
|
+
type: "boolean",
|
|
68
|
+
description: "Skip semantic search prompt"
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
name: "watch",
|
|
72
|
+
type: "boolean",
|
|
73
|
+
alias: "w",
|
|
74
|
+
description: "Watch for changes"
|
|
75
|
+
},
|
|
76
|
+
forceFlag,
|
|
77
|
+
jsonFlag,
|
|
78
|
+
prettyFlag
|
|
79
|
+
]
|
|
80
|
+
};
|
|
81
|
+
var searchSchema = {
|
|
82
|
+
name: "search",
|
|
83
|
+
flags: [
|
|
84
|
+
{
|
|
85
|
+
name: "keyword",
|
|
86
|
+
type: "boolean",
|
|
87
|
+
alias: "k",
|
|
88
|
+
description: "Force keyword search"
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
name: "heading-only",
|
|
92
|
+
type: "boolean",
|
|
93
|
+
alias: "H",
|
|
94
|
+
description: "Search headings only"
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
name: "mode",
|
|
98
|
+
type: "string",
|
|
99
|
+
alias: "m",
|
|
100
|
+
description: "Force search mode (semantic or keyword)"
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
name: "limit",
|
|
104
|
+
type: "string",
|
|
105
|
+
alias: "n",
|
|
106
|
+
description: "Maximum results"
|
|
107
|
+
},
|
|
108
|
+
{ name: "threshold", type: "string", description: "Similarity threshold" },
|
|
109
|
+
{
|
|
110
|
+
name: "context",
|
|
111
|
+
type: "string",
|
|
112
|
+
alias: "C",
|
|
113
|
+
description: "Lines of context around matches (like grep -C)"
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
name: "before-context",
|
|
117
|
+
type: "string",
|
|
118
|
+
alias: "B",
|
|
119
|
+
description: "Lines of context before matches (like grep -B)"
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
name: "after-context",
|
|
123
|
+
type: "string",
|
|
124
|
+
alias: "A",
|
|
125
|
+
description: "Lines of context after matches (like grep -A)"
|
|
126
|
+
},
|
|
127
|
+
jsonFlag,
|
|
128
|
+
prettyFlag
|
|
129
|
+
]
|
|
130
|
+
};
|
|
131
|
+
var contextSchema = {
|
|
132
|
+
name: "context",
|
|
133
|
+
flags: [
|
|
134
|
+
{ name: "tokens", type: "string", alias: "t", description: "Token budget" },
|
|
135
|
+
{ name: "brief", type: "boolean", description: "Minimal output" },
|
|
136
|
+
{ name: "full", type: "boolean", description: "Include full content" },
|
|
137
|
+
{
|
|
138
|
+
name: "section",
|
|
139
|
+
type: "string",
|
|
140
|
+
alias: "S",
|
|
141
|
+
description: "Filter by section name, number, or glob pattern"
|
|
142
|
+
},
|
|
143
|
+
{
|
|
144
|
+
name: "sections",
|
|
145
|
+
type: "boolean",
|
|
146
|
+
description: "List available sections"
|
|
147
|
+
},
|
|
148
|
+
{
|
|
149
|
+
name: "shallow",
|
|
150
|
+
type: "boolean",
|
|
151
|
+
description: "Exclude nested subsections when filtering"
|
|
152
|
+
},
|
|
153
|
+
jsonFlag,
|
|
154
|
+
prettyFlag
|
|
155
|
+
]
|
|
156
|
+
};
|
|
157
|
+
var treeSchema = {
|
|
158
|
+
name: "tree",
|
|
159
|
+
flags: [jsonFlag, prettyFlag]
|
|
160
|
+
};
|
|
161
|
+
var linksSchema = {
|
|
162
|
+
name: "links",
|
|
163
|
+
flags: [rootFlag, jsonFlag, prettyFlag]
|
|
164
|
+
};
|
|
165
|
+
var backlinksSchema = {
|
|
166
|
+
name: "backlinks",
|
|
167
|
+
flags: [rootFlag, jsonFlag, prettyFlag]
|
|
168
|
+
};
|
|
169
|
+
var statsSchema = {
|
|
170
|
+
name: "stats",
|
|
171
|
+
flags: [jsonFlag, prettyFlag]
|
|
172
|
+
};
|
|
173
|
+
var commandSchemas = {
|
|
174
|
+
index: indexSchema,
|
|
175
|
+
search: searchSchema,
|
|
176
|
+
context: contextSchema,
|
|
177
|
+
tree: treeSchema,
|
|
178
|
+
links: linksSchema,
|
|
179
|
+
backlinks: backlinksSchema,
|
|
180
|
+
stats: statsSchema
|
|
181
|
+
};
|
|
182
|
+
var getCommandSchema = (commandName) => {
|
|
183
|
+
return commandSchemas[commandName];
|
|
184
|
+
};
|
|
185
|
+
var getValidFlags = (schema) => {
|
|
186
|
+
const flags = /* @__PURE__ */ new Set();
|
|
187
|
+
for (const spec of schema.flags) {
|
|
188
|
+
flags.add(`--${spec.name}`);
|
|
189
|
+
if (spec.alias) {
|
|
190
|
+
flags.add(`-${spec.alias}`);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
return flags;
|
|
194
|
+
};
|
|
195
|
+
var flagTakesValue = (schema, flag) => {
|
|
196
|
+
if (flag.includes("=")) {
|
|
197
|
+
return false;
|
|
198
|
+
}
|
|
199
|
+
for (const spec of schema.flags) {
|
|
200
|
+
if (flag === `--${spec.name}` || spec.alias && flag === `-${spec.alias}`) {
|
|
201
|
+
return spec.type === "string";
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
return false;
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
// src/cli/typo-suggester.ts
|
|
208
|
+
var levenshteinDistance = (a, b) => {
|
|
209
|
+
const matrix = [];
|
|
210
|
+
for (let i = 0; i <= a.length; i++) {
|
|
211
|
+
matrix[i] = [i];
|
|
212
|
+
}
|
|
213
|
+
for (let j = 0; j <= b.length; j++) {
|
|
214
|
+
matrix[0][j] = j;
|
|
215
|
+
}
|
|
216
|
+
for (let i = 1; i <= a.length; i++) {
|
|
217
|
+
for (let j = 1; j <= b.length; j++) {
|
|
218
|
+
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
|
219
|
+
matrix[i][j] = Math.min(
|
|
220
|
+
matrix[i - 1][j] + 1,
|
|
221
|
+
// deletion
|
|
222
|
+
matrix[i][j - 1] + 1,
|
|
223
|
+
// insertion
|
|
224
|
+
matrix[i - 1][j - 1] + cost
|
|
225
|
+
// substitution
|
|
226
|
+
);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
return matrix[a.length][b.length];
|
|
230
|
+
};
|
|
231
|
+
var suggestFlag = (typo, schema, maxDistance = 2) => {
|
|
232
|
+
const normalizedTypo = typo.replace(/^-+/, "");
|
|
233
|
+
let bestMatch;
|
|
234
|
+
let bestDistance = Infinity;
|
|
235
|
+
for (const spec of schema.flags) {
|
|
236
|
+
const flagName = spec.name;
|
|
237
|
+
const distance = levenshteinDistance(normalizedTypo, flagName);
|
|
238
|
+
if (distance <= maxDistance && distance < bestDistance) {
|
|
239
|
+
bestDistance = distance;
|
|
240
|
+
bestMatch = {
|
|
241
|
+
flag: `--${spec.name}`,
|
|
242
|
+
distance,
|
|
243
|
+
description: spec.description
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
if (spec.alias) {
|
|
247
|
+
const aliasDistance = levenshteinDistance(normalizedTypo, spec.alias);
|
|
248
|
+
if (aliasDistance <= maxDistance && aliasDistance < bestDistance) {
|
|
249
|
+
bestDistance = aliasDistance;
|
|
250
|
+
bestMatch = {
|
|
251
|
+
flag: `--${spec.name}`,
|
|
252
|
+
distance: aliasDistance,
|
|
253
|
+
description: spec.description
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
if (!bestMatch || bestDistance > 0) {
|
|
259
|
+
for (const spec of schema.flags) {
|
|
260
|
+
if (spec.name.startsWith(normalizedTypo)) {
|
|
261
|
+
const prefixDistance = spec.name.length - normalizedTypo.length;
|
|
262
|
+
if (prefixDistance <= maxDistance && prefixDistance < bestDistance) {
|
|
263
|
+
bestDistance = prefixDistance;
|
|
264
|
+
bestMatch = {
|
|
265
|
+
flag: `--${spec.name}`,
|
|
266
|
+
distance: prefixDistance,
|
|
267
|
+
description: spec.description
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
return bestMatch;
|
|
274
|
+
};
|
|
275
|
+
var formatValidFlags = (schema) => {
|
|
276
|
+
const lines = [];
|
|
277
|
+
for (const spec of schema.flags) {
|
|
278
|
+
const alias = spec.alias ? `, -${spec.alias}` : "";
|
|
279
|
+
const desc = spec.description ? ` ${spec.description}` : "";
|
|
280
|
+
lines.push(` --${spec.name}${alias}${desc}`);
|
|
281
|
+
}
|
|
282
|
+
return lines.join("\n");
|
|
283
|
+
};
|
|
284
|
+
|
|
285
|
+
// src/cli/argv-preprocessor.ts
|
|
286
|
+
var isFlag = (arg) => {
|
|
287
|
+
return arg.startsWith("-");
|
|
288
|
+
};
|
|
289
|
+
var extractFlagName = (arg) => {
|
|
290
|
+
const eqIndex = arg.indexOf("=");
|
|
291
|
+
return eqIndex >= 0 ? arg.slice(0, eqIndex) : arg;
|
|
292
|
+
};
|
|
293
|
+
var validateFlag = (flag, schema) => {
|
|
294
|
+
const flagName = extractFlagName(flag);
|
|
295
|
+
const validFlags = getValidFlags(schema);
|
|
296
|
+
if (validFlags.has(flagName)) {
|
|
297
|
+
return void 0;
|
|
298
|
+
}
|
|
299
|
+
const suggestion = suggestFlag(flagName, schema);
|
|
300
|
+
let errorMsg = `Unknown option '${flagName}' for '${schema.name}'`;
|
|
301
|
+
if (suggestion) {
|
|
302
|
+
errorMsg += `
|
|
303
|
+
Did you mean '${suggestion.flag}'?`;
|
|
304
|
+
}
|
|
305
|
+
errorMsg += `
|
|
306
|
+
|
|
307
|
+
Valid options for '${schema.name}':
|
|
308
|
+
${formatValidFlags(schema)}`;
|
|
309
|
+
return errorMsg;
|
|
310
|
+
};
|
|
311
|
+
var preprocessArgv = (argv) => {
|
|
312
|
+
const result = preprocessArgvWithValidation(argv);
|
|
313
|
+
if (result.error) {
|
|
314
|
+
console.error(`
|
|
315
|
+
Error: ${result.error}`);
|
|
316
|
+
console.error('\nRun "mdcontext <command> --help" for usage information.');
|
|
317
|
+
process.exit(1);
|
|
318
|
+
}
|
|
319
|
+
return result.argv;
|
|
320
|
+
};
|
|
321
|
+
var preprocessArgvWithValidation = (argv) => {
|
|
322
|
+
const nodeAndScript = argv.slice(0, 2);
|
|
323
|
+
const userArgs = argv.slice(2);
|
|
324
|
+
if (userArgs.length === 0) {
|
|
325
|
+
return { argv };
|
|
326
|
+
}
|
|
327
|
+
const firstArg = userArgs[0];
|
|
328
|
+
if (!firstArg || isFlag(firstArg)) {
|
|
329
|
+
return { argv };
|
|
330
|
+
}
|
|
331
|
+
const subcommand = firstArg;
|
|
332
|
+
const restArgs = userArgs.slice(1);
|
|
333
|
+
const schema = getCommandSchema(subcommand);
|
|
334
|
+
if (!schema) {
|
|
335
|
+
return { argv };
|
|
336
|
+
}
|
|
337
|
+
const flags = [];
|
|
338
|
+
const positionals = [];
|
|
339
|
+
let i = 0;
|
|
340
|
+
while (i < restArgs.length) {
|
|
341
|
+
const arg = restArgs[i];
|
|
342
|
+
if (!arg) {
|
|
343
|
+
i++;
|
|
344
|
+
continue;
|
|
345
|
+
}
|
|
346
|
+
if (isFlag(arg)) {
|
|
347
|
+
if (arg === "--help" || arg === "-h") {
|
|
348
|
+
flags.push(arg);
|
|
349
|
+
i++;
|
|
350
|
+
continue;
|
|
351
|
+
}
|
|
352
|
+
if (arg === "--") {
|
|
353
|
+
i++;
|
|
354
|
+
while (i < restArgs.length) {
|
|
355
|
+
const remaining = restArgs[i];
|
|
356
|
+
if (remaining) positionals.push(remaining);
|
|
357
|
+
i++;
|
|
358
|
+
}
|
|
359
|
+
continue;
|
|
360
|
+
}
|
|
361
|
+
const error = validateFlag(arg, schema);
|
|
362
|
+
if (error) {
|
|
363
|
+
return { argv, error };
|
|
364
|
+
}
|
|
365
|
+
if (arg.includes("=")) {
|
|
366
|
+
flags.push(arg);
|
|
367
|
+
i++;
|
|
368
|
+
} else if (flagTakesValue(schema, arg)) {
|
|
369
|
+
flags.push(arg);
|
|
370
|
+
i++;
|
|
371
|
+
if (i < restArgs.length) {
|
|
372
|
+
const nextArg = restArgs[i];
|
|
373
|
+
if (nextArg && !isFlag(nextArg)) {
|
|
374
|
+
flags.push(nextArg);
|
|
375
|
+
i++;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
} else {
|
|
379
|
+
flags.push(arg);
|
|
380
|
+
i++;
|
|
381
|
+
}
|
|
382
|
+
} else {
|
|
383
|
+
positionals.push(arg);
|
|
384
|
+
i++;
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
return {
|
|
388
|
+
argv: [...nodeAndScript, subcommand, ...flags, ...positionals]
|
|
389
|
+
};
|
|
390
|
+
};
|
|
391
|
+
|
|
392
|
+
// src/cli/commands/backlinks.ts
|
|
393
|
+
import * as path2 from "path";
|
|
394
|
+
import { Args, Command, Options as Options2 } from "@effect/cli";
|
|
395
|
+
import { Console, Effect } from "effect";
|
|
396
|
+
|
|
397
|
+
// src/cli/options.ts
|
|
398
|
+
import { Options } from "@effect/cli";
|
|
399
|
+
var jsonOption = Options.boolean("json").pipe(
|
|
400
|
+
Options.withDescription("Output as JSON"),
|
|
401
|
+
Options.withDefault(false)
|
|
402
|
+
);
|
|
403
|
+
var prettyOption = Options.boolean("pretty").pipe(
|
|
404
|
+
Options.withDescription("Pretty-print JSON output"),
|
|
405
|
+
Options.withDefault(true)
|
|
406
|
+
);
|
|
407
|
+
var forceOption = Options.boolean("force").pipe(
|
|
408
|
+
Options.withDescription("Force full rebuild, ignoring cache"),
|
|
409
|
+
Options.withDefault(false)
|
|
410
|
+
);
|
|
411
|
+
|
|
412
|
+
// src/cli/utils.ts
|
|
413
|
+
import * as fsPromises from "fs/promises";
|
|
414
|
+
import * as path from "path";
|
|
415
|
+
var formatJson = (obj, pretty) => {
|
|
416
|
+
return pretty ? JSON.stringify(obj, null, 2) : JSON.stringify(obj);
|
|
417
|
+
};
|
|
418
|
+
var isMarkdownFile = (filename) => {
|
|
419
|
+
return filename.endsWith(".md") || filename.endsWith(".mdx");
|
|
420
|
+
};
|
|
421
|
+
var walkDir = async (dir) => {
|
|
422
|
+
const files = [];
|
|
423
|
+
const entries = await fsPromises.readdir(dir, { withFileTypes: true });
|
|
424
|
+
for (const entry of entries) {
|
|
425
|
+
const fullPath = path.join(dir, entry.name);
|
|
426
|
+
if (entry.name.startsWith(".") || entry.name === "node_modules") {
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
429
|
+
if (entry.isDirectory()) {
|
|
430
|
+
const subFiles = await walkDir(fullPath);
|
|
431
|
+
files.push(...subFiles);
|
|
432
|
+
} else if (entry.isFile() && isMarkdownFile(entry.name)) {
|
|
433
|
+
files.push(fullPath);
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
return files;
|
|
437
|
+
};
|
|
438
|
+
var isRegexPattern = (query) => {
|
|
439
|
+
return /[.*+?^${}()|[\]\\]/.test(query);
|
|
440
|
+
};
|
|
441
|
+
var hasEmbeddings = async (dir) => {
|
|
442
|
+
const vectorsPath = path.join(dir, ".mdcontext", "vectors.bin");
|
|
443
|
+
try {
|
|
444
|
+
await fsPromises.access(vectorsPath);
|
|
445
|
+
return true;
|
|
446
|
+
} catch {
|
|
447
|
+
return false;
|
|
448
|
+
}
|
|
449
|
+
};
|
|
450
|
+
var getIndexInfo = async (dir) => {
|
|
451
|
+
const sectionsPath = path.join(dir, ".mdcontext", "indexes", "sections.json");
|
|
452
|
+
const vectorsMetaPath = path.join(dir, ".mdcontext", "vectors.meta.json");
|
|
453
|
+
let exists = false;
|
|
454
|
+
let lastUpdated;
|
|
455
|
+
let sectionCount;
|
|
456
|
+
let embeddingsExist = false;
|
|
457
|
+
let vectorCount;
|
|
458
|
+
try {
|
|
459
|
+
const stat2 = await fsPromises.stat(sectionsPath);
|
|
460
|
+
exists = true;
|
|
461
|
+
lastUpdated = stat2.mtime.toISOString();
|
|
462
|
+
const content = await fsPromises.readFile(sectionsPath, "utf-8");
|
|
463
|
+
const sections = JSON.parse(content);
|
|
464
|
+
sectionCount = Object.keys(sections.sections || {}).length;
|
|
465
|
+
} catch {
|
|
466
|
+
}
|
|
467
|
+
try {
|
|
468
|
+
const content = await fsPromises.readFile(vectorsMetaPath, "utf-8");
|
|
469
|
+
const meta = JSON.parse(content);
|
|
470
|
+
embeddingsExist = true;
|
|
471
|
+
vectorCount = Object.keys(meta.entries || {}).length;
|
|
472
|
+
if (meta.updatedAt) {
|
|
473
|
+
lastUpdated = meta.updatedAt;
|
|
474
|
+
}
|
|
475
|
+
} catch {
|
|
476
|
+
}
|
|
477
|
+
return {
|
|
478
|
+
exists,
|
|
479
|
+
lastUpdated,
|
|
480
|
+
sectionCount,
|
|
481
|
+
embeddingsExist,
|
|
482
|
+
vectorCount
|
|
483
|
+
};
|
|
484
|
+
};
|
|
485
|
+
|
|
486
|
+
// src/cli/commands/backlinks.ts
|
|
487
|
+
var backlinksCommand = Command.make(
|
|
488
|
+
"backlinks",
|
|
489
|
+
{
|
|
490
|
+
file: Args.file({ name: "file" }).pipe(
|
|
491
|
+
Args.withDescription("Markdown file to find references to")
|
|
492
|
+
),
|
|
493
|
+
root: Options2.directory("root").pipe(
|
|
494
|
+
Options2.withAlias("r"),
|
|
495
|
+
Options2.withDescription("Root directory for resolving relative links"),
|
|
496
|
+
Options2.withDefault(".")
|
|
497
|
+
),
|
|
498
|
+
json: jsonOption,
|
|
499
|
+
pretty: prettyOption
|
|
500
|
+
},
|
|
501
|
+
({ file, root, json, pretty }) => Effect.gen(function* () {
|
|
502
|
+
const resolvedRoot = path2.resolve(root);
|
|
503
|
+
const resolvedFile = path2.resolve(file);
|
|
504
|
+
const relativePath = path2.relative(resolvedRoot, resolvedFile);
|
|
505
|
+
const links = yield* getIncomingLinks(resolvedRoot, resolvedFile);
|
|
506
|
+
if (json) {
|
|
507
|
+
yield* Console.log(
|
|
508
|
+
formatJson({ file: relativePath, backlinks: links }, pretty)
|
|
509
|
+
);
|
|
510
|
+
} else {
|
|
511
|
+
yield* Console.log(`Incoming links to ${relativePath}:`);
|
|
512
|
+
yield* Console.log("");
|
|
513
|
+
if (links.length === 0) {
|
|
514
|
+
yield* Console.log(" (none)");
|
|
515
|
+
} else {
|
|
516
|
+
for (const link of links) {
|
|
517
|
+
yield* Console.log(` <- ${link}`);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
yield* Console.log("");
|
|
521
|
+
yield* Console.log(`Total: ${links.length} backlinks`);
|
|
522
|
+
}
|
|
523
|
+
})
|
|
524
|
+
).pipe(Command.withDescription("What links to this?"));
|
|
525
|
+
|
|
526
|
+
// src/cli/commands/context.ts
|
|
527
|
+
import * as path3 from "path";
|
|
528
|
+
import { Args as Args2, Command as Command2, Options as Options3 } from "@effect/cli";
|
|
529
|
+
import { Console as Console2, Effect as Effect2 } from "effect";
|
|
530
|
+
|
|
531
|
+
// src/parser/section-filter.ts
|
|
532
|
+
var globMatch = (text, pattern) => {
|
|
533
|
+
const regexPattern = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".");
|
|
534
|
+
const regex = new RegExp(`^${regexPattern}$`, "i");
|
|
535
|
+
return regex.test(text);
|
|
536
|
+
};
|
|
537
|
+
var buildSectionList = (document) => {
|
|
538
|
+
const result = [];
|
|
539
|
+
const processSection = (section, prefix, index) => {
|
|
540
|
+
const number = prefix ? `${prefix}.${index + 1}` : `${index + 1}`;
|
|
541
|
+
result.push({
|
|
542
|
+
number,
|
|
543
|
+
heading: section.heading,
|
|
544
|
+
level: section.level,
|
|
545
|
+
tokenCount: section.metadata.tokenCount
|
|
546
|
+
});
|
|
547
|
+
section.children.forEach((child, i) => {
|
|
548
|
+
processSection(child, number, i);
|
|
549
|
+
});
|
|
550
|
+
};
|
|
551
|
+
document.sections.forEach((section, i) => {
|
|
552
|
+
processSection(section, "", i);
|
|
553
|
+
});
|
|
554
|
+
return result;
|
|
555
|
+
};
|
|
556
|
+
var formatSectionList = (sections) => {
|
|
557
|
+
const lines = [];
|
|
558
|
+
for (const section of sections) {
|
|
559
|
+
const depth = (section.number.match(/\./g) || []).length;
|
|
560
|
+
const indent = " ".repeat(depth);
|
|
561
|
+
lines.push(
|
|
562
|
+
`${indent}${section.number}. ${section.heading} (${section.tokenCount} tokens)`
|
|
563
|
+
);
|
|
564
|
+
}
|
|
565
|
+
return lines.join("\n");
|
|
566
|
+
};
|
|
567
|
+
var matchesSelector = (section, selector) => {
|
|
568
|
+
if (/^[\d.]+$/.test(selector)) {
|
|
569
|
+
return section.number === selector;
|
|
570
|
+
}
|
|
571
|
+
if (section.heading.toLowerCase() === selector.toLowerCase()) {
|
|
572
|
+
return true;
|
|
573
|
+
}
|
|
574
|
+
if (selector.includes("*") || selector.includes("?")) {
|
|
575
|
+
return globMatch(section.heading, selector);
|
|
576
|
+
}
|
|
577
|
+
return section.heading.toLowerCase().includes(selector.toLowerCase());
|
|
578
|
+
};
|
|
579
|
+
var findMatchingSections = (sectionList, selector) => {
|
|
580
|
+
return sectionList.filter((s) => matchesSelector(s, selector));
|
|
581
|
+
};
|
|
582
|
+
var getDescendantNumbers = (sectionList, parentNumber) => {
|
|
583
|
+
const result = /* @__PURE__ */ new Set();
|
|
584
|
+
const prefix = `${parentNumber}.`;
|
|
585
|
+
for (const section of sectionList) {
|
|
586
|
+
if (section.number.startsWith(prefix)) {
|
|
587
|
+
result.add(section.number);
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
return result;
|
|
591
|
+
};
|
|
592
|
+
var extractSectionContent = (document, selector, options = {}) => {
|
|
593
|
+
const sectionList = buildSectionList(document);
|
|
594
|
+
const matchedSections = findMatchingSections(sectionList, selector);
|
|
595
|
+
if (matchedSections.length === 0) {
|
|
596
|
+
return { sections: [], matchedNumbers: [] };
|
|
597
|
+
}
|
|
598
|
+
const numbersToInclude = /* @__PURE__ */ new Set();
|
|
599
|
+
const matchedNumbers = [];
|
|
600
|
+
for (const matched of matchedSections) {
|
|
601
|
+
numbersToInclude.add(matched.number);
|
|
602
|
+
matchedNumbers.push(matched.number);
|
|
603
|
+
if (!options.shallow) {
|
|
604
|
+
const descendants = getDescendantNumbers(sectionList, matched.number);
|
|
605
|
+
for (const desc of descendants) {
|
|
606
|
+
numbersToInclude.add(desc);
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
const numberToSection = /* @__PURE__ */ new Map();
|
|
611
|
+
const mapSections = (sections, prefix) => {
|
|
612
|
+
sections.forEach((section, i) => {
|
|
613
|
+
const number = prefix ? `${prefix}.${i + 1}` : `${i + 1}`;
|
|
614
|
+
numberToSection.set(number, section);
|
|
615
|
+
mapSections(section.children, number);
|
|
616
|
+
});
|
|
617
|
+
};
|
|
618
|
+
mapSections(document.sections, "");
|
|
619
|
+
const extractedSections = [];
|
|
620
|
+
for (const number of matchedNumbers) {
|
|
621
|
+
const section = numberToSection.get(number);
|
|
622
|
+
if (section) {
|
|
623
|
+
if (options.shallow) {
|
|
624
|
+
extractedSections.push({
|
|
625
|
+
...section,
|
|
626
|
+
children: []
|
|
627
|
+
});
|
|
628
|
+
} else {
|
|
629
|
+
extractedSections.push(section);
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
return { sections: extractedSections, matchedNumbers };
|
|
634
|
+
};
|
|
635
|
+
var formatExtractedSections = (sections) => {
|
|
636
|
+
const formatSection = (section, includeChildren) => {
|
|
637
|
+
const lines = [];
|
|
638
|
+
const headingPrefix = "#".repeat(section.level);
|
|
639
|
+
lines.push(`${headingPrefix} ${section.heading}`);
|
|
640
|
+
lines.push("");
|
|
641
|
+
const contentLines = section.content.split("\n");
|
|
642
|
+
const contentWithoutHeading = contentLines.filter((line, i) => i > 0 || !line.startsWith("#")).join("\n").trim();
|
|
643
|
+
if (contentWithoutHeading) {
|
|
644
|
+
lines.push(contentWithoutHeading);
|
|
645
|
+
}
|
|
646
|
+
if (includeChildren) {
|
|
647
|
+
for (const child of section.children) {
|
|
648
|
+
lines.push("");
|
|
649
|
+
lines.push(formatSection(child, true));
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
return lines.join("\n");
|
|
653
|
+
};
|
|
654
|
+
return sections.map((s) => formatSection(s, true)).join("\n\n");
|
|
655
|
+
};
|
|
656
|
+
|
|
657
|
+
// src/cli/commands/context.ts
|
|
658
|
+
var contextCommand = Command2.make(
|
|
659
|
+
"context",
|
|
660
|
+
{
|
|
661
|
+
files: Args2.file({ name: "files" }).pipe(
|
|
662
|
+
Args2.withDescription("Markdown file(s) to summarize"),
|
|
663
|
+
Args2.repeated
|
|
664
|
+
),
|
|
665
|
+
tokens: Options3.integer("tokens").pipe(
|
|
666
|
+
Options3.withAlias("t"),
|
|
667
|
+
Options3.withDescription("Token budget"),
|
|
668
|
+
Options3.withDefault(2e3)
|
|
669
|
+
),
|
|
670
|
+
brief: Options3.boolean("brief").pipe(
|
|
671
|
+
Options3.withDescription("Minimal output"),
|
|
672
|
+
Options3.withDefault(false)
|
|
673
|
+
),
|
|
674
|
+
full: Options3.boolean("full").pipe(
|
|
675
|
+
Options3.withDescription("Include full content"),
|
|
676
|
+
Options3.withDefault(false)
|
|
677
|
+
),
|
|
678
|
+
section: Options3.text("section").pipe(
|
|
679
|
+
Options3.withAlias("S"),
|
|
680
|
+
Options3.withDescription(
|
|
681
|
+
'Filter by section name, number (e.g., "5.3"), or glob pattern (e.g., "Memory*")'
|
|
682
|
+
),
|
|
683
|
+
Options3.optional
|
|
684
|
+
),
|
|
685
|
+
sections: Options3.boolean("sections").pipe(
|
|
686
|
+
Options3.withDescription("List available sections"),
|
|
687
|
+
Options3.withDefault(false)
|
|
688
|
+
),
|
|
689
|
+
shallow: Options3.boolean("shallow").pipe(
|
|
690
|
+
Options3.withDescription("Exclude nested subsections when filtering"),
|
|
691
|
+
Options3.withDefault(false)
|
|
692
|
+
),
|
|
693
|
+
json: jsonOption,
|
|
694
|
+
pretty: prettyOption
|
|
695
|
+
},
|
|
696
|
+
({ files, tokens, brief, full, section, sections, shallow, json, pretty }) => Effect2.gen(function* () {
|
|
697
|
+
const fileList = Array.isArray(files) ? files : [];
|
|
698
|
+
if (fileList.length === 0) {
|
|
699
|
+
yield* Effect2.fail(
|
|
700
|
+
new Error(
|
|
701
|
+
"At least one file is required. Usage: mdcontext context <file> [files...]"
|
|
702
|
+
)
|
|
703
|
+
);
|
|
704
|
+
}
|
|
705
|
+
const sectionSelector = section._tag === "Some" ? section.value : void 0;
|
|
706
|
+
if (sections) {
|
|
707
|
+
for (const file of fileList) {
|
|
708
|
+
const filePath = path3.resolve(file);
|
|
709
|
+
const document = yield* parseFile(filePath).pipe(
|
|
710
|
+
Effect2.mapError((e) => new Error(`${e._tag}: ${e.message}`))
|
|
711
|
+
);
|
|
712
|
+
const sectionList = buildSectionList(document);
|
|
713
|
+
if (json) {
|
|
714
|
+
const output = {
|
|
715
|
+
path: filePath,
|
|
716
|
+
title: document.title,
|
|
717
|
+
sections: sectionList.map((s) => ({
|
|
718
|
+
number: s.number,
|
|
719
|
+
heading: s.heading,
|
|
720
|
+
level: s.level,
|
|
721
|
+
tokens: s.tokenCount
|
|
722
|
+
}))
|
|
723
|
+
};
|
|
724
|
+
yield* Console2.log(formatJson(output, pretty));
|
|
725
|
+
} else {
|
|
726
|
+
yield* Console2.log(`# ${document.title}`);
|
|
727
|
+
yield* Console2.log(`Path: ${filePath}`);
|
|
728
|
+
yield* Console2.log("");
|
|
729
|
+
yield* Console2.log("Available sections:");
|
|
730
|
+
yield* Console2.log(formatSectionList(sectionList));
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
return;
|
|
734
|
+
}
|
|
735
|
+
if (sectionSelector) {
|
|
736
|
+
for (const file of fileList) {
|
|
737
|
+
const filePath = path3.resolve(file);
|
|
738
|
+
const document = yield* parseFile(filePath).pipe(
|
|
739
|
+
Effect2.mapError((e) => new Error(`${e._tag}: ${e.message}`))
|
|
740
|
+
);
|
|
741
|
+
const { sections: extractedSections, matchedNumbers } = extractSectionContent(document, sectionSelector, { shallow });
|
|
742
|
+
if (extractedSections.length === 0) {
|
|
743
|
+
yield* Console2.error(
|
|
744
|
+
`No sections found matching "${sectionSelector}" in ${file}`
|
|
745
|
+
);
|
|
746
|
+
yield* Console2.error("Use --sections to list available sections.");
|
|
747
|
+
continue;
|
|
748
|
+
}
|
|
749
|
+
if (json) {
|
|
750
|
+
const output = {
|
|
751
|
+
path: filePath,
|
|
752
|
+
title: document.title,
|
|
753
|
+
selector: sectionSelector,
|
|
754
|
+
shallow,
|
|
755
|
+
matchedSections: matchedNumbers,
|
|
756
|
+
content: formatExtractedSections(extractedSections),
|
|
757
|
+
sections: extractedSections.map((s) => ({
|
|
758
|
+
heading: s.heading,
|
|
759
|
+
level: s.level,
|
|
760
|
+
tokens: s.metadata.tokenCount
|
|
761
|
+
}))
|
|
762
|
+
};
|
|
763
|
+
yield* Console2.log(formatJson(output, pretty));
|
|
764
|
+
} else {
|
|
765
|
+
yield* Console2.log(`# ${document.title}`);
|
|
766
|
+
yield* Console2.log(`Path: ${filePath}`);
|
|
767
|
+
yield* Console2.log(`Sections: ${matchedNumbers.join(", ")}`);
|
|
768
|
+
yield* Console2.log("");
|
|
769
|
+
yield* Console2.log(formatExtractedSections(extractedSections));
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
return;
|
|
773
|
+
}
|
|
774
|
+
const level = full ? "full" : brief ? "brief" : "summary";
|
|
775
|
+
const effectiveMaxTokens = full ? void 0 : tokens;
|
|
776
|
+
const firstFile = fileList[0];
|
|
777
|
+
if (fileList.length === 1 && firstFile) {
|
|
778
|
+
const filePath = path3.resolve(firstFile);
|
|
779
|
+
const summary = yield* summarizeFile(filePath, {
|
|
780
|
+
level,
|
|
781
|
+
maxTokens: effectiveMaxTokens
|
|
782
|
+
});
|
|
783
|
+
if (json) {
|
|
784
|
+
yield* Console2.log(formatJson(summary, pretty));
|
|
785
|
+
} else {
|
|
786
|
+
yield* Console2.log(
|
|
787
|
+
formatSummary(summary, { maxTokens: effectiveMaxTokens })
|
|
788
|
+
);
|
|
789
|
+
}
|
|
790
|
+
} else {
|
|
791
|
+
const root = process.cwd();
|
|
792
|
+
const assembled = yield* assembleContext(root, fileList, {
|
|
793
|
+
budget: full ? Number.MAX_SAFE_INTEGER : tokens,
|
|
794
|
+
level
|
|
795
|
+
});
|
|
796
|
+
if (json) {
|
|
797
|
+
yield* Console2.log(formatJson(assembled, pretty));
|
|
798
|
+
} else {
|
|
799
|
+
yield* Console2.log(formatAssembledContext(assembled));
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
})
|
|
803
|
+
).pipe(Command2.withDescription("Get LLM-ready summary"));
|
|
804
|
+
|
|
805
|
+
// src/cli/commands/index-cmd.ts
|
|
806
|
+
import * as path4 from "path";
|
|
807
|
+
import * as readline from "readline";
|
|
808
|
+
import { Args as Args3, Command as Command3, Options as Options4 } from "@effect/cli";
|
|
809
|
+
import { Console as Console3, Effect as Effect3 } from "effect";
|
|
810
|
+
var promptUser = (message) => {
|
|
811
|
+
return new Promise((resolve8) => {
|
|
812
|
+
const rl = readline.createInterface({
|
|
813
|
+
input: process.stdin,
|
|
814
|
+
output: process.stdout
|
|
815
|
+
});
|
|
816
|
+
rl.question(message, (answer) => {
|
|
817
|
+
rl.close();
|
|
818
|
+
resolve8(answer.trim().toLowerCase());
|
|
819
|
+
});
|
|
820
|
+
});
|
|
821
|
+
};
|
|
822
|
+
var indexCommand = Command3.make(
|
|
823
|
+
"index",
|
|
824
|
+
{
|
|
825
|
+
path: Args3.directory({ name: "path" }).pipe(
|
|
826
|
+
Args3.withDescription("Directory to index"),
|
|
827
|
+
Args3.withDefault(".")
|
|
828
|
+
),
|
|
829
|
+
embed: Options4.boolean("embed").pipe(
|
|
830
|
+
Options4.withAlias("e"),
|
|
831
|
+
Options4.withDescription("Also build semantic embeddings"),
|
|
832
|
+
Options4.withDefault(false)
|
|
833
|
+
),
|
|
834
|
+
noEmbed: Options4.boolean("no-embed").pipe(
|
|
835
|
+
Options4.withDescription("Skip semantic search prompt"),
|
|
836
|
+
Options4.withDefault(false)
|
|
837
|
+
),
|
|
838
|
+
exclude: Options4.text("exclude").pipe(
|
|
839
|
+
Options4.withAlias("x"),
|
|
840
|
+
Options4.withDescription(
|
|
841
|
+
"Exclude files matching patterns (comma-separated globs)"
|
|
842
|
+
),
|
|
843
|
+
Options4.optional
|
|
844
|
+
),
|
|
845
|
+
watch: Options4.boolean("watch").pipe(
|
|
846
|
+
Options4.withAlias("w"),
|
|
847
|
+
Options4.withDescription("Watch for changes"),
|
|
848
|
+
Options4.withDefault(false)
|
|
849
|
+
),
|
|
850
|
+
force: forceOption,
|
|
851
|
+
json: jsonOption,
|
|
852
|
+
pretty: prettyOption
|
|
853
|
+
},
|
|
854
|
+
({
|
|
855
|
+
path: dirPath,
|
|
856
|
+
embed,
|
|
857
|
+
noEmbed,
|
|
858
|
+
exclude,
|
|
859
|
+
watch: watchMode,
|
|
860
|
+
force,
|
|
861
|
+
json,
|
|
862
|
+
pretty
|
|
863
|
+
}) => Effect3.gen(function* () {
|
|
864
|
+
const resolvedDir = path4.resolve(dirPath);
|
|
865
|
+
const excludePatterns = exclude._tag === "Some" ? exclude.value.split(",").map((p) => p.trim()) : void 0;
|
|
866
|
+
if (watchMode) {
|
|
867
|
+
yield* Console3.log(`Watching ${resolvedDir} for changes...`);
|
|
868
|
+
yield* Console3.log("Press Ctrl+C to stop.");
|
|
869
|
+
yield* Console3.log("");
|
|
870
|
+
const watcher = yield* watchDirectory(resolvedDir, {
|
|
871
|
+
force,
|
|
872
|
+
onIndex: (result) => {
|
|
873
|
+
if (json) {
|
|
874
|
+
console.log(formatJson(result, pretty));
|
|
875
|
+
} else {
|
|
876
|
+
console.log(
|
|
877
|
+
`Re-indexed ${result.documentsIndexed} documents (${result.duration}ms)`
|
|
878
|
+
);
|
|
879
|
+
}
|
|
880
|
+
},
|
|
881
|
+
onError: (error) => {
|
|
882
|
+
console.error(`Watch error: ${error.message}`);
|
|
883
|
+
}
|
|
884
|
+
});
|
|
885
|
+
yield* Effect3.async(() => {
|
|
886
|
+
process.on("SIGINT", () => {
|
|
887
|
+
watcher.stop();
|
|
888
|
+
console.log("\nStopped watching.");
|
|
889
|
+
process.exit(0);
|
|
890
|
+
});
|
|
891
|
+
});
|
|
892
|
+
} else {
|
|
893
|
+
yield* Console3.log(`Indexing ${resolvedDir}...`);
|
|
894
|
+
const result = yield* buildIndex(resolvedDir, { force });
|
|
895
|
+
if (!json) {
|
|
896
|
+
yield* Console3.log("");
|
|
897
|
+
const newlyIndexed = result.documentsIndexed < result.totalDocuments ? ` (${result.documentsIndexed} updated)` : "";
|
|
898
|
+
yield* Console3.log(
|
|
899
|
+
`Indexed ${result.totalDocuments} documents${newlyIndexed}`
|
|
900
|
+
);
|
|
901
|
+
yield* Console3.log(` Sections: ${result.totalSections}`);
|
|
902
|
+
yield* Console3.log(` Links: ${result.totalLinks}`);
|
|
903
|
+
yield* Console3.log(` Duration: ${result.duration}ms`);
|
|
904
|
+
if (result.errors.length > 0) {
|
|
905
|
+
yield* Console3.log("");
|
|
906
|
+
yield* Console3.log(`Errors (${result.errors.length}):`);
|
|
907
|
+
for (const error of result.errors) {
|
|
908
|
+
yield* Console3.log(` ${error.path}: ${error.message}`);
|
|
909
|
+
}
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
const embedsExist = yield* Effect3.promise(
|
|
913
|
+
() => hasEmbeddings(resolvedDir)
|
|
914
|
+
);
|
|
915
|
+
if (embed) {
|
|
916
|
+
yield* Console3.log("");
|
|
917
|
+
const estimate = yield* estimateEmbeddingCost(resolvedDir, {
|
|
918
|
+
excludePatterns
|
|
919
|
+
}).pipe(handleApiKeyError);
|
|
920
|
+
if (!json) {
|
|
921
|
+
yield* Console3.log(`Found ${estimate.totalFiles} files to embed:`);
|
|
922
|
+
for (const dir of estimate.byDirectory) {
|
|
923
|
+
const costStr = dir.estimatedCost < 1e-3 ? "<$0.001" : `~$${dir.estimatedCost.toFixed(4)}`;
|
|
924
|
+
yield* Console3.log(
|
|
925
|
+
` ${dir.directory.padEnd(20)} ${String(dir.fileCount).padStart(3)} files ${costStr}`
|
|
926
|
+
);
|
|
927
|
+
}
|
|
928
|
+
yield* Console3.log("");
|
|
929
|
+
yield* Console3.log(
|
|
930
|
+
`Total: ~${estimate.totalTokens.toLocaleString()} tokens, ~$${estimate.totalCost.toFixed(4)}, ~${estimate.estimatedTimeSeconds}s`
|
|
931
|
+
);
|
|
932
|
+
yield* Console3.log("");
|
|
933
|
+
}
|
|
934
|
+
if (!force) {
|
|
935
|
+
yield* Console3.log("Checking embeddings...");
|
|
936
|
+
} else {
|
|
937
|
+
yield* Console3.log("Rebuilding embeddings (--force specified)...");
|
|
938
|
+
}
|
|
939
|
+
const embedResult = yield* buildEmbeddings(resolvedDir, {
|
|
940
|
+
force,
|
|
941
|
+
excludePatterns,
|
|
942
|
+
onFileProgress: (progress) => {
|
|
943
|
+
if (!json) {
|
|
944
|
+
process.stdout.write(
|
|
945
|
+
`\r [${progress.fileIndex}/${progress.totalFiles}] ${progress.filePath} (${progress.sectionCount} sections)...`
|
|
946
|
+
);
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
}).pipe(handleApiKeyError);
|
|
950
|
+
if (!json) {
|
|
951
|
+
process.stdout.write(`\r${" ".repeat(80)}\r`);
|
|
952
|
+
yield* Console3.log("");
|
|
953
|
+
if (embedResult.cacheHit) {
|
|
954
|
+
yield* Console3.log(
|
|
955
|
+
`Embeddings already exist (${embedResult.existingVectors} vectors)`
|
|
956
|
+
);
|
|
957
|
+
yield* Console3.log(" Use --force to rebuild");
|
|
958
|
+
yield* Console3.log("");
|
|
959
|
+
yield* Console3.log(
|
|
960
|
+
`Skipped embedding generation (saved ~$${(embedResult.estimatedSavings ?? 0).toFixed(4)})`
|
|
961
|
+
);
|
|
962
|
+
} else {
|
|
963
|
+
yield* Console3.log(
|
|
964
|
+
`Completed in ${(embedResult.duration / 1e3).toFixed(1)}s`
|
|
965
|
+
);
|
|
966
|
+
yield* Console3.log(` Files: ${embedResult.filesProcessed}`);
|
|
967
|
+
yield* Console3.log(` Sections: ${embedResult.sectionsEmbedded}`);
|
|
968
|
+
yield* Console3.log(
|
|
969
|
+
` Tokens: ${embedResult.tokensUsed.toLocaleString()}`
|
|
970
|
+
);
|
|
971
|
+
yield* Console3.log(` Cost: $${embedResult.cost.toFixed(6)}`);
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
} else if (!noEmbed && !embedsExist && !json) {
|
|
975
|
+
yield* Console3.log("");
|
|
976
|
+
yield* Console3.log(
|
|
977
|
+
"Enable semantic search? This allows natural language queries like:"
|
|
978
|
+
);
|
|
979
|
+
yield* Console3.log(
|
|
980
|
+
' "how does authentication work" instead of exact keyword matches'
|
|
981
|
+
);
|
|
982
|
+
yield* Console3.log("");
|
|
983
|
+
const estimate = yield* estimateEmbeddingCost(resolvedDir).pipe(
|
|
984
|
+
Effect3.catchAll(() => Effect3.succeed(null))
|
|
985
|
+
);
|
|
986
|
+
if (estimate) {
|
|
987
|
+
yield* Console3.log(
|
|
988
|
+
`Cost: ~$${estimate.totalCost.toFixed(4)} for this corpus (~${estimate.estimatedTimeSeconds}s)`
|
|
989
|
+
);
|
|
990
|
+
}
|
|
991
|
+
yield* Console3.log("Requires: OPENAI_API_KEY environment variable");
|
|
992
|
+
yield* Console3.log("");
|
|
993
|
+
const answer = yield* Effect3.promise(
|
|
994
|
+
() => promptUser("Create semantic index? [y/N]: ")
|
|
995
|
+
);
|
|
996
|
+
if (answer === "y" || answer === "yes") {
|
|
997
|
+
if (!process.env.OPENAI_API_KEY) {
|
|
998
|
+
yield* Console3.log("");
|
|
999
|
+
yield* Console3.log("OPENAI_API_KEY not set.");
|
|
1000
|
+
yield* Console3.log("");
|
|
1001
|
+
yield* Console3.log(
|
|
1002
|
+
"To enable semantic search, set your OpenAI API key:"
|
|
1003
|
+
);
|
|
1004
|
+
yield* Console3.log(" export OPENAI_API_KEY=sk-...");
|
|
1005
|
+
yield* Console3.log("");
|
|
1006
|
+
yield* Console3.log("Or add to .env file in project root.");
|
|
1007
|
+
} else {
|
|
1008
|
+
yield* Console3.log("");
|
|
1009
|
+
yield* Console3.log("Building embeddings...");
|
|
1010
|
+
const embedResult = yield* buildEmbeddings(resolvedDir, {
|
|
1011
|
+
force: false,
|
|
1012
|
+
onFileProgress: (progress) => {
|
|
1013
|
+
process.stdout.write(
|
|
1014
|
+
`\r [${progress.fileIndex}/${progress.totalFiles}] ${progress.filePath} (${progress.sectionCount} sections)...`
|
|
1015
|
+
);
|
|
1016
|
+
}
|
|
1017
|
+
}).pipe(
|
|
1018
|
+
handleApiKeyError,
|
|
1019
|
+
Effect3.catchAll(() => Effect3.succeed(null))
|
|
1020
|
+
);
|
|
1021
|
+
if (embedResult) {
|
|
1022
|
+
process.stdout.write(`\r${" ".repeat(80)}\r`);
|
|
1023
|
+
yield* Console3.log("");
|
|
1024
|
+
yield* Console3.log(
|
|
1025
|
+
`Completed in ${(embedResult.duration / 1e3).toFixed(1)}s`
|
|
1026
|
+
);
|
|
1027
|
+
yield* Console3.log(` Files: ${embedResult.filesProcessed}`);
|
|
1028
|
+
yield* Console3.log(
|
|
1029
|
+
` Sections: ${embedResult.sectionsEmbedded}`
|
|
1030
|
+
);
|
|
1031
|
+
yield* Console3.log(
|
|
1032
|
+
` Tokens: ${embedResult.tokensUsed.toLocaleString()}`
|
|
1033
|
+
);
|
|
1034
|
+
yield* Console3.log(` Cost: $${embedResult.cost.toFixed(6)}`);
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
if (json) {
|
|
1040
|
+
yield* Console3.log(formatJson(result, pretty));
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
})
|
|
1044
|
+
).pipe(Command3.withDescription("Index markdown files"));
|
|
1045
|
+
|
|
1046
|
+
// src/cli/commands/links.ts
|
|
1047
|
+
import * as path5 from "path";
|
|
1048
|
+
import { Args as Args4, Command as Command4, Options as Options5 } from "@effect/cli";
|
|
1049
|
+
import { Console as Console4, Effect as Effect4 } from "effect";
|
|
1050
|
+
var linksCommand = Command4.make(
|
|
1051
|
+
"links",
|
|
1052
|
+
{
|
|
1053
|
+
file: Args4.file({ name: "file" }).pipe(
|
|
1054
|
+
Args4.withDescription("Markdown file to analyze")
|
|
1055
|
+
),
|
|
1056
|
+
root: Options5.directory("root").pipe(
|
|
1057
|
+
Options5.withAlias("r"),
|
|
1058
|
+
Options5.withDescription("Root directory for resolving relative links"),
|
|
1059
|
+
Options5.withDefault(".")
|
|
1060
|
+
),
|
|
1061
|
+
json: jsonOption,
|
|
1062
|
+
pretty: prettyOption
|
|
1063
|
+
},
|
|
1064
|
+
({ file, root, json, pretty }) => Effect4.gen(function* () {
|
|
1065
|
+
const resolvedRoot = path5.resolve(root);
|
|
1066
|
+
const resolvedFile = path5.resolve(file);
|
|
1067
|
+
const relativePath = path5.relative(resolvedRoot, resolvedFile);
|
|
1068
|
+
const links = yield* getOutgoingLinks(resolvedRoot, resolvedFile);
|
|
1069
|
+
if (json) {
|
|
1070
|
+
yield* Console4.log(formatJson({ file: relativePath, links }, pretty));
|
|
1071
|
+
} else {
|
|
1072
|
+
yield* Console4.log(`Outgoing links from ${relativePath}:`);
|
|
1073
|
+
yield* Console4.log("");
|
|
1074
|
+
if (links.length === 0) {
|
|
1075
|
+
yield* Console4.log(" (none)");
|
|
1076
|
+
} else {
|
|
1077
|
+
for (const link of links) {
|
|
1078
|
+
yield* Console4.log(` -> ${link}`);
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
yield* Console4.log("");
|
|
1082
|
+
yield* Console4.log(`Total: ${links.length} links`);
|
|
1083
|
+
}
|
|
1084
|
+
})
|
|
1085
|
+
).pipe(Command4.withDescription("What does this link to?"));
|
|
1086
|
+
|
|
1087
|
+
// src/cli/commands/search.ts
|
|
1088
|
+
import * as path6 from "path";
|
|
1089
|
+
import * as readline2 from "readline";
|
|
1090
|
+
import { Args as Args5, Command as Command5, Options as Options6 } from "@effect/cli";
|
|
1091
|
+
import { Console as Console5, Effect as Effect5, Option } from "effect";
|
|
1092
|
+
var AUTO_INDEX_THRESHOLD_SECONDS = 10;
|
|
1093
|
+
var promptUser2 = (message) => {
|
|
1094
|
+
return new Promise((resolve8) => {
|
|
1095
|
+
const rl = readline2.createInterface({
|
|
1096
|
+
input: process.stdin,
|
|
1097
|
+
output: process.stdout
|
|
1098
|
+
});
|
|
1099
|
+
rl.question(message, (answer) => {
|
|
1100
|
+
rl.close();
|
|
1101
|
+
resolve8(answer.trim().toLowerCase());
|
|
1102
|
+
});
|
|
1103
|
+
});
|
|
1104
|
+
};
|
|
1105
|
+
var searchCommand = Command5.make(
|
|
1106
|
+
"search",
|
|
1107
|
+
{
|
|
1108
|
+
query: Args5.text({ name: "query" }).pipe(
|
|
1109
|
+
Args5.withDescription("Search query (natural language or regex pattern)")
|
|
1110
|
+
),
|
|
1111
|
+
path: Args5.directory({ name: "path" }).pipe(
|
|
1112
|
+
Args5.withDescription("Directory to search in"),
|
|
1113
|
+
Args5.withDefault(".")
|
|
1114
|
+
),
|
|
1115
|
+
keyword: Options6.boolean("keyword").pipe(
|
|
1116
|
+
Options6.withAlias("k"),
|
|
1117
|
+
Options6.withDescription("Force keyword search (content text match)"),
|
|
1118
|
+
Options6.withDefault(false)
|
|
1119
|
+
),
|
|
1120
|
+
headingOnly: Options6.boolean("heading-only").pipe(
|
|
1121
|
+
Options6.withAlias("H"),
|
|
1122
|
+
Options6.withDescription("Search headings only (not content)"),
|
|
1123
|
+
Options6.withDefault(false)
|
|
1124
|
+
),
|
|
1125
|
+
mode: Options6.choice("mode", ["semantic", "keyword"]).pipe(
|
|
1126
|
+
Options6.withAlias("m"),
|
|
1127
|
+
Options6.withDescription("Force search mode: semantic or keyword"),
|
|
1128
|
+
Options6.optional
|
|
1129
|
+
),
|
|
1130
|
+
limit: Options6.integer("limit").pipe(
|
|
1131
|
+
Options6.withAlias("n"),
|
|
1132
|
+
Options6.withDescription("Maximum results"),
|
|
1133
|
+
Options6.withDefault(10)
|
|
1134
|
+
),
|
|
1135
|
+
threshold: Options6.float("threshold").pipe(
|
|
1136
|
+
Options6.withDescription("Similarity threshold for semantic search (0-1)"),
|
|
1137
|
+
Options6.withDefault(0.45)
|
|
1138
|
+
),
|
|
1139
|
+
context: Options6.integer("context").pipe(
|
|
1140
|
+
Options6.withAlias("C"),
|
|
1141
|
+
Options6.withDescription("Lines of context around matches (like grep -C)"),
|
|
1142
|
+
Options6.optional
|
|
1143
|
+
),
|
|
1144
|
+
beforeContext: Options6.integer("before-context").pipe(
|
|
1145
|
+
Options6.withAlias("B"),
|
|
1146
|
+
Options6.withDescription("Lines of context before matches (like grep -B)"),
|
|
1147
|
+
Options6.optional
|
|
1148
|
+
),
|
|
1149
|
+
afterContext: Options6.integer("after-context").pipe(
|
|
1150
|
+
Options6.withAlias("A"),
|
|
1151
|
+
Options6.withDescription("Lines of context after matches (like grep -A)"),
|
|
1152
|
+
Options6.optional
|
|
1153
|
+
),
|
|
1154
|
+
autoIndexThreshold: Options6.integer("auto-index-threshold").pipe(
|
|
1155
|
+
Options6.withDescription(
|
|
1156
|
+
"Auto-create semantic index if estimated time is under this threshold (seconds)"
|
|
1157
|
+
),
|
|
1158
|
+
Options6.withDefault(AUTO_INDEX_THRESHOLD_SECONDS)
|
|
1159
|
+
),
|
|
1160
|
+
json: jsonOption,
|
|
1161
|
+
pretty: prettyOption
|
|
1162
|
+
},
|
|
1163
|
+
({
|
|
1164
|
+
query,
|
|
1165
|
+
path: dirPath,
|
|
1166
|
+
keyword,
|
|
1167
|
+
headingOnly,
|
|
1168
|
+
mode,
|
|
1169
|
+
limit,
|
|
1170
|
+
threshold,
|
|
1171
|
+
context,
|
|
1172
|
+
beforeContext,
|
|
1173
|
+
afterContext,
|
|
1174
|
+
autoIndexThreshold,
|
|
1175
|
+
json,
|
|
1176
|
+
pretty
|
|
1177
|
+
}) => Effect5.gen(function* () {
|
|
1178
|
+
const resolvedDir = path6.resolve(dirPath);
|
|
1179
|
+
const indexInfo = yield* Effect5.promise(() => getIndexInfo(resolvedDir));
|
|
1180
|
+
if (!indexInfo.exists && !json) {
|
|
1181
|
+
yield* Console5.log("No index found.");
|
|
1182
|
+
yield* Console5.log("");
|
|
1183
|
+
yield* Console5.log("Run: mdcontext index /path/to/docs");
|
|
1184
|
+
yield* Console5.log(" Add --embed for semantic search capabilities");
|
|
1185
|
+
return;
|
|
1186
|
+
}
|
|
1187
|
+
let embedsExist = indexInfo.embeddingsExist;
|
|
1188
|
+
let useKeyword;
|
|
1189
|
+
let modeReason;
|
|
1190
|
+
const modeValue = Option.getOrUndefined(mode);
|
|
1191
|
+
if (modeValue === "semantic") {
|
|
1192
|
+
if (!embedsExist) {
|
|
1193
|
+
embedsExist = yield* handleMissingEmbeddings(
|
|
1194
|
+
resolvedDir,
|
|
1195
|
+
autoIndexThreshold,
|
|
1196
|
+
json
|
|
1197
|
+
);
|
|
1198
|
+
if (!embedsExist) {
|
|
1199
|
+
return;
|
|
1200
|
+
}
|
|
1201
|
+
}
|
|
1202
|
+
useKeyword = false;
|
|
1203
|
+
modeReason = "--mode semantic";
|
|
1204
|
+
} else if (modeValue === "keyword") {
|
|
1205
|
+
useKeyword = true;
|
|
1206
|
+
modeReason = "--mode keyword";
|
|
1207
|
+
} else if (keyword) {
|
|
1208
|
+
useKeyword = true;
|
|
1209
|
+
modeReason = "--keyword flag";
|
|
1210
|
+
} else if (isAdvancedQuery(query)) {
|
|
1211
|
+
useKeyword = true;
|
|
1212
|
+
modeReason = "boolean/phrase pattern detected";
|
|
1213
|
+
} else if (isRegexPattern(query)) {
|
|
1214
|
+
useKeyword = true;
|
|
1215
|
+
modeReason = "regex pattern detected";
|
|
1216
|
+
} else if (!embedsExist) {
|
|
1217
|
+
useKeyword = true;
|
|
1218
|
+
modeReason = "no embeddings";
|
|
1219
|
+
} else {
|
|
1220
|
+
useKeyword = false;
|
|
1221
|
+
modeReason = "embeddings available";
|
|
1222
|
+
}
|
|
1223
|
+
const modeIndicator = useKeyword ? "[keyword]" : "[semantic]";
|
|
1224
|
+
if (!json && indexInfo.lastUpdated) {
|
|
1225
|
+
const lastUpdatedDate = new Date(indexInfo.lastUpdated);
|
|
1226
|
+
const dateStr = lastUpdatedDate.toLocaleDateString("en-CA");
|
|
1227
|
+
const timeStr = lastUpdatedDate.toLocaleTimeString("en-US", {
|
|
1228
|
+
hour: "2-digit",
|
|
1229
|
+
minute: "2-digit",
|
|
1230
|
+
hour12: false
|
|
1231
|
+
});
|
|
1232
|
+
yield* Console5.log(`Using index from ${dateStr} ${timeStr}`);
|
|
1233
|
+
yield* Console5.log(` Sections: ${indexInfo.sectionCount ?? 0}`);
|
|
1234
|
+
if (indexInfo.embeddingsExist) {
|
|
1235
|
+
yield* Console5.log(
|
|
1236
|
+
` Embeddings: yes (${indexInfo.vectorCount ?? 0} vectors)`
|
|
1237
|
+
);
|
|
1238
|
+
} else {
|
|
1239
|
+
yield* Console5.log(" Embeddings: no");
|
|
1240
|
+
}
|
|
1241
|
+
yield* Console5.log("");
|
|
1242
|
+
}
|
|
1243
|
+
const contextValue = Option.getOrUndefined(context);
|
|
1244
|
+
const beforeValue = Option.getOrUndefined(beforeContext);
|
|
1245
|
+
const afterValue = Option.getOrUndefined(afterContext);
|
|
1246
|
+
const contextBefore = beforeValue ?? contextValue ?? 1;
|
|
1247
|
+
const contextAfter = afterValue ?? contextValue ?? 1;
|
|
1248
|
+
if (useKeyword) {
|
|
1249
|
+
const results = headingOnly ? yield* search(resolvedDir, { heading: query, limit }) : yield* searchContent(resolvedDir, {
|
|
1250
|
+
content: query,
|
|
1251
|
+
limit,
|
|
1252
|
+
contextBefore,
|
|
1253
|
+
contextAfter
|
|
1254
|
+
});
|
|
1255
|
+
if (json) {
|
|
1256
|
+
const output = {
|
|
1257
|
+
mode: "keyword",
|
|
1258
|
+
modeReason,
|
|
1259
|
+
query,
|
|
1260
|
+
contextBefore,
|
|
1261
|
+
contextAfter,
|
|
1262
|
+
results: results.map((r) => ({
|
|
1263
|
+
path: r.section.documentPath,
|
|
1264
|
+
heading: r.section.heading,
|
|
1265
|
+
level: r.section.level,
|
|
1266
|
+
tokens: r.section.tokenCount,
|
|
1267
|
+
line: r.section.startLine,
|
|
1268
|
+
matches: r.matches?.map((m) => ({
|
|
1269
|
+
lineNumber: m.lineNumber,
|
|
1270
|
+
line: m.line,
|
|
1271
|
+
contextLines: m.contextLines
|
|
1272
|
+
}))
|
|
1273
|
+
}))
|
|
1274
|
+
};
|
|
1275
|
+
yield* Console5.log(formatJson(output, pretty));
|
|
1276
|
+
} else {
|
|
1277
|
+
const searchType = headingOnly ? "Heading" : "Content";
|
|
1278
|
+
const showReason = modeReason !== "--mode keyword" && modeReason !== "--keyword flag";
|
|
1279
|
+
const modeStr = showReason ? `${modeIndicator} (${modeReason})` : modeIndicator;
|
|
1280
|
+
yield* Console5.log(`${modeStr} ${searchType} search: "${query}"`);
|
|
1281
|
+
yield* Console5.log(`Results: ${results.length}`);
|
|
1282
|
+
yield* Console5.log("");
|
|
1283
|
+
for (const result of results) {
|
|
1284
|
+
const levelMarker = "#".repeat(result.section.level);
|
|
1285
|
+
yield* Console5.log(
|
|
1286
|
+
` ${result.section.documentPath}:${result.section.startLine}`
|
|
1287
|
+
);
|
|
1288
|
+
yield* Console5.log(
|
|
1289
|
+
` ${levelMarker} ${result.section.heading} (${result.section.tokenCount} tokens)`
|
|
1290
|
+
);
|
|
1291
|
+
if (result.matches && result.matches.length > 0) {
|
|
1292
|
+
yield* Console5.log("");
|
|
1293
|
+
for (const match of result.matches.slice(0, 3)) {
|
|
1294
|
+
if (match.contextLines && match.contextLines.length > 0) {
|
|
1295
|
+
for (const ctxLine of match.contextLines) {
|
|
1296
|
+
const marker = ctxLine.isMatch ? ">" : " ";
|
|
1297
|
+
yield* Console5.log(
|
|
1298
|
+
` ${marker} ${ctxLine.lineNumber}: ${ctxLine.line}`
|
|
1299
|
+
);
|
|
1300
|
+
}
|
|
1301
|
+
} else {
|
|
1302
|
+
yield* Console5.log(` Line ${match.lineNumber}:`);
|
|
1303
|
+
const snippetLines = match.snippet.split("\n");
|
|
1304
|
+
for (const line of snippetLines) {
|
|
1305
|
+
yield* Console5.log(` ${line}`);
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
1308
|
+
yield* Console5.log("");
|
|
1309
|
+
}
|
|
1310
|
+
if (result.matches.length > 3) {
|
|
1311
|
+
yield* Console5.log(
|
|
1312
|
+
` ... and ${result.matches.length - 3} more matches`
|
|
1313
|
+
);
|
|
1314
|
+
}
|
|
1315
|
+
}
|
|
1316
|
+
yield* Console5.log("");
|
|
1317
|
+
}
|
|
1318
|
+
if (!indexInfo.embeddingsExist) {
|
|
1319
|
+
yield* Console5.log(
|
|
1320
|
+
"Tip: Run 'mdcontext index --embed' to enable semantic search"
|
|
1321
|
+
);
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
} else {
|
|
1325
|
+
const results = yield* semanticSearch(resolvedDir, query, {
|
|
1326
|
+
limit,
|
|
1327
|
+
threshold
|
|
1328
|
+
}).pipe(handleApiKeyError);
|
|
1329
|
+
if (json) {
|
|
1330
|
+
const output = {
|
|
1331
|
+
mode: "semantic",
|
|
1332
|
+
modeReason,
|
|
1333
|
+
query,
|
|
1334
|
+
results
|
|
1335
|
+
};
|
|
1336
|
+
yield* Console5.log(formatJson(output, pretty));
|
|
1337
|
+
} else {
|
|
1338
|
+
const showSemanticReason = modeReason !== "--mode semantic";
|
|
1339
|
+
const semanticModeStr = showSemanticReason ? `${modeIndicator} (${modeReason})` : modeIndicator;
|
|
1340
|
+
yield* Console5.log(`${semanticModeStr} Semantic search: "${query}"`);
|
|
1341
|
+
yield* Console5.log(`Results: ${results.length}`);
|
|
1342
|
+
yield* Console5.log("");
|
|
1343
|
+
for (const result of results) {
|
|
1344
|
+
const similarity = (result.similarity * 100).toFixed(1);
|
|
1345
|
+
yield* Console5.log(` ${result.documentPath}`);
|
|
1346
|
+
yield* Console5.log(` ${result.heading} (${similarity}% match)`);
|
|
1347
|
+
yield* Console5.log("");
|
|
1348
|
+
}
|
|
1349
|
+
yield* Console5.log("Tip: Use --mode keyword for exact text matching");
|
|
1350
|
+
}
|
|
1351
|
+
}
|
|
1352
|
+
})
|
|
1353
|
+
).pipe(Command5.withDescription("Search by meaning or structure"));
|
|
1354
|
+
var handleMissingEmbeddings = (resolvedDir, autoIndexThreshold, json) => Effect5.gen(function* () {
|
|
1355
|
+
const estimate = yield* estimateEmbeddingCost(resolvedDir).pipe(
|
|
1356
|
+
Effect5.catchAll(() => Effect5.succeed(null))
|
|
1357
|
+
);
|
|
1358
|
+
if (!estimate) {
|
|
1359
|
+
yield* Console5.error(
|
|
1360
|
+
"No semantic index found and could not estimate cost."
|
|
1361
|
+
);
|
|
1362
|
+
yield* Console5.error('Run "mdcontext index --embed" first.');
|
|
1363
|
+
return false;
|
|
1364
|
+
}
|
|
1365
|
+
if (estimate.estimatedTimeSeconds <= autoIndexThreshold) {
|
|
1366
|
+
if (!json) {
|
|
1367
|
+
yield* Console5.log(
|
|
1368
|
+
`Creating semantic index (~${estimate.estimatedTimeSeconds}s, ~$${estimate.totalCost.toFixed(4)})...`
|
|
1369
|
+
);
|
|
1370
|
+
}
|
|
1371
|
+
const result = yield* buildEmbeddings(resolvedDir, {
|
|
1372
|
+
force: false,
|
|
1373
|
+
onFileProgress: (progress) => {
|
|
1374
|
+
if (!json) {
|
|
1375
|
+
process.stdout.write(
|
|
1376
|
+
`\r [${progress.fileIndex}/${progress.totalFiles}] ${progress.filePath}...`
|
|
1377
|
+
);
|
|
1378
|
+
}
|
|
1379
|
+
}
|
|
1380
|
+
}).pipe(
|
|
1381
|
+
handleApiKeyError,
|
|
1382
|
+
Effect5.catchAll(() => Effect5.succeed(null))
|
|
1383
|
+
);
|
|
1384
|
+
if (!result) {
|
|
1385
|
+
return false;
|
|
1386
|
+
}
|
|
1387
|
+
if (!json) {
|
|
1388
|
+
process.stdout.write(`\r${" ".repeat(80)}\r`);
|
|
1389
|
+
yield* Console5.log(
|
|
1390
|
+
`Index created (${result.sectionsEmbedded} sections, $${result.cost.toFixed(6)})`
|
|
1391
|
+
);
|
|
1392
|
+
yield* Console5.log("");
|
|
1393
|
+
}
|
|
1394
|
+
return true;
|
|
1395
|
+
}
|
|
1396
|
+
if (!json) {
|
|
1397
|
+
yield* Console5.log("");
|
|
1398
|
+
yield* Console5.log("No semantic index found.");
|
|
1399
|
+
yield* Console5.log("");
|
|
1400
|
+
yield* Console5.log("Options:");
|
|
1401
|
+
yield* Console5.log(
|
|
1402
|
+
` 1. Create now (recommended, ~${estimate.estimatedTimeSeconds}s, ~$${estimate.totalCost.toFixed(4)})`
|
|
1403
|
+
);
|
|
1404
|
+
yield* Console5.log(" 2. Use keyword search instead");
|
|
1405
|
+
yield* Console5.log("");
|
|
1406
|
+
}
|
|
1407
|
+
const answer = yield* Effect5.promise(() => promptUser2("Choice [1]: "));
|
|
1408
|
+
const choice = answer === "" || answer === "1" ? "1" : answer;
|
|
1409
|
+
if (choice === "1") {
|
|
1410
|
+
if (!json) {
|
|
1411
|
+
yield* Console5.log("");
|
|
1412
|
+
yield* Console5.log("Building embeddings...");
|
|
1413
|
+
}
|
|
1414
|
+
const result = yield* buildEmbeddings(resolvedDir, {
|
|
1415
|
+
force: false,
|
|
1416
|
+
onFileProgress: (progress) => {
|
|
1417
|
+
if (!json) {
|
|
1418
|
+
process.stdout.write(
|
|
1419
|
+
`\r [${progress.fileIndex}/${progress.totalFiles}] ${progress.filePath}...`
|
|
1420
|
+
);
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
}).pipe(
|
|
1424
|
+
handleApiKeyError,
|
|
1425
|
+
Effect5.catchAll(() => Effect5.succeed(null))
|
|
1426
|
+
);
|
|
1427
|
+
if (!result) {
|
|
1428
|
+
return false;
|
|
1429
|
+
}
|
|
1430
|
+
if (!json) {
|
|
1431
|
+
process.stdout.write(`\r${" ".repeat(80)}\r`);
|
|
1432
|
+
yield* Console5.log(
|
|
1433
|
+
`Index created (${result.sectionsEmbedded} sections, $${result.cost.toFixed(6)})`
|
|
1434
|
+
);
|
|
1435
|
+
yield* Console5.log("");
|
|
1436
|
+
}
|
|
1437
|
+
return true;
|
|
1438
|
+
}
|
|
1439
|
+
yield* Console5.log("");
|
|
1440
|
+
yield* Console5.log("Falling back to keyword search.");
|
|
1441
|
+
return false;
|
|
1442
|
+
});
|
|
1443
|
+
|
|
1444
|
+
// src/cli/commands/stats.ts
|
|
1445
|
+
import * as path7 from "path";
|
|
1446
|
+
import { Args as Args6, Command as Command6 } from "@effect/cli";
|
|
1447
|
+
import { Console as Console6, Effect as Effect6 } from "effect";
|
|
1448
|
+
var statsCommand = Command6.make(
|
|
1449
|
+
"stats",
|
|
1450
|
+
{
|
|
1451
|
+
path: Args6.directory({ name: "path" }).pipe(
|
|
1452
|
+
Args6.withDescription("Directory to show stats for"),
|
|
1453
|
+
Args6.withDefault(".")
|
|
1454
|
+
),
|
|
1455
|
+
json: jsonOption,
|
|
1456
|
+
pretty: prettyOption
|
|
1457
|
+
},
|
|
1458
|
+
({ path: dirPath, json, pretty }) => Effect6.gen(function* () {
|
|
1459
|
+
const resolvedRoot = path7.resolve(dirPath);
|
|
1460
|
+
const storage = createStorage(resolvedRoot);
|
|
1461
|
+
const docIndex = yield* loadDocumentIndex(storage);
|
|
1462
|
+
const sectionIndex = yield* loadSectionIndex(storage);
|
|
1463
|
+
if (!docIndex || !sectionIndex) {
|
|
1464
|
+
if (json) {
|
|
1465
|
+
yield* Console6.log(formatJson({ error: "No index found" }, pretty));
|
|
1466
|
+
} else {
|
|
1467
|
+
yield* Console6.log("No index found.");
|
|
1468
|
+
yield* Console6.log("Run 'mdcontext index <path>' to create an index.");
|
|
1469
|
+
}
|
|
1470
|
+
return;
|
|
1471
|
+
}
|
|
1472
|
+
const docs = Object.values(docIndex.documents);
|
|
1473
|
+
const sections = Object.values(sectionIndex.sections);
|
|
1474
|
+
const tokenCounts = docs.map((d) => d.tokenCount).sort((a, b) => a - b);
|
|
1475
|
+
const totalTokens = tokenCounts.reduce((sum, t) => sum + t, 0);
|
|
1476
|
+
const sectionsByLevel = {};
|
|
1477
|
+
for (const section of sections) {
|
|
1478
|
+
sectionsByLevel[section.level] = (sectionsByLevel[section.level] || 0) + 1;
|
|
1479
|
+
}
|
|
1480
|
+
const indexStats = {
|
|
1481
|
+
documentCount: docs.length,
|
|
1482
|
+
totalTokens,
|
|
1483
|
+
avgTokensPerDoc: docs.length > 0 ? Math.round(totalTokens / docs.length) : 0,
|
|
1484
|
+
totalSections: sections.length,
|
|
1485
|
+
sectionsByLevel,
|
|
1486
|
+
tokenDistribution: {
|
|
1487
|
+
min: tokenCounts[0] || 0,
|
|
1488
|
+
max: tokenCounts[tokenCounts.length - 1] || 0,
|
|
1489
|
+
median: tokenCounts[Math.floor(tokenCounts.length / 2)] || 0
|
|
1490
|
+
}
|
|
1491
|
+
};
|
|
1492
|
+
const embeddingStats = yield* getEmbeddingStats(resolvedRoot);
|
|
1493
|
+
if (json) {
|
|
1494
|
+
yield* Console6.log(
|
|
1495
|
+
formatJson({ ...indexStats, embeddings: embeddingStats }, pretty)
|
|
1496
|
+
);
|
|
1497
|
+
} else {
|
|
1498
|
+
yield* Console6.log("Index statistics:");
|
|
1499
|
+
yield* Console6.log("");
|
|
1500
|
+
yield* Console6.log(" Documents");
|
|
1501
|
+
yield* Console6.log(` Count: ${indexStats.documentCount}`);
|
|
1502
|
+
yield* Console6.log(
|
|
1503
|
+
` Tokens: ${indexStats.totalTokens.toLocaleString()}`
|
|
1504
|
+
);
|
|
1505
|
+
yield* Console6.log(` Avg/doc: ${indexStats.avgTokensPerDoc}`);
|
|
1506
|
+
yield* Console6.log("");
|
|
1507
|
+
yield* Console6.log(" Token distribution");
|
|
1508
|
+
yield* Console6.log(
|
|
1509
|
+
` Min: ${indexStats.tokenDistribution.min}`
|
|
1510
|
+
);
|
|
1511
|
+
yield* Console6.log(
|
|
1512
|
+
` Median: ${indexStats.tokenDistribution.median}`
|
|
1513
|
+
);
|
|
1514
|
+
yield* Console6.log(
|
|
1515
|
+
` Max: ${indexStats.tokenDistribution.max}`
|
|
1516
|
+
);
|
|
1517
|
+
yield* Console6.log("");
|
|
1518
|
+
yield* Console6.log(" Sections");
|
|
1519
|
+
yield* Console6.log(` Total: ${indexStats.totalSections}`);
|
|
1520
|
+
const levels = Object.keys(sectionsByLevel).map(Number).sort((a, b) => a - b);
|
|
1521
|
+
for (const level of levels) {
|
|
1522
|
+
yield* Console6.log(
|
|
1523
|
+
` h${level}: ${sectionsByLevel[level]}`
|
|
1524
|
+
);
|
|
1525
|
+
}
|
|
1526
|
+
yield* Console6.log("");
|
|
1527
|
+
yield* Console6.log(" Embeddings");
|
|
1528
|
+
if (embeddingStats.hasEmbeddings) {
|
|
1529
|
+
yield* Console6.log(` Vectors: ${embeddingStats.count}`);
|
|
1530
|
+
yield* Console6.log(` Provider: ${embeddingStats.provider}`);
|
|
1531
|
+
yield* Console6.log(` Dimensions: ${embeddingStats.dimensions}`);
|
|
1532
|
+
yield* Console6.log(
|
|
1533
|
+
` Cost: $${embeddingStats.totalCost.toFixed(6)}`
|
|
1534
|
+
);
|
|
1535
|
+
} else {
|
|
1536
|
+
yield* Console6.log(" Not enabled");
|
|
1537
|
+
yield* Console6.log(
|
|
1538
|
+
" Run 'mdcontext index --embed' to build embeddings."
|
|
1539
|
+
);
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
})
|
|
1543
|
+
).pipe(Command6.withDescription("Index statistics"));
|
|
1544
|
+
|
|
1545
|
+
// src/cli/commands/tree.ts
|
|
1546
|
+
import * as fs from "fs";
|
|
1547
|
+
import * as path8 from "path";
|
|
1548
|
+
import { Args as Args7, Command as Command7 } from "@effect/cli";
|
|
1549
|
+
import { Console as Console7, Effect as Effect7 } from "effect";
|
|
1550
|
+
var treeCommand = Command7.make(
|
|
1551
|
+
"tree",
|
|
1552
|
+
{
|
|
1553
|
+
pathArg: Args7.text({ name: "path" }).pipe(
|
|
1554
|
+
Args7.withDescription("Directory (shows files) or file (shows outline)"),
|
|
1555
|
+
Args7.withDefault(".")
|
|
1556
|
+
),
|
|
1557
|
+
json: jsonOption,
|
|
1558
|
+
pretty: prettyOption
|
|
1559
|
+
},
|
|
1560
|
+
({ pathArg, json, pretty }) => Effect7.gen(function* () {
|
|
1561
|
+
const resolvedPath = path8.resolve(pathArg);
|
|
1562
|
+
const stat2 = yield* Effect7.try(() => fs.statSync(resolvedPath));
|
|
1563
|
+
if (stat2.isFile()) {
|
|
1564
|
+
const result = yield* parseFile(resolvedPath).pipe(
|
|
1565
|
+
Effect7.mapError((e) => new Error(`${e._tag}: ${e.message}`))
|
|
1566
|
+
);
|
|
1567
|
+
const extractStructure = (section) => ({
|
|
1568
|
+
heading: section.heading,
|
|
1569
|
+
level: section.level,
|
|
1570
|
+
tokens: section.metadata.tokenCount,
|
|
1571
|
+
children: section.children.map(extractStructure)
|
|
1572
|
+
});
|
|
1573
|
+
if (json) {
|
|
1574
|
+
const structure = {
|
|
1575
|
+
title: result.title,
|
|
1576
|
+
path: result.path,
|
|
1577
|
+
totalTokens: result.metadata.tokenCount,
|
|
1578
|
+
sections: result.sections.map(extractStructure)
|
|
1579
|
+
};
|
|
1580
|
+
yield* Console7.log(formatJson(structure, pretty));
|
|
1581
|
+
} else {
|
|
1582
|
+
yield* Console7.log(`# ${result.title}`);
|
|
1583
|
+
yield* Console7.log(`Total tokens: ${result.metadata.tokenCount}`);
|
|
1584
|
+
yield* Console7.log("");
|
|
1585
|
+
const printOutline = (section, depth = 0) => Effect7.gen(function* () {
|
|
1586
|
+
const indent = " ".repeat(depth);
|
|
1587
|
+
const marker = "#".repeat(section.level);
|
|
1588
|
+
yield* Console7.log(
|
|
1589
|
+
`${indent}${marker} ${section.heading} [${section.metadata.tokenCount} tokens]`
|
|
1590
|
+
);
|
|
1591
|
+
for (const child of section.children) {
|
|
1592
|
+
yield* printOutline(child, depth + 1);
|
|
1593
|
+
}
|
|
1594
|
+
});
|
|
1595
|
+
for (const section of result.sections) {
|
|
1596
|
+
yield* printOutline(section);
|
|
1597
|
+
}
|
|
1598
|
+
}
|
|
1599
|
+
} else {
|
|
1600
|
+
const files = yield* Effect7.promise(() => walkDir(resolvedPath));
|
|
1601
|
+
const tree = files.sort().map((f) => ({
|
|
1602
|
+
path: f,
|
|
1603
|
+
relativePath: path8.relative(resolvedPath, f)
|
|
1604
|
+
}));
|
|
1605
|
+
if (json) {
|
|
1606
|
+
yield* Console7.log(formatJson(tree, pretty));
|
|
1607
|
+
} else {
|
|
1608
|
+
yield* Console7.log(`Markdown files in ${resolvedPath}:`);
|
|
1609
|
+
yield* Console7.log("");
|
|
1610
|
+
for (const file of tree) {
|
|
1611
|
+
yield* Console7.log(` ${file.relativePath}`);
|
|
1612
|
+
}
|
|
1613
|
+
yield* Console7.log("");
|
|
1614
|
+
yield* Console7.log(`Total: ${tree.length} files`);
|
|
1615
|
+
}
|
|
1616
|
+
}
|
|
1617
|
+
})
|
|
1618
|
+
).pipe(Command7.withDescription("Show files or document outline"));
|
|
1619
|
+
|
|
1620
|
+
// src/cli/help.ts
|
|
1621
|
+
var helpContent = {
|
|
1622
|
+
index: {
|
|
1623
|
+
description: "Index markdown files for fast searching",
|
|
1624
|
+
usage: "mdcontext index [path] [options]",
|
|
1625
|
+
examples: [
|
|
1626
|
+
"mdcontext index # Index current directory",
|
|
1627
|
+
"mdcontext index docs/ # Index specific directory",
|
|
1628
|
+
"mdcontext index --embed # Include semantic embeddings",
|
|
1629
|
+
"mdcontext index --watch # Watch for file changes",
|
|
1630
|
+
"mdcontext index --embed --watch # Full setup with live updates",
|
|
1631
|
+
"mdcontext index --force # Rebuild from scratch"
|
|
1632
|
+
],
|
|
1633
|
+
options: [
|
|
1634
|
+
{
|
|
1635
|
+
name: "-e, --embed",
|
|
1636
|
+
description: "Build semantic embeddings (enables AI-powered search)"
|
|
1637
|
+
},
|
|
1638
|
+
{
|
|
1639
|
+
name: "--no-embed",
|
|
1640
|
+
description: "Skip the prompt to enable semantic search"
|
|
1641
|
+
},
|
|
1642
|
+
{
|
|
1643
|
+
name: "-w, --watch",
|
|
1644
|
+
description: "Watch for changes and re-index automatically"
|
|
1645
|
+
},
|
|
1646
|
+
{ name: "--force", description: "Rebuild from scratch, ignoring cache" },
|
|
1647
|
+
{ name: "--json", description: "Output results as JSON" },
|
|
1648
|
+
{ name: "--pretty", description: "Pretty-print JSON output" }
|
|
1649
|
+
],
|
|
1650
|
+
notes: [
|
|
1651
|
+
"After indexing, prompts to enable semantic search (use --no-embed to skip).",
|
|
1652
|
+
"Embedding requires OPENAI_API_KEY environment variable.",
|
|
1653
|
+
"Index is stored in .mdcontext/ directory."
|
|
1654
|
+
]
|
|
1655
|
+
},
|
|
1656
|
+
search: {
|
|
1657
|
+
description: "Search markdown content by meaning or heading pattern",
|
|
1658
|
+
usage: "mdcontext search [options] <query> [path]",
|
|
1659
|
+
examples: [
|
|
1660
|
+
'mdcontext search "auth" # Simple term search',
|
|
1661
|
+
'mdcontext search "auth AND deploy" # Both terms required',
|
|
1662
|
+
'mdcontext search "error OR bug" # Either term matches',
|
|
1663
|
+
'mdcontext search "impl NOT test" # Exclude "test"',
|
|
1664
|
+
'mdcontext search "auth AND (error OR bug)" # Grouped expressions',
|
|
1665
|
+
`mdcontext search '"exact phrase"' # Exact phrase match`,
|
|
1666
|
+
`mdcontext search '"context resumption" AND drift' # Phrase + boolean`,
|
|
1667
|
+
'mdcontext search -H "API.*" # Regex on headings only',
|
|
1668
|
+
'mdcontext search --mode keyword "auth" # Force keyword mode',
|
|
1669
|
+
'mdcontext search --mode semantic "auth" # Force semantic mode',
|
|
1670
|
+
'mdcontext search -n 5 "setup" # Limit to 5 results',
|
|
1671
|
+
'mdcontext search "config" docs/ # Search in specific directory',
|
|
1672
|
+
"",
|
|
1673
|
+
"# Context lines (like grep):",
|
|
1674
|
+
'mdcontext search "checkpoint" -C 3 # 3 lines before AND after',
|
|
1675
|
+
'mdcontext search "error" -B 2 -A 5 # 2 before, 5 after'
|
|
1676
|
+
],
|
|
1677
|
+
options: [
|
|
1678
|
+
{
|
|
1679
|
+
name: "-k, --keyword",
|
|
1680
|
+
description: "Force keyword search (content text match)"
|
|
1681
|
+
},
|
|
1682
|
+
{
|
|
1683
|
+
name: "-H, --heading-only",
|
|
1684
|
+
description: "Search headings only (not content)"
|
|
1685
|
+
},
|
|
1686
|
+
{
|
|
1687
|
+
name: "-m, --mode <mode>",
|
|
1688
|
+
description: "Force search mode: semantic or keyword"
|
|
1689
|
+
},
|
|
1690
|
+
{
|
|
1691
|
+
name: "-n, --limit <n>",
|
|
1692
|
+
description: "Maximum number of results (default: 10)"
|
|
1693
|
+
},
|
|
1694
|
+
{
|
|
1695
|
+
name: "-C <n>",
|
|
1696
|
+
description: "Show N context lines before AND after each match"
|
|
1697
|
+
},
|
|
1698
|
+
{
|
|
1699
|
+
name: "-B <n>",
|
|
1700
|
+
description: "Show N context lines before each match"
|
|
1701
|
+
},
|
|
1702
|
+
{
|
|
1703
|
+
name: "-A <n>",
|
|
1704
|
+
description: "Show N context lines after each match"
|
|
1705
|
+
},
|
|
1706
|
+
{
|
|
1707
|
+
name: "--threshold <n>",
|
|
1708
|
+
description: "Similarity threshold 0-1 for semantic search (default: 0.5)"
|
|
1709
|
+
},
|
|
1710
|
+
{ name: "--json", description: "Output results as JSON" },
|
|
1711
|
+
{ name: "--pretty", description: "Pretty-print JSON output" }
|
|
1712
|
+
],
|
|
1713
|
+
notes: [
|
|
1714
|
+
"Auto-detects mode: semantic if embeddings exist, keyword otherwise.",
|
|
1715
|
+
"Boolean operators: AND, OR, NOT (case-insensitive).",
|
|
1716
|
+
'Quoted phrases match exactly: "context resumption".',
|
|
1717
|
+
'Regex patterns (e.g., "API.*") always use keyword search.',
|
|
1718
|
+
'Run "mdcontext index --embed" first for semantic search.'
|
|
1719
|
+
]
|
|
1720
|
+
},
|
|
1721
|
+
context: {
|
|
1722
|
+
description: "Get LLM-ready summary of markdown files",
|
|
1723
|
+
usage: "mdcontext context [options] <files>...",
|
|
1724
|
+
examples: [
|
|
1725
|
+
"mdcontext context README.md # Summarize single file",
|
|
1726
|
+
"mdcontext context *.md # Summarize all markdown files",
|
|
1727
|
+
"mdcontext context -t 1000 *.md # Fit within 1000 token budget",
|
|
1728
|
+
"mdcontext context --brief *.md # Minimal output (headings only)",
|
|
1729
|
+
"mdcontext context --full doc.md # Include full content",
|
|
1730
|
+
"mdcontext context *.md | pbcopy # Copy to clipboard (macOS)",
|
|
1731
|
+
"",
|
|
1732
|
+
"# Section filtering:",
|
|
1733
|
+
"mdcontext context doc.md --sections # List available sections",
|
|
1734
|
+
'mdcontext context doc.md --section "Setup" # Extract by section name',
|
|
1735
|
+
'mdcontext context doc.md --section "2.1" # Extract by section number',
|
|
1736
|
+
'mdcontext context doc.md --section "API*" # Glob pattern matching',
|
|
1737
|
+
'mdcontext context doc.md --section "Config" --shallow # Top-level only'
|
|
1738
|
+
],
|
|
1739
|
+
options: [
|
|
1740
|
+
{
|
|
1741
|
+
name: "-t, --tokens <n>",
|
|
1742
|
+
description: "Token budget for output (default: 2000)"
|
|
1743
|
+
},
|
|
1744
|
+
{
|
|
1745
|
+
name: "--brief",
|
|
1746
|
+
description: "Minimal output (headings and key points only)"
|
|
1747
|
+
},
|
|
1748
|
+
{
|
|
1749
|
+
name: "--full",
|
|
1750
|
+
description: "Include full content (no summarization)"
|
|
1751
|
+
},
|
|
1752
|
+
{
|
|
1753
|
+
name: "--section <name>",
|
|
1754
|
+
description: "Extract specific section by name, number, or glob pattern"
|
|
1755
|
+
},
|
|
1756
|
+
{
|
|
1757
|
+
name: "--sections",
|
|
1758
|
+
description: "List available sections with numbers and token counts"
|
|
1759
|
+
},
|
|
1760
|
+
{
|
|
1761
|
+
name: "--shallow",
|
|
1762
|
+
description: "Exclude nested subsections when using --section"
|
|
1763
|
+
},
|
|
1764
|
+
{ name: "--json", description: "Output as JSON" },
|
|
1765
|
+
{ name: "--pretty", description: "Pretty-print JSON output" }
|
|
1766
|
+
],
|
|
1767
|
+
notes: [
|
|
1768
|
+
"Token budget controls how much content is included.",
|
|
1769
|
+
"Lower tokens = more aggressive summarization.",
|
|
1770
|
+
"Output is formatted for direct use in LLM prompts.",
|
|
1771
|
+
"Use --sections to discover section names before filtering."
|
|
1772
|
+
]
|
|
1773
|
+
},
|
|
1774
|
+
tree: {
|
|
1775
|
+
description: "Show file tree or document outline",
|
|
1776
|
+
usage: "mdcontext tree [path] [options]",
|
|
1777
|
+
examples: [
|
|
1778
|
+
"mdcontext tree # List markdown files in current dir",
|
|
1779
|
+
"mdcontext tree docs/ # List files in specific directory",
|
|
1780
|
+
"mdcontext tree README.md # Show document outline (headings)",
|
|
1781
|
+
"mdcontext tree doc.md --json # Outline as JSON"
|
|
1782
|
+
],
|
|
1783
|
+
options: [
|
|
1784
|
+
{ name: "--json", description: "Output as JSON" },
|
|
1785
|
+
{ name: "--pretty", description: "Pretty-print JSON output" }
|
|
1786
|
+
],
|
|
1787
|
+
notes: [
|
|
1788
|
+
"Pass a directory to list markdown files.",
|
|
1789
|
+
"Pass a file to show its heading structure with token counts."
|
|
1790
|
+
]
|
|
1791
|
+
},
|
|
1792
|
+
links: {
|
|
1793
|
+
description: "Show what a file links to (outgoing links)",
|
|
1794
|
+
usage: "mdcontext links <file> [options]",
|
|
1795
|
+
examples: [
|
|
1796
|
+
"mdcontext links README.md # Show outgoing links",
|
|
1797
|
+
"mdcontext links doc.md --json # Output as JSON",
|
|
1798
|
+
"mdcontext links doc.md -r docs/ # Resolve links relative to docs/"
|
|
1799
|
+
],
|
|
1800
|
+
options: [
|
|
1801
|
+
{
|
|
1802
|
+
name: "-r, --root <dir>",
|
|
1803
|
+
description: "Root directory for resolving relative links"
|
|
1804
|
+
},
|
|
1805
|
+
{ name: "--json", description: "Output as JSON" },
|
|
1806
|
+
{ name: "--pretty", description: "Pretty-print JSON output" }
|
|
1807
|
+
]
|
|
1808
|
+
},
|
|
1809
|
+
backlinks: {
|
|
1810
|
+
description: "Show what links to a file (incoming links)",
|
|
1811
|
+
usage: "mdcontext backlinks <file> [options]",
|
|
1812
|
+
examples: [
|
|
1813
|
+
"mdcontext backlinks api.md # What links to api.md?",
|
|
1814
|
+
"mdcontext backlinks doc.md --json # Output as JSON",
|
|
1815
|
+
"mdcontext backlinks doc.md -r ./ # Resolve from current directory"
|
|
1816
|
+
],
|
|
1817
|
+
options: [
|
|
1818
|
+
{
|
|
1819
|
+
name: "-r, --root <dir>",
|
|
1820
|
+
description: "Root directory for resolving relative links"
|
|
1821
|
+
},
|
|
1822
|
+
{ name: "--json", description: "Output as JSON" },
|
|
1823
|
+
{ name: "--pretty", description: "Pretty-print JSON output" }
|
|
1824
|
+
],
|
|
1825
|
+
notes: ['Requires index to exist. Run "mdcontext index" first.']
|
|
1826
|
+
},
|
|
1827
|
+
stats: {
|
|
1828
|
+
description: "Show index statistics",
|
|
1829
|
+
usage: "mdcontext stats [path] [options]",
|
|
1830
|
+
examples: [
|
|
1831
|
+
"mdcontext stats # Show stats for current directory",
|
|
1832
|
+
"mdcontext stats docs/ # Show stats for specific directory",
|
|
1833
|
+
"mdcontext stats --json # Output as JSON"
|
|
1834
|
+
],
|
|
1835
|
+
options: [
|
|
1836
|
+
{ name: "--json", description: "Output as JSON" },
|
|
1837
|
+
{ name: "--pretty", description: "Pretty-print JSON output" }
|
|
1838
|
+
],
|
|
1839
|
+
notes: ["Shows embedding count, dimensions, and cost if embeddings exist."]
|
|
1840
|
+
}
|
|
1841
|
+
};
|
|
1842
|
+
var showSubcommandHelp = (command) => {
|
|
1843
|
+
const help = helpContent[command];
|
|
1844
|
+
if (!help) {
|
|
1845
|
+
console.log(`Unknown command: ${command}`);
|
|
1846
|
+
console.log('Run "mdcontext --help" for available commands.');
|
|
1847
|
+
process.exit(1);
|
|
1848
|
+
}
|
|
1849
|
+
console.log(`
|
|
1850
|
+
\x1B[1mmdcontext ${command}\x1B[0m - ${help.description}`);
|
|
1851
|
+
console.log(`
|
|
1852
|
+
\x1B[33mUSAGE\x1B[0m`);
|
|
1853
|
+
console.log(` ${help.usage}`);
|
|
1854
|
+
console.log(`
|
|
1855
|
+
\x1B[33mEXAMPLES\x1B[0m`);
|
|
1856
|
+
for (const example of help.examples) {
|
|
1857
|
+
console.log(` ${example}`);
|
|
1858
|
+
}
|
|
1859
|
+
console.log(`
|
|
1860
|
+
\x1B[33mOPTIONS\x1B[0m`);
|
|
1861
|
+
for (const opt of help.options) {
|
|
1862
|
+
const paddedName = opt.name.padEnd(24);
|
|
1863
|
+
console.log(` ${paddedName}${opt.description}`);
|
|
1864
|
+
}
|
|
1865
|
+
if (help.notes && help.notes.length > 0) {
|
|
1866
|
+
console.log(`
|
|
1867
|
+
\x1B[33mNOTES\x1B[0m`);
|
|
1868
|
+
for (const note of help.notes) {
|
|
1869
|
+
console.log(` ${note}`);
|
|
1870
|
+
}
|
|
1871
|
+
}
|
|
1872
|
+
console.log("");
|
|
1873
|
+
};
|
|
1874
|
+
var showMainHelp = () => {
|
|
1875
|
+
const help = `
|
|
1876
|
+
\x1B[1mmdcontext\x1B[0m - Token-efficient markdown analysis for LLMs
|
|
1877
|
+
|
|
1878
|
+
\x1B[33mCOMMANDS\x1B[0m
|
|
1879
|
+
index [path] Index markdown files (default: .)
|
|
1880
|
+
search <query> [path] Search by meaning or structure
|
|
1881
|
+
context <files>... Get LLM-ready summary
|
|
1882
|
+
tree [path] Show files or document outline
|
|
1883
|
+
links <file> Show outgoing links
|
|
1884
|
+
backlinks <file> Show incoming links
|
|
1885
|
+
stats [path] Index statistics
|
|
1886
|
+
|
|
1887
|
+
\x1B[33mEXAMPLES\x1B[0m
|
|
1888
|
+
mdcontext tree # List all markdown files
|
|
1889
|
+
mdcontext tree README.md # Show document outline
|
|
1890
|
+
mdcontext index # Index current directory
|
|
1891
|
+
mdcontext index --embed # Index with semantic embeddings
|
|
1892
|
+
mdcontext search "auth" # Simple term search
|
|
1893
|
+
mdcontext search "auth AND deploy" # Boolean AND (both required)
|
|
1894
|
+
mdcontext search "error OR bug" # Boolean OR (either matches)
|
|
1895
|
+
mdcontext search '"exact phrase"' # Quoted phrase (exact match)
|
|
1896
|
+
mdcontext search "how to deploy" # Semantic search (if embeddings exist)
|
|
1897
|
+
mdcontext context README.md # Summarize a file
|
|
1898
|
+
mdcontext context *.md -t 2000 # Multi-file with token budget
|
|
1899
|
+
|
|
1900
|
+
\x1B[33mWORKFLOWS\x1B[0m
|
|
1901
|
+
\x1B[2m# Quick context for LLM\x1B[0m
|
|
1902
|
+
mdcontext context README.md docs/*.md | pbcopy
|
|
1903
|
+
|
|
1904
|
+
\x1B[2m# Find relevant documentation\x1B[0m
|
|
1905
|
+
mdcontext search "error handling"
|
|
1906
|
+
|
|
1907
|
+
\x1B[2m# Complex queries with boolean operators\x1B[0m
|
|
1908
|
+
mdcontext search "auth AND (error OR exception) NOT test"
|
|
1909
|
+
|
|
1910
|
+
\x1B[2m# Explore a new codebase\x1B[0m
|
|
1911
|
+
mdcontext tree && mdcontext stats
|
|
1912
|
+
|
|
1913
|
+
\x1B[2m# Build semantic search\x1B[0m
|
|
1914
|
+
mdcontext index --embed && mdcontext search "authentication flow"
|
|
1915
|
+
|
|
1916
|
+
\x1B[33mGLOBAL OPTIONS\x1B[0m
|
|
1917
|
+
--json Output as JSON
|
|
1918
|
+
--pretty Pretty-print JSON
|
|
1919
|
+
--help, -h Show help
|
|
1920
|
+
--version, -v Show version
|
|
1921
|
+
|
|
1922
|
+
Run \x1B[36mmdcontext <command> --help\x1B[0m for command-specific options.
|
|
1923
|
+
`;
|
|
1924
|
+
console.log(help);
|
|
1925
|
+
};
|
|
1926
|
+
var checkSubcommandHelp = () => {
|
|
1927
|
+
const args = process.argv.slice(2);
|
|
1928
|
+
if (args.length < 2) return false;
|
|
1929
|
+
const command = args[0];
|
|
1930
|
+
const hasHelpFlag = args.includes("--help") || args.includes("-h");
|
|
1931
|
+
if (hasHelpFlag && command && helpContent[command]) {
|
|
1932
|
+
showSubcommandHelp(command);
|
|
1933
|
+
process.exit(0);
|
|
1934
|
+
}
|
|
1935
|
+
return false;
|
|
1936
|
+
};
|
|
1937
|
+
var shouldShowMainHelp = () => {
|
|
1938
|
+
const args = process.argv.slice(2);
|
|
1939
|
+
const showHelp = args.length === 0 || args.includes("--help") || args.includes("-h") || args.length === 1 && args[0] === "help";
|
|
1940
|
+
return showHelp && !args.some((a) => !a.startsWith("-") && a !== "help");
|
|
1941
|
+
};
|
|
1942
|
+
|
|
1943
|
+
// src/cli/main.ts
|
|
1944
|
+
var mainCommand = Command8.make("mdcontext").pipe(
|
|
1945
|
+
Command8.withDescription("Token-efficient markdown analysis for LLMs"),
|
|
1946
|
+
Command8.withSubcommands([
|
|
1947
|
+
indexCommand,
|
|
1948
|
+
searchCommand,
|
|
1949
|
+
contextCommand,
|
|
1950
|
+
treeCommand,
|
|
1951
|
+
linksCommand,
|
|
1952
|
+
backlinksCommand,
|
|
1953
|
+
statsCommand
|
|
1954
|
+
])
|
|
1955
|
+
);
|
|
1956
|
+
var cli = Command8.run(mainCommand, {
|
|
1957
|
+
name: "mdcontext",
|
|
1958
|
+
version: "0.1.0"
|
|
1959
|
+
});
|
|
1960
|
+
var cliConfigLayer = CliConfig.layer({
|
|
1961
|
+
showBuiltIns: false
|
|
1962
|
+
});
|
|
1963
|
+
var formatCliError = (error) => {
|
|
1964
|
+
if (error && typeof error === "object") {
|
|
1965
|
+
const err = error;
|
|
1966
|
+
if (err._tag === "ValidationError" && err.error) {
|
|
1967
|
+
const validationError = err.error;
|
|
1968
|
+
if (validationError._tag === "Paragraph" && validationError.value) {
|
|
1969
|
+
const paragraph = validationError.value;
|
|
1970
|
+
if (paragraph._tag === "Text" && typeof paragraph.value === "string") {
|
|
1971
|
+
return paragraph.value;
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1974
|
+
}
|
|
1975
|
+
if (err._tag === "MissingValue" && err.error) {
|
|
1976
|
+
const missingError = err.error;
|
|
1977
|
+
if (missingError._tag === "Paragraph" && missingError.value) {
|
|
1978
|
+
const paragraph = missingError.value;
|
|
1979
|
+
if (paragraph._tag === "Text" && typeof paragraph.value === "string") {
|
|
1980
|
+
return paragraph.value;
|
|
1981
|
+
}
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
}
|
|
1985
|
+
return String(error);
|
|
1986
|
+
};
|
|
1987
|
+
var isValidationError = (error) => {
|
|
1988
|
+
if (error && typeof error === "object") {
|
|
1989
|
+
const err = error;
|
|
1990
|
+
return err._tag === "ValidationError" || err._tag === "MissingValue" || err._tag === "InvalidValue";
|
|
1991
|
+
}
|
|
1992
|
+
return false;
|
|
1993
|
+
};
|
|
1994
|
+
checkSubcommandHelp();
|
|
1995
|
+
if (shouldShowMainHelp()) {
|
|
1996
|
+
showMainHelp();
|
|
1997
|
+
process.exit(0);
|
|
1998
|
+
}
|
|
1999
|
+
var processedArgv = preprocessArgv(process.argv);
|
|
2000
|
+
Effect8.suspend(() => cli(processedArgv)).pipe(
|
|
2001
|
+
Effect8.provide(Layer.merge(NodeContext.layer, cliConfigLayer)),
|
|
2002
|
+
Effect8.catchAll(
|
|
2003
|
+
(error) => Effect8.sync(() => {
|
|
2004
|
+
if (isValidationError(error)) {
|
|
2005
|
+
const message = formatCliError(error);
|
|
2006
|
+
console.error(`
|
|
2007
|
+
Error: ${message}`);
|
|
2008
|
+
console.error('\nRun "mdcontext --help" for usage information.');
|
|
2009
|
+
process.exit(1);
|
|
2010
|
+
}
|
|
2011
|
+
throw error;
|
|
2012
|
+
})
|
|
2013
|
+
),
|
|
2014
|
+
NodeRuntime.runMain
|
|
2015
|
+
);
|