mdcontext 0.0.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.changeset/README.md +28 -0
- package/.changeset/config.json +11 -0
- package/.claude/settings.local.json +25 -0
- package/.github/workflows/ci.yml +83 -0
- package/.github/workflows/claude-code-review.yml +44 -0
- package/.github/workflows/claude.yml +85 -0
- package/.github/workflows/release.yml +113 -0
- package/.tldrignore +112 -0
- package/BACKLOG.md +338 -0
- package/CONTRIBUTING.md +186 -0
- package/NOTES/NOTES +44 -0
- package/README.md +434 -11
- package/biome.json +36 -0
- package/cspell.config.yaml +14 -0
- package/dist/chunk-23UPXDNL.js +3044 -0
- package/dist/chunk-2W7MO2DL.js +1366 -0
- package/dist/chunk-3NUAZGMA.js +1689 -0
- package/dist/chunk-7TOWB2XB.js +366 -0
- package/dist/chunk-7XOTOADQ.js +3065 -0
- package/dist/chunk-AH2PDM2K.js +3042 -0
- package/dist/chunk-BNXWSZ63.js +3742 -0
- package/dist/chunk-BTL5DJVU.js +3222 -0
- package/dist/chunk-HDHYG7E4.js +104 -0
- package/dist/chunk-HLR4KZBP.js +3234 -0
- package/dist/chunk-IP3FRFEB.js +1045 -0
- package/dist/chunk-KHU56VDO.js +3042 -0
- package/dist/chunk-KRYIFLQR.js +88 -0
- package/dist/chunk-LBSDNLEM.js +287 -0
- package/dist/chunk-MNTQ7HCP.js +2643 -0
- package/dist/chunk-MUJELQQ6.js +1387 -0
- package/dist/chunk-MXJGMSLV.js +2199 -0
- package/dist/chunk-N6QJGC3Z.js +2636 -0
- package/dist/chunk-OBELGBPM.js +1713 -0
- package/dist/chunk-OT7R5XTA.js +3192 -0
- package/dist/chunk-P7X4RA2T.js +106 -0
- package/dist/chunk-PIDUQNC2.js +3185 -0
- package/dist/chunk-POGCDIH4.js +3187 -0
- package/dist/chunk-PSIEOQGZ.js +3043 -0
- package/dist/chunk-PVRT3IHA.js +3238 -0
- package/dist/chunk-QNN4TT23.js +1430 -0
- package/dist/chunk-RE3R45RJ.js +3042 -0
- package/dist/chunk-S7E6TFX6.js +803 -0
- package/dist/chunk-SG6GLU4U.js +1378 -0
- package/dist/chunk-SJCDV2ST.js +274 -0
- package/dist/chunk-SYE5XLF3.js +104 -0
- package/dist/chunk-T5VLYBZD.js +103 -0
- package/dist/chunk-TOQB7VWU.js +3238 -0
- package/dist/chunk-VFNMZ4ZQ.js +3228 -0
- package/dist/chunk-VVTGZNBT.js +1629 -0
- package/dist/chunk-W7Q4RFEV.js +104 -0
- package/dist/chunk-XTYYVRLO.js +3190 -0
- package/dist/chunk-Y6MDYVJD.js +3063 -0
- package/dist/cli/main.d.ts +1 -0
- package/dist/cli/main.js +5458 -0
- package/dist/index.d.ts +653 -0
- package/dist/index.js +79 -0
- package/dist/mcp/server.d.ts +1 -0
- package/dist/mcp/server.js +472 -0
- package/dist/schema-BAWSG7KY.js +22 -0
- package/dist/schema-E3QUPL26.js +20 -0
- package/dist/schema-EHL7WUT6.js +20 -0
- package/docs/019-USAGE.md +625 -0
- package/docs/020-current-implementation.md +364 -0
- package/docs/021-DOGFOODING-FINDINGS.md +175 -0
- package/docs/BACKLOG.md +80 -0
- package/docs/CONFIG.md +1123 -0
- package/docs/DESIGN.md +439 -0
- package/docs/ERRORS.md +383 -0
- package/docs/PROJECT.md +88 -0
- package/docs/ROADMAP.md +407 -0
- package/docs/summarization.md +320 -0
- package/docs/test-links.md +9 -0
- package/justfile +40 -0
- package/package.json +74 -9
- package/pnpm-workspace.yaml +5 -0
- package/research/INDEX.md +315 -0
- package/research/code-review/README.md +90 -0
- package/research/code-review/cli-error-handling-review.md +979 -0
- package/research/code-review/code-review-validation-report.md +464 -0
- package/research/code-review/main-ts-review.md +1128 -0
- package/research/config-analysis/01-current-implementation.md +470 -0
- package/research/config-analysis/02-strategy-recommendation.md +428 -0
- package/research/config-analysis/03-task-candidates.md +715 -0
- package/research/config-analysis/033-research-configuration-management.md +828 -0
- package/research/config-analysis/034-research-effect-cli-config.md +1504 -0
- package/research/config-analysis/04-consolidated-task-candidates.md +277 -0
- package/research/config-docs/SUMMARY.md +357 -0
- package/research/config-docs/TEST-RESULTS.md +776 -0
- package/research/config-docs/TODO.md +542 -0
- package/research/config-docs/analysis.md +744 -0
- package/research/config-docs/fix-validation.md +502 -0
- package/research/config-docs/help-audit.md +264 -0
- package/research/config-docs/help-system-analysis.md +890 -0
- package/research/dogfood/consolidated-tool-evaluation.md +373 -0
- package/research/dogfood/strategy-a/a-synthesis.md +184 -0
- package/research/dogfood/strategy-a/a1-docs.md +226 -0
- package/research/dogfood/strategy-a/a2-amorphic.md +156 -0
- package/research/dogfood/strategy-a/a3-llm.md +164 -0
- package/research/dogfood/strategy-b/b-synthesis.md +228 -0
- package/research/dogfood/strategy-b/b1-architecture.md +207 -0
- package/research/dogfood/strategy-b/b2-gaps.md +258 -0
- package/research/dogfood/strategy-b/b3-workflows.md +250 -0
- package/research/dogfood/strategy-c/c-synthesis.md +451 -0
- package/research/dogfood/strategy-c/c1-explorer.md +192 -0
- package/research/dogfood/strategy-c/c2-diver-memory.md +145 -0
- package/research/dogfood/strategy-c/c3-diver-control.md +148 -0
- package/research/dogfood/strategy-c/c4-diver-failure.md +151 -0
- package/research/dogfood/strategy-c/c5-diver-execution.md +221 -0
- package/research/dogfood/strategy-c/c6-diver-org.md +221 -0
- package/research/effect-cli-error-handling.md +845 -0
- package/research/effect-errors-as-values.md +943 -0
- package/research/errors-task-analysis/00-consolidated-tasks.md +207 -0
- package/research/errors-task-analysis/cli-commands-analysis.md +909 -0
- package/research/errors-task-analysis/embeddings-analysis.md +709 -0
- package/research/errors-task-analysis/index-search-analysis.md +812 -0
- package/research/frontmatter/COMMENTS-ARE-SKIPPED.md +149 -0
- package/research/frontmatter/LLM-CODE-NAVIGATION.md +276 -0
- package/research/issue-review.md +603 -0
- package/research/llm-summarization/agent-cli-tools-2026.md +1082 -0
- package/research/llm-summarization/alternative-providers-2026.md +1428 -0
- package/research/llm-summarization/anthropic-2026.md +367 -0
- package/research/llm-summarization/claude-cli-integration.md +1706 -0
- package/research/llm-summarization/cli-integration-patterns.md +3155 -0
- package/research/llm-summarization/openai-2026.md +473 -0
- package/research/llm-summarization/openai-compatible-providers-2026.md +1022 -0
- package/research/llm-summarization/opencode-cli-integration.md +1552 -0
- package/research/llm-summarization/prompt-engineering-2026.md +1426 -0
- package/research/llm-summarization/prototype-results.md +56 -0
- package/research/llm-summarization/provider-switching-patterns-2026.md +2153 -0
- package/research/llm-summarization/typescript-llm-libraries-2026.md +2436 -0
- package/research/mdcontext-error-analysis.md +521 -0
- package/research/mdcontext-pudding/00-EXECUTIVE-SUMMARY.md +282 -0
- package/research/mdcontext-pudding/01-index-embed.md +956 -0
- package/research/mdcontext-pudding/02-search-COMMANDS.md +142 -0
- package/research/mdcontext-pudding/02-search-SUMMARY.md +146 -0
- package/research/mdcontext-pudding/02-search.md +970 -0
- package/research/mdcontext-pudding/03-context.md +779 -0
- package/research/mdcontext-pudding/04-navigation-and-analytics.md +803 -0
- package/research/mdcontext-pudding/04-tree.md +704 -0
- package/research/mdcontext-pudding/05-config.md +1038 -0
- package/research/mdcontext-pudding/06-links-summary.txt +87 -0
- package/research/mdcontext-pudding/06-links.md +679 -0
- package/research/mdcontext-pudding/07-stats.md +693 -0
- package/research/mdcontext-pudding/BUG-FIX-PLAN.md +388 -0
- package/research/mdcontext-pudding/P0-BUG-VALIDATION.md +167 -0
- package/research/mdcontext-pudding/README.md +168 -0
- package/research/mdcontext-pudding/TESTING-SUMMARY.md +128 -0
- package/research/npm_publish/011-npm-workflow-research-agent2.md +792 -0
- package/research/npm_publish/012-npm-workflow-research-agent1.md +530 -0
- package/research/npm_publish/013-npm-workflow-research-agent3.md +722 -0
- package/research/npm_publish/014-npm-workflow-synthesis.md +556 -0
- package/research/npm_publish/031-npm-workflow-task-analysis.md +134 -0
- package/research/research-quality-review.md +834 -0
- package/research/semantic-search/002-research-embedding-models.md +490 -0
- package/research/semantic-search/003-research-rag-alternatives.md +523 -0
- package/research/semantic-search/004-research-vector-search.md +841 -0
- package/research/semantic-search/032-research-semantic-search.md +427 -0
- package/research/semantic-search/embedding-text-analysis.md +156 -0
- package/research/semantic-search/multi-word-failure-reproduction.md +171 -0
- package/research/semantic-search/query-processing-analysis.md +207 -0
- package/research/semantic-search/root-cause-and-solution.md +114 -0
- package/research/semantic-search/threshold-validation-report.md +69 -0
- package/research/semantic-search/vector-search-analysis.md +63 -0
- package/research/task-management-2026/00-synthesis-recommendations.md +295 -0
- package/research/task-management-2026/01-ai-workflow-tools.md +416 -0
- package/research/task-management-2026/02-agent-framework-patterns.md +476 -0
- package/research/task-management-2026/03-lightweight-file-based.md +567 -0
- package/research/task-management-2026/04-established-tools-ai-features.md +541 -0
- package/research/task-management-2026/linear/01-core-features-workflow.md +771 -0
- package/research/task-management-2026/linear/02-api-integrations.md +930 -0
- package/research/task-management-2026/linear/03-ai-features.md +368 -0
- package/research/task-management-2026/linear/04-pricing-setup.md +205 -0
- package/research/task-management-2026/linear/05-usage-patterns-best-practices.md +605 -0
- package/research/test-path-issues.md +276 -0
- package/review/ALP-76/1-error-type-design.md +962 -0
- package/review/ALP-76/2-error-handling-patterns.md +906 -0
- package/review/ALP-76/3-error-presentation.md +624 -0
- package/review/ALP-76/4-test-coverage.md +625 -0
- package/review/ALP-76/5-migration-completeness.md +440 -0
- package/review/ALP-76/6-effect-best-practices.md +755 -0
- package/scripts/apply-branch-protection.sh +47 -0
- package/scripts/branch-protection-templates.json +79 -0
- package/scripts/prototype-summarization.ts +346 -0
- package/scripts/rebuild-hnswlib.js +58 -0
- package/scripts/setup-branch-protection.sh +64 -0
- package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/active-provider.json +7 -0
- package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/bm25.json +541 -0
- package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/bm25.meta.json +5 -0
- package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/config.json +8 -0
- package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/embeddings/openai_text-embedding-3-small_512/vectors.bin +0 -0
- package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/embeddings/openai_text-embedding-3-small_512/vectors.meta.bin +0 -0
- package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/indexes/documents.json +60 -0
- package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/indexes/links.json +13 -0
- package/src/__tests__/fixtures/semantic-search/multi-word-corpus/.mdcontext/indexes/sections.json +1197 -0
- package/src/__tests__/fixtures/semantic-search/multi-word-corpus/configuration-management.md +99 -0
- package/src/__tests__/fixtures/semantic-search/multi-word-corpus/distributed-systems.md +92 -0
- package/src/__tests__/fixtures/semantic-search/multi-word-corpus/error-handling.md +78 -0
- package/src/__tests__/fixtures/semantic-search/multi-word-corpus/failure-automation.md +55 -0
- package/src/__tests__/fixtures/semantic-search/multi-word-corpus/job-context.md +69 -0
- package/src/__tests__/fixtures/semantic-search/multi-word-corpus/process-orchestration.md +99 -0
- package/src/cli/argv-preprocessor.test.ts +210 -0
- package/src/cli/argv-preprocessor.ts +202 -0
- package/src/cli/cli.test.ts +627 -0
- package/src/cli/commands/backlinks.ts +54 -0
- package/src/cli/commands/config-cmd.ts +642 -0
- package/src/cli/commands/context.ts +285 -0
- package/src/cli/commands/duplicates.ts +122 -0
- package/src/cli/commands/embeddings.ts +529 -0
- package/src/cli/commands/index-cmd.ts +480 -0
- package/src/cli/commands/index.ts +16 -0
- package/src/cli/commands/links.ts +52 -0
- package/src/cli/commands/search.ts +1281 -0
- package/src/cli/commands/stats.ts +149 -0
- package/src/cli/commands/tree.ts +128 -0
- package/src/cli/config-layer.ts +176 -0
- package/src/cli/error-handler.test.ts +235 -0
- package/src/cli/error-handler.ts +655 -0
- package/src/cli/flag-schemas.ts +341 -0
- package/src/cli/help.ts +588 -0
- package/src/cli/index.ts +9 -0
- package/src/cli/main.ts +435 -0
- package/src/cli/options.ts +41 -0
- package/src/cli/shared-error-handling.ts +199 -0
- package/src/cli/typo-suggester.test.ts +105 -0
- package/src/cli/typo-suggester.ts +130 -0
- package/src/cli/utils.ts +259 -0
- package/src/config/file-provider.test.ts +320 -0
- package/src/config/file-provider.ts +273 -0
- package/src/config/index.ts +72 -0
- package/src/config/integration.test.ts +667 -0
- package/src/config/precedence.test.ts +277 -0
- package/src/config/precedence.ts +451 -0
- package/src/config/schema.test.ts +414 -0
- package/src/config/schema.ts +603 -0
- package/src/config/service.test.ts +320 -0
- package/src/config/service.ts +243 -0
- package/src/config/testing.test.ts +264 -0
- package/src/config/testing.ts +110 -0
- package/src/core/index.ts +1 -0
- package/src/core/types.ts +113 -0
- package/src/duplicates/detector.test.ts +183 -0
- package/src/duplicates/detector.ts +414 -0
- package/src/duplicates/index.ts +18 -0
- package/src/embeddings/embedding-namespace.test.ts +300 -0
- package/src/embeddings/embedding-namespace.ts +947 -0
- package/src/embeddings/heading-boost.test.ts +222 -0
- package/src/embeddings/hnsw-build-options.test.ts +198 -0
- package/src/embeddings/hyde.test.ts +272 -0
- package/src/embeddings/hyde.ts +264 -0
- package/src/embeddings/index.ts +10 -0
- package/src/embeddings/openai-provider.ts +414 -0
- package/src/embeddings/pricing.json +22 -0
- package/src/embeddings/provider-constants.ts +204 -0
- package/src/embeddings/provider-errors.test.ts +967 -0
- package/src/embeddings/provider-errors.ts +565 -0
- package/src/embeddings/provider-factory.test.ts +240 -0
- package/src/embeddings/provider-factory.ts +225 -0
- package/src/embeddings/provider-integration.test.ts +788 -0
- package/src/embeddings/query-preprocessing.test.ts +187 -0
- package/src/embeddings/semantic-search-threshold.test.ts +508 -0
- package/src/embeddings/semantic-search.ts +1270 -0
- package/src/embeddings/types.ts +359 -0
- package/src/embeddings/vector-store.ts +708 -0
- package/src/embeddings/voyage-provider.ts +313 -0
- package/src/errors/errors.test.ts +845 -0
- package/src/errors/index.ts +533 -0
- package/src/index/ignore-patterns.test.ts +354 -0
- package/src/index/ignore-patterns.ts +305 -0
- package/src/index/index.ts +4 -0
- package/src/index/indexer.ts +684 -0
- package/src/index/storage.ts +260 -0
- package/src/index/types.ts +147 -0
- package/src/index/watcher.ts +189 -0
- package/src/index.ts +30 -0
- package/src/integration/search-keyword.test.ts +678 -0
- package/src/mcp/server.ts +612 -0
- package/src/parser/index.ts +1 -0
- package/src/parser/parser.test.ts +291 -0
- package/src/parser/parser.ts +394 -0
- package/src/parser/section-filter.test.ts +277 -0
- package/src/parser/section-filter.ts +392 -0
- package/src/search/__tests__/hybrid-search.test.ts +650 -0
- package/src/search/bm25-store.ts +366 -0
- package/src/search/cross-encoder.test.ts +253 -0
- package/src/search/cross-encoder.ts +406 -0
- package/src/search/fuzzy-search.test.ts +419 -0
- package/src/search/fuzzy-search.ts +273 -0
- package/src/search/hybrid-search.ts +448 -0
- package/src/search/path-matcher.test.ts +276 -0
- package/src/search/path-matcher.ts +33 -0
- package/src/search/query-parser.test.ts +260 -0
- package/src/search/query-parser.ts +319 -0
- package/src/search/searcher.test.ts +280 -0
- package/src/search/searcher.ts +724 -0
- package/src/search/wink-bm25.d.ts +30 -0
- package/src/summarization/cli-providers/claude.ts +202 -0
- package/src/summarization/cli-providers/detection.test.ts +273 -0
- package/src/summarization/cli-providers/detection.ts +118 -0
- package/src/summarization/cli-providers/index.ts +8 -0
- package/src/summarization/cost.test.ts +139 -0
- package/src/summarization/cost.ts +102 -0
- package/src/summarization/error-handler.test.ts +127 -0
- package/src/summarization/error-handler.ts +111 -0
- package/src/summarization/index.ts +102 -0
- package/src/summarization/pipeline.test.ts +498 -0
- package/src/summarization/pipeline.ts +231 -0
- package/src/summarization/prompts.test.ts +269 -0
- package/src/summarization/prompts.ts +133 -0
- package/src/summarization/provider-factory.test.ts +396 -0
- package/src/summarization/provider-factory.ts +178 -0
- package/src/summarization/types.ts +184 -0
- package/src/summarize/budget-bugs.test.ts +620 -0
- package/src/summarize/formatters.ts +419 -0
- package/src/summarize/index.ts +20 -0
- package/src/summarize/summarizer.test.ts +275 -0
- package/src/summarize/summarizer.ts +597 -0
- package/src/summarize/verify-bugs.test.ts +238 -0
- package/src/types/huggingface-transformers.d.ts +66 -0
- package/src/utils/index.ts +1 -0
- package/src/utils/tokens.test.ts +142 -0
- package/src/utils/tokens.ts +186 -0
- package/tests/fixtures/cli/.mdcontext/active-provider.json +7 -0
- package/tests/fixtures/cli/.mdcontext/config.json +8 -0
- package/tests/fixtures/cli/.mdcontext/embeddings/openai_text-embedding-3-small_512/vectors.bin +0 -0
- package/tests/fixtures/cli/.mdcontext/embeddings/openai_text-embedding-3-small_512/vectors.meta.bin +0 -0
- package/tests/fixtures/cli/.mdcontext/indexes/documents.json +33 -0
- package/tests/fixtures/cli/.mdcontext/indexes/links.json +12 -0
- package/tests/fixtures/cli/.mdcontext/indexes/sections.json +247 -0
- package/tests/fixtures/cli/README.md +9 -0
- package/tests/fixtures/cli/api-reference.md +11 -0
- package/tests/fixtures/cli/getting-started.md +11 -0
- package/tests/integration/embed-index.test.ts +712 -0
- package/tests/integration/search-context.test.ts +469 -0
- package/tests/integration/search-semantic.test.ts +522 -0
- package/tsconfig.json +26 -0
- package/vitest.config.ts +16 -0
- package/vitest.setup.ts +12 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,653 @@
|
|
|
1
|
+
import { Config, Option, Effect } from 'effect';
|
|
2
|
+
import * as effect_Cause from 'effect/Cause';
|
|
3
|
+
import * as effect_Types from 'effect/Types';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Effect Config Schema Module
|
|
7
|
+
*
|
|
8
|
+
* Defines all configuration options using Effect's Config combinators.
|
|
9
|
+
* This provides native integration with Effect's ConfigProvider system,
|
|
10
|
+
* enabling compile-time tracking of config requirements through the type system.
|
|
11
|
+
*
|
|
12
|
+
* ## Configuration Structure
|
|
13
|
+
*
|
|
14
|
+
* - index: Settings for the indexing process
|
|
15
|
+
* - search: Settings for search operations
|
|
16
|
+
* - embeddings: Settings for semantic embeddings (OpenAI)
|
|
17
|
+
* - output: Settings for CLI output formatting
|
|
18
|
+
* - paths: Settings for file paths and directories
|
|
19
|
+
*
|
|
20
|
+
* ## Usage
|
|
21
|
+
*
|
|
22
|
+
* ```typescript
|
|
23
|
+
* import { MdContextConfig, IndexConfig } from './config/schema.js'
|
|
24
|
+
* import { Effect } from 'effect'
|
|
25
|
+
*
|
|
26
|
+
* const program = Effect.gen(function* () {
|
|
27
|
+
* const config = yield* MdContextConfig
|
|
28
|
+
* console.log(`Max depth: ${config.index.maxDepth}`)
|
|
29
|
+
* })
|
|
30
|
+
* ```
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Complete mdcontext configuration
|
|
35
|
+
*
|
|
36
|
+
* Combines all configuration sections into a single nested config.
|
|
37
|
+
* Use with Effect's ConfigProvider to load from various sources.
|
|
38
|
+
*/
|
|
39
|
+
declare const MdContextConfig: Config.Config<{
|
|
40
|
+
index: {
|
|
41
|
+
maxDepth: number;
|
|
42
|
+
excludePatterns: string[] | readonly ["node_modules", ".git", "dist", "build"];
|
|
43
|
+
fileExtensions: string[] | readonly [".md", ".mdx"];
|
|
44
|
+
followSymlinks: boolean;
|
|
45
|
+
indexDir: string;
|
|
46
|
+
};
|
|
47
|
+
search: {
|
|
48
|
+
defaultLimit: number;
|
|
49
|
+
maxLimit: number;
|
|
50
|
+
minSimilarity: number;
|
|
51
|
+
includeSnippets: boolean;
|
|
52
|
+
snippetLength: number;
|
|
53
|
+
autoIndexThreshold: number;
|
|
54
|
+
};
|
|
55
|
+
embeddings: {
|
|
56
|
+
provider: "openai" | "ollama" | "lm-studio" | "openrouter" | "voyage";
|
|
57
|
+
baseURL: Option.Option<string>;
|
|
58
|
+
model: string;
|
|
59
|
+
dimensions: number;
|
|
60
|
+
batchSize: number;
|
|
61
|
+
maxRetries: number;
|
|
62
|
+
retryDelayMs: number;
|
|
63
|
+
timeoutMs: number;
|
|
64
|
+
apiKey: Option.Option<string>;
|
|
65
|
+
hnswM: number;
|
|
66
|
+
hnswEfConstruction: number;
|
|
67
|
+
};
|
|
68
|
+
summarization: {
|
|
69
|
+
briefTokenBudget: number;
|
|
70
|
+
summaryTokenBudget: number;
|
|
71
|
+
compressionRatio: number;
|
|
72
|
+
minSectionTokens: number;
|
|
73
|
+
maxTopics: number;
|
|
74
|
+
minPartialBudget: number;
|
|
75
|
+
};
|
|
76
|
+
aiSummarization: {
|
|
77
|
+
mode: "cli" | "api";
|
|
78
|
+
provider: "openai" | "claude" | "copilot" | "cline" | "aider" | "opencode" | "amp" | "deepseek" | "anthropic" | "gemini" | "qwen";
|
|
79
|
+
model: Option.Option<string>;
|
|
80
|
+
stream: boolean;
|
|
81
|
+
baseURL: Option.Option<string>;
|
|
82
|
+
apiKey: Option.Option<string>;
|
|
83
|
+
};
|
|
84
|
+
output: {
|
|
85
|
+
format: "text" | "json";
|
|
86
|
+
color: boolean;
|
|
87
|
+
prettyJson: boolean;
|
|
88
|
+
verbose: boolean;
|
|
89
|
+
debug: boolean;
|
|
90
|
+
};
|
|
91
|
+
paths: {
|
|
92
|
+
root: Option.Option<string>;
|
|
93
|
+
configFile: Option.Option<string>;
|
|
94
|
+
cacheDir: string;
|
|
95
|
+
};
|
|
96
|
+
}>;
|
|
97
|
+
/**
|
|
98
|
+
* Inferred type for the complete configuration
|
|
99
|
+
*/
|
|
100
|
+
type MdContextConfig = Config.Config.Success<typeof MdContextConfig>;
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Deeply partial type for MdContextConfig
|
|
104
|
+
*/
|
|
105
|
+
type PartialMdContextConfig = {
|
|
106
|
+
[K in keyof MdContextConfig]?: Partial<MdContextConfig[K]>;
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Core data types for mdcontext
|
|
111
|
+
* Based on DESIGN.md specifications
|
|
112
|
+
*/
|
|
113
|
+
interface MdDocument {
|
|
114
|
+
readonly id: string;
|
|
115
|
+
readonly path: string;
|
|
116
|
+
readonly title: string;
|
|
117
|
+
readonly frontmatter: Record<string, unknown>;
|
|
118
|
+
readonly sections: readonly MdSection[];
|
|
119
|
+
readonly links: readonly MdLink[];
|
|
120
|
+
readonly codeBlocks: readonly MdCodeBlock[];
|
|
121
|
+
readonly metadata: DocumentMetadata;
|
|
122
|
+
}
|
|
123
|
+
interface DocumentMetadata {
|
|
124
|
+
readonly wordCount: number;
|
|
125
|
+
readonly tokenCount: number;
|
|
126
|
+
readonly headingCount: number;
|
|
127
|
+
readonly linkCount: number;
|
|
128
|
+
readonly codeBlockCount: number;
|
|
129
|
+
readonly lastModified: Date;
|
|
130
|
+
readonly indexedAt: Date;
|
|
131
|
+
}
|
|
132
|
+
type HeadingLevel = 1 | 2 | 3 | 4 | 5 | 6;
|
|
133
|
+
interface MdSection {
|
|
134
|
+
readonly id: string;
|
|
135
|
+
readonly heading: string;
|
|
136
|
+
readonly level: HeadingLevel;
|
|
137
|
+
readonly content: string;
|
|
138
|
+
readonly plainText: string;
|
|
139
|
+
readonly startLine: number;
|
|
140
|
+
readonly endLine: number;
|
|
141
|
+
readonly children: readonly MdSection[];
|
|
142
|
+
readonly metadata: SectionMetadata;
|
|
143
|
+
}
|
|
144
|
+
interface SectionMetadata {
|
|
145
|
+
readonly wordCount: number;
|
|
146
|
+
readonly tokenCount: number;
|
|
147
|
+
readonly hasCode: boolean;
|
|
148
|
+
readonly hasList: boolean;
|
|
149
|
+
readonly hasTable: boolean;
|
|
150
|
+
}
|
|
151
|
+
type LinkType = 'internal' | 'external' | 'image';
|
|
152
|
+
interface MdLink {
|
|
153
|
+
readonly type: LinkType;
|
|
154
|
+
readonly href: string;
|
|
155
|
+
readonly text: string;
|
|
156
|
+
readonly sectionId: string;
|
|
157
|
+
readonly line: number;
|
|
158
|
+
}
|
|
159
|
+
interface MdCodeBlock {
|
|
160
|
+
readonly language: string | null;
|
|
161
|
+
readonly content: string;
|
|
162
|
+
readonly sectionId: string;
|
|
163
|
+
readonly startLine: number;
|
|
164
|
+
readonly endLine: number;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Parse error from markdown parsing
|
|
168
|
+
*
|
|
169
|
+
* Note: This interface is used by parser.ts. For the TaggedError version
|
|
170
|
+
* that works with Effect's error handling, see src/errors/index.ts ParseError.
|
|
171
|
+
*/
|
|
172
|
+
interface ParseError {
|
|
173
|
+
readonly _tag: 'ParseError';
|
|
174
|
+
readonly message: string;
|
|
175
|
+
readonly line?: number | undefined;
|
|
176
|
+
readonly column?: number | undefined;
|
|
177
|
+
}
|
|
178
|
+
declare const ParseError: (message: string, line?: number, column?: number) => ParseError;
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Centralized error types for mdcontext
|
|
182
|
+
*
|
|
183
|
+
* This module defines all domain errors using Effect's Data.TaggedError pattern.
|
|
184
|
+
* Each error has a unique `_tag` discriminant that enables:
|
|
185
|
+
* - Exhaustive error handling with `catchTag` / `catchTags`
|
|
186
|
+
* - Type-safe error composition in Effect pipelines
|
|
187
|
+
* - Pattern matching for user-friendly error messages at CLI boundary
|
|
188
|
+
*
|
|
189
|
+
* ## Error Message Convention
|
|
190
|
+
*
|
|
191
|
+
* The `message` field in errors should contain **technical details** from the
|
|
192
|
+
* underlying operation, NOT user-facing formatted messages.
|
|
193
|
+
*
|
|
194
|
+
* **Good (technical):**
|
|
195
|
+
* ```typescript
|
|
196
|
+
* new FileReadError({
|
|
197
|
+
* path: '/path/to/file',
|
|
198
|
+
* message: e.message, // e.g., "ENOENT: no such file or directory"
|
|
199
|
+
* cause: e,
|
|
200
|
+
* })
|
|
201
|
+
* ```
|
|
202
|
+
*
|
|
203
|
+
* **Bad (user-facing):**
|
|
204
|
+
* ```typescript
|
|
205
|
+
* new FileReadError({
|
|
206
|
+
* path: '/path/to/file',
|
|
207
|
+
* message: 'Cannot read file. Please check permissions.', // NO!
|
|
208
|
+
* })
|
|
209
|
+
* ```
|
|
210
|
+
*
|
|
211
|
+
* User-friendly messages are generated at the CLI boundary by the error handler
|
|
212
|
+
* in `src/cli/error-handler.ts`. This separation enables:
|
|
213
|
+
* - i18n/localization in the future
|
|
214
|
+
* - Testing error data without string matching
|
|
215
|
+
* - Consistent formatting in one place
|
|
216
|
+
*
|
|
217
|
+
* ## Error Taxonomy
|
|
218
|
+
*
|
|
219
|
+
* - File System: FileReadError, FileWriteError, DirectoryCreateError, DirectoryWalkError
|
|
220
|
+
* - Parsing: ParseError (for markdown parsing failures)
|
|
221
|
+
* - API: ApiKeyMissingError, ApiKeyInvalidError
|
|
222
|
+
* - Embeddings: EmbeddingError (rate limits, quota, network failures)
|
|
223
|
+
* - Index: IndexNotFoundError, IndexCorruptedError, IndexBuildError
|
|
224
|
+
* - Search: DocumentNotFoundError, EmbeddingsNotFoundError
|
|
225
|
+
* - Vector Store: VectorStoreError
|
|
226
|
+
* - Config: ConfigError
|
|
227
|
+
* - CLI: CliValidationError
|
|
228
|
+
*
|
|
229
|
+
* ## Usage
|
|
230
|
+
*
|
|
231
|
+
* ```typescript
|
|
232
|
+
* import { FileReadError, ApiKeyMissingError } from './errors/index.js'
|
|
233
|
+
* import { Effect } from 'effect'
|
|
234
|
+
*
|
|
235
|
+
* const program = Effect.gen(function* () {
|
|
236
|
+
* // ... operations that may fail
|
|
237
|
+
* }).pipe(
|
|
238
|
+
* Effect.catchTag('FileReadError', (e) => ...),
|
|
239
|
+
* Effect.catchTag('ApiKeyMissingError', (e) => ...)
|
|
240
|
+
* )
|
|
241
|
+
* ```
|
|
242
|
+
*/
|
|
243
|
+
/**
|
|
244
|
+
* Standardized error codes for programmatic handling.
|
|
245
|
+
*
|
|
246
|
+
* Error codes enable:
|
|
247
|
+
* - Scripting and automation (check error codes in CI/CD)
|
|
248
|
+
* - Machine-readable error handling without parsing messages
|
|
249
|
+
* - Stable identifiers that don't change when messages are updated
|
|
250
|
+
*
|
|
251
|
+
* Naming convention: E{category}{number}
|
|
252
|
+
* - E1xx: File system errors
|
|
253
|
+
* - E2xx: Parse errors
|
|
254
|
+
* - E3xx: API/authentication errors
|
|
255
|
+
* - E4xx: Index errors
|
|
256
|
+
* - E5xx: Search errors
|
|
257
|
+
* - E6xx: Vector store errors
|
|
258
|
+
* - E7xx: Config errors
|
|
259
|
+
* - E8xx: Watch errors
|
|
260
|
+
* - E9xx: CLI errors
|
|
261
|
+
*/
|
|
262
|
+
declare const ErrorCode: {
|
|
263
|
+
readonly FILE_READ: "E100";
|
|
264
|
+
readonly FILE_WRITE: "E101";
|
|
265
|
+
readonly DIRECTORY_CREATE: "E102";
|
|
266
|
+
readonly DIRECTORY_WALK: "E103";
|
|
267
|
+
readonly PARSE: "E200";
|
|
268
|
+
readonly API_KEY_MISSING: "E300";
|
|
269
|
+
readonly API_KEY_INVALID: "E301";
|
|
270
|
+
readonly EMBEDDING_RATE_LIMIT: "E310";
|
|
271
|
+
readonly EMBEDDING_QUOTA: "E311";
|
|
272
|
+
readonly EMBEDDING_NETWORK: "E312";
|
|
273
|
+
readonly EMBEDDING_MODEL: "E313";
|
|
274
|
+
readonly EMBEDDING_UNKNOWN: "E319";
|
|
275
|
+
readonly INDEX_NOT_FOUND: "E400";
|
|
276
|
+
readonly INDEX_CORRUPTED: "E401";
|
|
277
|
+
readonly INDEX_BUILD: "E402";
|
|
278
|
+
readonly DOCUMENT_NOT_FOUND: "E500";
|
|
279
|
+
readonly EMBEDDINGS_NOT_FOUND: "E501";
|
|
280
|
+
readonly VECTOR_STORE: "E600";
|
|
281
|
+
readonly DIMENSION_MISMATCH: "E601";
|
|
282
|
+
readonly CONFIG: "E700";
|
|
283
|
+
readonly WATCH: "E800";
|
|
284
|
+
readonly CLI_VALIDATION: "E900";
|
|
285
|
+
};
|
|
286
|
+
declare const FileReadError_base: new <A extends Record<string, any> = {}>(args: effect_Types.Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => effect_Cause.YieldableError & {
|
|
287
|
+
readonly _tag: "FileReadError";
|
|
288
|
+
} & Readonly<A>;
|
|
289
|
+
/**
|
|
290
|
+
* Error reading a file from the filesystem
|
|
291
|
+
*/
|
|
292
|
+
declare class FileReadError extends FileReadError_base<{
|
|
293
|
+
readonly path: string;
|
|
294
|
+
readonly message: string;
|
|
295
|
+
readonly cause?: unknown;
|
|
296
|
+
}> {
|
|
297
|
+
get code(): typeof ErrorCode.FILE_READ;
|
|
298
|
+
}
|
|
299
|
+
declare const FileWriteError_base: new <A extends Record<string, any> = {}>(args: effect_Types.Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => effect_Cause.YieldableError & {
|
|
300
|
+
readonly _tag: "FileWriteError";
|
|
301
|
+
} & Readonly<A>;
|
|
302
|
+
/**
|
|
303
|
+
* Error writing a file to the filesystem
|
|
304
|
+
*/
|
|
305
|
+
declare class FileWriteError extends FileWriteError_base<{
|
|
306
|
+
readonly path: string;
|
|
307
|
+
readonly message: string;
|
|
308
|
+
readonly cause?: unknown;
|
|
309
|
+
}> {
|
|
310
|
+
get code(): typeof ErrorCode.FILE_WRITE;
|
|
311
|
+
}
|
|
312
|
+
declare const DirectoryCreateError_base: new <A extends Record<string, any> = {}>(args: effect_Types.Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => effect_Cause.YieldableError & {
|
|
313
|
+
readonly _tag: "DirectoryCreateError";
|
|
314
|
+
} & Readonly<A>;
|
|
315
|
+
/**
|
|
316
|
+
* Error creating a directory
|
|
317
|
+
*/
|
|
318
|
+
declare class DirectoryCreateError extends DirectoryCreateError_base<{
|
|
319
|
+
readonly path: string;
|
|
320
|
+
readonly message: string;
|
|
321
|
+
readonly cause?: unknown;
|
|
322
|
+
}> {
|
|
323
|
+
get code(): typeof ErrorCode.DIRECTORY_CREATE;
|
|
324
|
+
}
|
|
325
|
+
declare const DirectoryWalkError_base: new <A extends Record<string, any> = {}>(args: effect_Types.Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => effect_Cause.YieldableError & {
|
|
326
|
+
readonly _tag: "DirectoryWalkError";
|
|
327
|
+
} & Readonly<A>;
|
|
328
|
+
/**
|
|
329
|
+
* Error walking/traversing a directory tree
|
|
330
|
+
*/
|
|
331
|
+
declare class DirectoryWalkError extends DirectoryWalkError_base<{
|
|
332
|
+
readonly path: string;
|
|
333
|
+
readonly message: string;
|
|
334
|
+
readonly cause?: unknown;
|
|
335
|
+
}> {
|
|
336
|
+
get code(): typeof ErrorCode.DIRECTORY_WALK;
|
|
337
|
+
}
|
|
338
|
+
declare const IndexCorruptedError_base: new <A extends Record<string, any> = {}>(args: effect_Types.Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => effect_Cause.YieldableError & {
|
|
339
|
+
readonly _tag: "IndexCorruptedError";
|
|
340
|
+
} & Readonly<A>;
|
|
341
|
+
/**
|
|
342
|
+
* Index exists but is corrupted or invalid
|
|
343
|
+
*/
|
|
344
|
+
declare class IndexCorruptedError extends IndexCorruptedError_base<{
|
|
345
|
+
readonly path: string;
|
|
346
|
+
readonly reason: 'InvalidJson' | 'VersionMismatch' | 'MissingData' | 'Unknown';
|
|
347
|
+
readonly details?: string;
|
|
348
|
+
}> {
|
|
349
|
+
get code(): typeof ErrorCode.INDEX_CORRUPTED;
|
|
350
|
+
get message(): string;
|
|
351
|
+
}
|
|
352
|
+
declare const WatchError_base: new <A extends Record<string, any> = {}>(args: effect_Types.Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => effect_Cause.YieldableError & {
|
|
353
|
+
readonly _tag: "WatchError";
|
|
354
|
+
} & Readonly<A>;
|
|
355
|
+
/**
|
|
356
|
+
* File watcher error
|
|
357
|
+
*/
|
|
358
|
+
declare class WatchError extends WatchError_base<{
|
|
359
|
+
readonly path: string;
|
|
360
|
+
readonly message: string;
|
|
361
|
+
readonly cause?: unknown;
|
|
362
|
+
}> {
|
|
363
|
+
get code(): typeof ErrorCode.WATCH;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* Index data types for mdcontext
|
|
368
|
+
*/
|
|
369
|
+
interface IndexConfig {
|
|
370
|
+
readonly version: number;
|
|
371
|
+
readonly rootPath: string;
|
|
372
|
+
readonly include: readonly string[];
|
|
373
|
+
readonly exclude: readonly string[];
|
|
374
|
+
readonly createdAt: string;
|
|
375
|
+
readonly updatedAt: string;
|
|
376
|
+
}
|
|
377
|
+
interface DocumentIndex {
|
|
378
|
+
readonly version: number;
|
|
379
|
+
readonly rootPath: string;
|
|
380
|
+
readonly documents: Record<string, DocumentEntry>;
|
|
381
|
+
}
|
|
382
|
+
interface DocumentEntry {
|
|
383
|
+
readonly id: string;
|
|
384
|
+
readonly path: string;
|
|
385
|
+
readonly title: string;
|
|
386
|
+
readonly mtime: number;
|
|
387
|
+
readonly hash: string;
|
|
388
|
+
readonly tokenCount: number;
|
|
389
|
+
readonly sectionCount: number;
|
|
390
|
+
}
|
|
391
|
+
interface SectionIndex {
|
|
392
|
+
readonly version: number;
|
|
393
|
+
readonly sections: Record<string, SectionEntry>;
|
|
394
|
+
readonly byHeading: Record<string, readonly string[]>;
|
|
395
|
+
readonly byDocument: Record<string, readonly string[]>;
|
|
396
|
+
}
|
|
397
|
+
interface SectionEntry {
|
|
398
|
+
readonly id: string;
|
|
399
|
+
readonly documentId: string;
|
|
400
|
+
readonly documentPath: string;
|
|
401
|
+
readonly heading: string;
|
|
402
|
+
readonly level: number;
|
|
403
|
+
readonly startLine: number;
|
|
404
|
+
readonly endLine: number;
|
|
405
|
+
readonly tokenCount: number;
|
|
406
|
+
readonly hasCode: boolean;
|
|
407
|
+
readonly hasList: boolean;
|
|
408
|
+
readonly hasTable: boolean;
|
|
409
|
+
}
|
|
410
|
+
interface LinkIndex {
|
|
411
|
+
readonly version: number;
|
|
412
|
+
readonly forward: Record<string, readonly string[]>;
|
|
413
|
+
readonly backward: Record<string, readonly string[]>;
|
|
414
|
+
readonly broken: readonly string[];
|
|
415
|
+
}
|
|
416
|
+
/**
|
|
417
|
+
* Reason why a file was skipped during indexing
|
|
418
|
+
*/
|
|
419
|
+
type SkipReason = 'unchanged' | 'excluded' | 'hidden' | 'not-markdown' | 'binary' | 'oversized';
|
|
420
|
+
/**
|
|
421
|
+
* Information about a skipped file
|
|
422
|
+
*/
|
|
423
|
+
interface SkippedFile {
|
|
424
|
+
readonly path: string;
|
|
425
|
+
readonly reason: SkipReason;
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* Summary of skipped files by reason
|
|
429
|
+
*/
|
|
430
|
+
interface SkipSummary {
|
|
431
|
+
readonly unchanged: number;
|
|
432
|
+
readonly excluded: number;
|
|
433
|
+
readonly hidden: number;
|
|
434
|
+
readonly total: number;
|
|
435
|
+
}
|
|
436
|
+
interface IndexResult {
|
|
437
|
+
readonly documentsIndexed: number;
|
|
438
|
+
readonly sectionsIndexed: number;
|
|
439
|
+
readonly linksIndexed: number;
|
|
440
|
+
readonly totalDocuments: number;
|
|
441
|
+
readonly totalSections: number;
|
|
442
|
+
readonly totalLinks: number;
|
|
443
|
+
readonly duration: number;
|
|
444
|
+
/** Non-fatal file processing errors (files that couldn't be indexed) */
|
|
445
|
+
readonly errors: readonly FileProcessingError[];
|
|
446
|
+
readonly skipped: SkipSummary;
|
|
447
|
+
}
|
|
448
|
+
/**
|
|
449
|
+
* Non-fatal error during file processing in index build.
|
|
450
|
+
* These are collected and reported but don't stop the build.
|
|
451
|
+
*
|
|
452
|
+
* Note: This is distinct from IndexBuildError in errors/index.ts,
|
|
453
|
+
* which is a TaggedError for fatal build failures.
|
|
454
|
+
*/
|
|
455
|
+
interface FileProcessingError {
|
|
456
|
+
readonly path: string;
|
|
457
|
+
readonly message: string;
|
|
458
|
+
}
|
|
459
|
+
declare const INDEX_DIR = ".mdcontext";
|
|
460
|
+
declare const INDEX_VERSION = 1;
|
|
461
|
+
declare const getIndexPaths: (rootPath: string) => {
|
|
462
|
+
root: string;
|
|
463
|
+
config: string;
|
|
464
|
+
documents: string;
|
|
465
|
+
sections: string;
|
|
466
|
+
links: string;
|
|
467
|
+
cache: string;
|
|
468
|
+
parsed: string;
|
|
469
|
+
};
|
|
470
|
+
|
|
471
|
+
/**
|
|
472
|
+
* Indexer service for building and updating indexes
|
|
473
|
+
*/
|
|
474
|
+
|
|
475
|
+
interface IndexProgress {
|
|
476
|
+
readonly current: number;
|
|
477
|
+
readonly total: number;
|
|
478
|
+
readonly filePath: string;
|
|
479
|
+
}
|
|
480
|
+
interface IndexOptions {
|
|
481
|
+
readonly force?: boolean | undefined;
|
|
482
|
+
/** CLI/config exclude patterns (overrides ignore files) */
|
|
483
|
+
readonly exclude?: readonly string[] | undefined;
|
|
484
|
+
/** Whether to honor .gitignore (default: true) */
|
|
485
|
+
readonly honorGitignore?: boolean | undefined;
|
|
486
|
+
/** Whether to honor .mdcontextignore (default: true) */
|
|
487
|
+
readonly honorMdcontextignore?: boolean | undefined;
|
|
488
|
+
/** Callback for progress updates during file indexing */
|
|
489
|
+
readonly onProgress?: ((progress: IndexProgress) => void) | undefined;
|
|
490
|
+
}
|
|
491
|
+
declare const buildIndex: (rootPath: string, options?: IndexOptions) => Effect.Effect<IndexResult, DirectoryWalkError | DirectoryCreateError | FileReadError | FileWriteError | IndexCorruptedError>;
|
|
492
|
+
declare const getOutgoingLinks: (rootPath: string, filePath: string) => Effect.Effect<readonly string[], FileReadError | IndexCorruptedError>;
|
|
493
|
+
declare const getIncomingLinks: (rootPath: string, filePath: string) => Effect.Effect<readonly string[], FileReadError | IndexCorruptedError>;
|
|
494
|
+
declare const getBrokenLinks: (rootPath: string) => Effect.Effect<readonly string[], FileReadError | IndexCorruptedError>;
|
|
495
|
+
interface BuildBM25Options {
|
|
496
|
+
readonly force?: boolean;
|
|
497
|
+
readonly onProgress?: (progress: {
|
|
498
|
+
current: number;
|
|
499
|
+
total: number;
|
|
500
|
+
}) => void;
|
|
501
|
+
}
|
|
502
|
+
interface BuildBM25Result {
|
|
503
|
+
readonly sectionsIndexed: number;
|
|
504
|
+
readonly duration: number;
|
|
505
|
+
}
|
|
506
|
+
/**
|
|
507
|
+
* Build BM25 keyword index for all sections.
|
|
508
|
+
*
|
|
509
|
+
* @param rootPath - Root directory containing indexed markdown files
|
|
510
|
+
* @param options - Build options (force rebuild, progress callback)
|
|
511
|
+
* @returns Result with section count and timing
|
|
512
|
+
*/
|
|
513
|
+
declare const buildBM25Index: (rootPath: string, options?: BuildBM25Options) => Effect.Effect<BuildBM25Result, FileReadError | IndexCorruptedError | FileWriteError>;
|
|
514
|
+
|
|
515
|
+
/**
|
|
516
|
+
* Index storage operations
|
|
517
|
+
*/
|
|
518
|
+
|
|
519
|
+
declare const computeHash: (content: string) => string;
|
|
520
|
+
interface IndexStorage {
|
|
521
|
+
readonly rootPath: string;
|
|
522
|
+
readonly paths: ReturnType<typeof getIndexPaths>;
|
|
523
|
+
}
|
|
524
|
+
declare const createStorage: (rootPath: string) => IndexStorage;
|
|
525
|
+
declare const initializeIndex: (storage: IndexStorage) => Effect.Effect<void, DirectoryCreateError | FileReadError | FileWriteError | IndexCorruptedError>;
|
|
526
|
+
declare const loadConfig: (storage: IndexStorage) => Effect.Effect<IndexConfig | null, FileReadError | IndexCorruptedError>;
|
|
527
|
+
declare const saveConfig: (storage: IndexStorage, config: IndexConfig) => Effect.Effect<void, DirectoryCreateError | FileWriteError>;
|
|
528
|
+
declare const loadDocumentIndex: (storage: IndexStorage) => Effect.Effect<DocumentIndex | null, FileReadError | IndexCorruptedError>;
|
|
529
|
+
declare const saveDocumentIndex: (storage: IndexStorage, index: DocumentIndex) => Effect.Effect<void, DirectoryCreateError | FileWriteError>;
|
|
530
|
+
declare const createEmptyDocumentIndex: (rootPath: string) => DocumentIndex;
|
|
531
|
+
declare const loadSectionIndex: (storage: IndexStorage) => Effect.Effect<SectionIndex | null, FileReadError | IndexCorruptedError>;
|
|
532
|
+
declare const saveSectionIndex: (storage: IndexStorage, index: SectionIndex) => Effect.Effect<void, DirectoryCreateError | FileWriteError>;
|
|
533
|
+
declare const createEmptySectionIndex: () => SectionIndex;
|
|
534
|
+
declare const loadLinkIndex: (storage: IndexStorage) => Effect.Effect<LinkIndex | null, FileReadError | IndexCorruptedError>;
|
|
535
|
+
declare const saveLinkIndex: (storage: IndexStorage, index: LinkIndex) => Effect.Effect<void, DirectoryCreateError | FileWriteError>;
|
|
536
|
+
declare const createEmptyLinkIndex: () => LinkIndex;
|
|
537
|
+
declare const indexExists: (storage: IndexStorage) => Effect.Effect<boolean, FileReadError>;
|
|
538
|
+
|
|
539
|
+
/**
|
|
540
|
+
* File watcher for automatic re-indexing
|
|
541
|
+
*
|
|
542
|
+
* ## Why Not Effect Streams?
|
|
543
|
+
*
|
|
544
|
+
* We evaluated using Effect Streams (ALP-101) but decided the current approach is better:
|
|
545
|
+
*
|
|
546
|
+
* 1. **chokidar is battle-tested** - Handles OS-specific quirks (FSEvents on macOS,
|
|
547
|
+
* inotify on Linux, ReadDirectoryChangesW on Windows)
|
|
548
|
+
*
|
|
549
|
+
* 2. **Debouncing handles backpressure** - The 300ms debounce already batches rapid
|
|
550
|
+
* changes, so Stream backpressure isn't needed
|
|
551
|
+
*
|
|
552
|
+
* 3. **Simple use case** - File change → rebuild index. No complex transformations
|
|
553
|
+
* or compositions that would benefit from Stream operators
|
|
554
|
+
*
|
|
555
|
+
* 4. **Already Effect-based** - The setup/teardown is wrapped in Effect for proper
|
|
556
|
+
* error handling, and we use typed errors (WatchError, IndexBuildError)
|
|
557
|
+
*
|
|
558
|
+
* If future requirements need more sophisticated event processing (filtering by
|
|
559
|
+
* content type, incremental updates, event replay), reconsider Streams then.
|
|
560
|
+
*/
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* Union of errors that can occur during watch operations
|
|
564
|
+
*/
|
|
565
|
+
type WatchDirectoryError = WatchError | DirectoryWalkError | DirectoryCreateError | FileReadError | FileWriteError | IndexCorruptedError;
|
|
566
|
+
interface WatcherOptions extends IndexOptions {
|
|
567
|
+
readonly debounceMs?: number;
|
|
568
|
+
readonly onIndex?: (result: {
|
|
569
|
+
documentsIndexed: number;
|
|
570
|
+
duration: number;
|
|
571
|
+
}) => void;
|
|
572
|
+
readonly onError?: (error: WatchError) => void;
|
|
573
|
+
/** Whether to honor .gitignore for file watching (default: true) */
|
|
574
|
+
readonly honorGitignore?: boolean;
|
|
575
|
+
/** Whether to honor .mdcontextignore for file watching (default: true) */
|
|
576
|
+
readonly honorMdcontextignore?: boolean;
|
|
577
|
+
}
|
|
578
|
+
interface Watcher {
|
|
579
|
+
readonly stop: () => void;
|
|
580
|
+
}
|
|
581
|
+
declare const watchDirectory: (rootPath: string, options?: WatcherOptions) => Effect.Effect<Watcher, WatchDirectoryError>;
|
|
582
|
+
|
|
583
|
+
/**
|
|
584
|
+
* Markdown parser using remark/unified
|
|
585
|
+
* Handles GFM (tables, task lists) and YAML frontmatter
|
|
586
|
+
*/
|
|
587
|
+
|
|
588
|
+
interface ParseOptions {
|
|
589
|
+
readonly path?: string;
|
|
590
|
+
readonly lastModified?: Date;
|
|
591
|
+
}
|
|
592
|
+
declare const parse: (content: string, options?: ParseOptions) => Effect.Effect<MdDocument, ParseError>;
|
|
593
|
+
/**
|
|
594
|
+
* Parse a markdown file from the filesystem
|
|
595
|
+
*
|
|
596
|
+
* @throws ParseError - File content cannot be parsed
|
|
597
|
+
* @throws FileReadError - File cannot be read from filesystem
|
|
598
|
+
*/
|
|
599
|
+
declare const parseFile: (filePath: string) => Effect.Effect<MdDocument, ParseError | FileReadError>;
|
|
600
|
+
|
|
601
|
+
/**
|
|
602
|
+
* Token counting utilities using tiktoken
|
|
603
|
+
*/
|
|
604
|
+
|
|
605
|
+
/**
|
|
606
|
+
* Count tokens in a string using the cl100k_base encoding
|
|
607
|
+
* (compatible with GPT-4, GPT-3.5-turbo, and Claude models)
|
|
608
|
+
*/
|
|
609
|
+
declare const countTokens: (text: string) => Effect.Effect<number, never, never>;
|
|
610
|
+
/**
|
|
611
|
+
* Synchronous token counting with improved approximation
|
|
612
|
+
*
|
|
613
|
+
* Uses heuristics calibrated against cl100k_base encoding:
|
|
614
|
+
* - Base prose: ~3.5 chars/token (conservative to never under-count)
|
|
615
|
+
* - Code blocks: Content at ~2.8 chars/token + fixed overhead per block
|
|
616
|
+
* - Inline code: ~2.5 chars/token + 2 tokens per backtick pair
|
|
617
|
+
* - Paths: ~3.0 chars/token (slashes tokenize separately)
|
|
618
|
+
* - Newlines: ~1 token each (they often become separate tokens)
|
|
619
|
+
* - Punctuation/symbols: adds ~0.8 tokens per mark
|
|
620
|
+
* - CJK characters: ~1.2 tokens per character
|
|
621
|
+
* - Emojis: ~2.5 tokens per emoji
|
|
622
|
+
*
|
|
623
|
+
* Safety margin of 10% to handle edge cases and ensure budget compliance.
|
|
624
|
+
* The conservative ratios combined with safety margin ensure we NEVER under-count.
|
|
625
|
+
*/
|
|
626
|
+
declare const countTokensApprox: (text: string) => number;
|
|
627
|
+
/**
|
|
628
|
+
* Count words in text
|
|
629
|
+
*/
|
|
630
|
+
declare const countWords: (text: string) => number;
|
|
631
|
+
/**
|
|
632
|
+
* Free the tiktoken encoder to release WebAssembly resources.
|
|
633
|
+
* Call this in test teardown to prevent process hang.
|
|
634
|
+
*/
|
|
635
|
+
declare const freeEncoder: () => void;
|
|
636
|
+
|
|
637
|
+
/**
|
|
638
|
+
* Type-safe configuration helper for mdcontext.config.ts files.
|
|
639
|
+
*
|
|
640
|
+
* @example
|
|
641
|
+
* ```typescript
|
|
642
|
+
* import { defineConfig } from 'mdcontext'
|
|
643
|
+
*
|
|
644
|
+
* export default defineConfig({
|
|
645
|
+
* index: {
|
|
646
|
+
* maxDepth: 5,
|
|
647
|
+
* },
|
|
648
|
+
* })
|
|
649
|
+
* ```
|
|
650
|
+
*/
|
|
651
|
+
declare const defineConfig: <T extends PartialMdContextConfig>(config: T) => T;
|
|
652
|
+
|
|
653
|
+
export { type BuildBM25Options, type BuildBM25Result, type DocumentEntry, type DocumentIndex, type DocumentMetadata, type FileProcessingError, type HeadingLevel, INDEX_DIR, INDEX_VERSION, type IndexConfig, type IndexOptions, type IndexProgress, type IndexResult, type IndexStorage, type LinkIndex, type LinkType, type MdCodeBlock, type MdDocument, type MdLink, type MdSection, ParseError, type ParseOptions, type PartialMdContextConfig, type SectionEntry, type SectionIndex, type SectionMetadata, type SkipReason, type SkipSummary, type SkippedFile, type WatchDirectoryError, type Watcher, type WatcherOptions, buildBM25Index, buildIndex, computeHash, countTokens, countTokensApprox, countWords, createEmptyDocumentIndex, createEmptyLinkIndex, createEmptySectionIndex, createStorage, defineConfig, freeEncoder, getBrokenLinks, getIncomingLinks, getIndexPaths, getOutgoingLinks, indexExists, initializeIndex, loadConfig, loadDocumentIndex, loadLinkIndex, loadSectionIndex, parse, parseFile, saveConfig, saveDocumentIndex, saveLinkIndex, saveSectionIndex, watchDirectory };
|