opencode-rag-plugin 1.8.4 → 1.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/ReadMe.md +25 -5
  2. package/dist/api.d.ts +27 -0
  3. package/dist/api.js +125 -0
  4. package/dist/api.js.map +1 -0
  5. package/dist/chunker/base.d.ts +2 -0
  6. package/dist/chunker/base.js +18 -7
  7. package/dist/chunker/base.js.map +1 -1
  8. package/dist/chunker/bash.d.ts +8 -0
  9. package/dist/chunker/bash.js +11 -0
  10. package/dist/chunker/bash.js.map +1 -0
  11. package/dist/chunker/dockerfile.d.ts +8 -0
  12. package/dist/chunker/dockerfile.js +29 -0
  13. package/dist/chunker/dockerfile.js.map +1 -0
  14. package/dist/chunker/factory.d.ts +9 -1
  15. package/dist/chunker/factory.js +24 -3
  16. package/dist/chunker/factory.js.map +1 -1
  17. package/dist/chunker/grammar.d.ts +1 -0
  18. package/dist/chunker/grammar.js +42 -2
  19. package/dist/chunker/grammar.js.map +1 -1
  20. package/dist/chunker/ini.d.ts +8 -0
  21. package/dist/chunker/ini.js +11 -0
  22. package/dist/chunker/ini.js.map +1 -0
  23. package/dist/chunker/markdown.d.ts +4 -3
  24. package/dist/chunker/markdown.js +4 -91
  25. package/dist/chunker/markdown.js.map +1 -1
  26. package/dist/chunker/php.d.ts +8 -0
  27. package/dist/chunker/php.js +12 -0
  28. package/dist/chunker/php.js.map +1 -0
  29. package/dist/chunker/powershell.d.ts +8 -0
  30. package/dist/chunker/powershell.js +11 -0
  31. package/dist/chunker/powershell.js.map +1 -0
  32. package/dist/chunker/razor.js +0 -1
  33. package/dist/chunker/razor.js.map +1 -1
  34. package/dist/chunker/sql.d.ts +8 -0
  35. package/dist/chunker/sql.js +11 -0
  36. package/dist/chunker/sql.js.map +1 -0
  37. package/dist/chunker/tex.js +2 -2
  38. package/dist/chunker/tex.js.map +1 -1
  39. package/dist/chunker/toml.d.ts +8 -0
  40. package/dist/chunker/toml.js +13 -0
  41. package/dist/chunker/toml.js.map +1 -0
  42. package/dist/chunker/yaml.d.ts +8 -0
  43. package/dist/chunker/yaml.js +12 -0
  44. package/dist/chunker/yaml.js.map +1 -0
  45. package/dist/cli.js +50 -11
  46. package/dist/cli.js.map +1 -1
  47. package/dist/core/config.d.ts +10 -1
  48. package/dist/core/config.js +127 -11
  49. package/dist/core/config.js.map +1 -1
  50. package/dist/embedder/openai.js +1 -5
  51. package/dist/embedder/openai.js.map +1 -1
  52. package/dist/eval/storage.d.ts +1 -0
  53. package/dist/eval/storage.js +4 -0
  54. package/dist/eval/storage.js.map +1 -1
  55. package/dist/eval/types.js +0 -1
  56. package/dist/eval/types.js.map +1 -1
  57. package/dist/index.d.ts +2 -0
  58. package/dist/index.js +3 -0
  59. package/dist/index.js.map +1 -1
  60. package/dist/indexer.d.ts +9 -0
  61. package/dist/indexer.js +27 -6
  62. package/dist/indexer.js.map +1 -1
  63. package/dist/mcp/cli.d.ts +4 -0
  64. package/dist/mcp/cli.js +15 -0
  65. package/dist/mcp/cli.js.map +1 -0
  66. package/dist/mcp/handlers.d.ts +46 -0
  67. package/dist/mcp/handlers.js +300 -0
  68. package/dist/mcp/handlers.js.map +1 -0
  69. package/dist/mcp/server.d.ts +12 -0
  70. package/dist/mcp/server.js +124 -0
  71. package/dist/mcp/server.js.map +1 -0
  72. package/dist/opencode/read-format.d.ts +1 -1
  73. package/dist/opencode/read-format.js +1 -1
  74. package/dist/opencode/tools.js +0 -5
  75. package/dist/opencode/tools.js.map +1 -1
  76. package/dist/plugin.js +18 -37
  77. package/dist/plugin.js.map +1 -1
  78. package/dist/retriever/keyword-index.js +7 -1
  79. package/dist/retriever/keyword-index.js.map +1 -1
  80. package/dist/retriever/retriever.js +51 -46
  81. package/dist/retriever/retriever.js.map +1 -1
  82. package/dist/watcher.js +1 -0
  83. package/dist/watcher.js.map +1 -1
  84. package/dist/web/api.js +10 -1
  85. package/dist/web/api.js.map +1 -1
  86. package/package.json +19 -2
package/ReadMe.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # OpenCodeRAG
2
2
 
3
- OpenCodeRAG is a **local-first RAG plugin** for semantic code search. It converts your codebase into vector indices and retrieves relevant code chunks on natural language queries. The primary aim is to save tokens by replacing full-file reads with targeted chunk retrieval and to speed-up tool calls for large codebases. Integrates seamlessly with [OpenCode](https://opencode.ai) and works standalone via CLI.
3
+ OpenCodeRAG is a **local-first RAG plugin** for semantic code search. It converts your codebase into vector indices and retrieves relevant code chunks on natural language queries. The primary aim is to save tokens by replacing full-file reads with targeted chunk retrieval and to speed-up tool calls for large codebases. Integrates seamlessly with [OpenCode](https://opencode.ai) and works as standalone MCP server or CLI tool.
4
4
 
5
5
  You don't need a dedicated GPU to run embedding LLMs, smaller models can still run performant on modern CPUs.
6
6
 
@@ -16,7 +16,7 @@ git clone https://github.com/your-org/OpenCodeRAG.git
16
16
  cd OpenCodeRAG
17
17
  npm install --legacy-peer-deps
18
18
  npm run build
19
- ./install.sh # global install (optional)
19
+ ./install.sh
20
20
 
21
21
  # 2. Initialize in your project
22
22
  cd /path/to/your/project
@@ -35,13 +35,15 @@ opencode-rag query "authentication middleware"
35
35
 
36
36
  | Feature | Description |
37
37
  |---|---|
38
- | **AST chunking** | 17 languages via tree-sitter (TS, JS, Python, Java, Go, Rust, C/C++, C#, Ruby, Kotlin, Swift, JSON, HTML, CSS, XML) |
38
+ | **MCP server** | `opencode-rag mcp` stdio-based MCP server exposing `search_semantic`, `get_file_skeleton`, `find_usages` tools for any MCP-compatible client |
39
+ | **AST chunking** | 25 languages via tree-sitter (TS, JS, Python, Java, Go, Rust, C/C++, C#, Ruby, Kotlin, Swift, Bash, PHP, PowerShell, SQL, JSON, HTML, CSS, XML, YAML, TOML, INI, Dockerfile, Markdown) |
39
40
  | **Document support** | Markdown, LaTeX, PDF, DOCX, DOC, Excel |
40
41
  | **Hybrid search** | Vector similarity + TF×IDF keyword fusion |
41
- | **OpenCode plugin** | Auto-inject context, read-tool override, TUI settings, Ctrl+Enter to add RAG context |
42
+ | **OpenCode plugin** | Auto-inject context, read-tool override, TUI settings, Ctrl+Enter to add RAG context, MCP registration on `init` |
42
43
  | **Incremental indexing** | File-hash manifest, background watcher, auto-rebuild on corruption |
43
44
  | **Privacy-first** | All processing stays local with Ollama |
44
- | **CLI** | `index`, `query`, `status`, `list`, `show`, `dump`, `clear`, `init`, `ui` |
45
+ | **CLI** | `index`, `query`, `status`, `list`, `show`, `dump`, `clear`, `init`, `ui`, `mcp` |
46
+ | **Programmatic API** | TypeScript `search()`, `indexWorkspace()`, `getContext()`, `validateConfig()`, `scanWorkspace()` |
45
47
  | **Proxy-aware** | Corporate proxy support with raw-socket localhost bypass |
46
48
  | **OpenAI / Cohere** | Alternate embedding providers with API key auto-resolution |
47
49
 
@@ -70,6 +72,24 @@ Launch with `opencode-rag ui`. See [Web UI documentation](doc/webui.md) for deta
70
72
  | [Troubleshooting](doc/troubleshooting.md) | Common issues, logging, debugging |
71
73
  | [Roadmap](doc/roadmap.md) | Completed items, short/mid/long-term plans |
72
74
 
75
+ ## MCP Server
76
+
77
+ OpenCodeRAG ships a stdio-based [MCP (Model Context Protocol)](https://spec.modelcontextprotocol.io/) server that exposes semantic code tools to any MCP-compatible client (Claude Desktop, OpenCode, Cursor, etc.).
78
+
79
+ ```bash
80
+ opencode-rag mcp
81
+ ```
82
+
83
+ ### MCP Tools
84
+
85
+ | Tool | Description |
86
+ |------|-------------|
87
+ | `search_semantic` | Vector + keyword hybrid search across the indexed codebase |
88
+ | `get_file_skeleton` | AST-based file outline (functions, classes, methods) |
89
+ | `find_usages` | Find all references to a symbol by name |
90
+
91
+ Clients can configure the MCP server manually, or `opencode-rag init` auto-registers it.
92
+
73
93
  ## Agent Discovery
74
94
 
75
95
  OpenCodeRAG registers tools that agents can invoke directly. Agents discover these tools via the OpenCode **skill system** - when `opencode-rag init` runs, it creates `.opencode/skills/opencode-rag/SKILL.md` which teaches agents the recommended workflow:
package/dist/api.d.ts ADDED
@@ -0,0 +1,27 @@
1
+ import { type IndexRunStats } from "./indexer.js";
2
+ import type { SearchResult } from "./core/interfaces.js";
3
+ export interface SearchOptions {
4
+ cwd?: string;
5
+ configPath?: string;
6
+ topK?: number;
7
+ minScore?: number;
8
+ keywordWeight?: number;
9
+ pathHints?: string[];
10
+ languageHints?: string[];
11
+ }
12
+ export interface IndexOptions {
13
+ configPath?: string;
14
+ force?: boolean;
15
+ onProgress?: (message: string) => void;
16
+ }
17
+ export interface ContextResult {
18
+ chunks: SearchResult[];
19
+ text: string;
20
+ }
21
+ export declare function search(query: string, options?: SearchOptions): Promise<SearchResult[]>;
22
+ export declare function indexWorkspace(cwd?: string, options?: IndexOptions): Promise<IndexRunStats>;
23
+ export declare function getContext(query: string, options?: SearchOptions): Promise<ContextResult>;
24
+ export { validateConfig } from "./core/config.js";
25
+ export type { ConfigValidationResult } from "./core/config.js";
26
+ export { scanWorkspace, getIndexStatusSummary } from "./indexer.js";
27
+ export type { WorkspaceFile, IndexRunStats } from "./indexer.js";
package/dist/api.js ADDED
@@ -0,0 +1,125 @@
1
+ import path from "node:path";
2
+ import { existsSync } from "node:fs";
3
+ import { loadConfig, DEFAULT_CONFIG } from "./core/config.js";
4
+ import { resolveApiKey } from "./core/resolve-api-key.js";
5
+ import { loadChunkersFromConfig } from "./chunker/loader.js";
6
+ import { createEmbedder } from "./embedder/factory.js";
7
+ import { createDescriptionProvider } from "./describer/factory.js";
8
+ import { LanceDBStore } from "./vectorstore/lancedb.js";
9
+ import { retrieve } from "./retriever/retriever.js";
10
+ import { KeywordIndex } from "./retriever/keyword-index.js";
11
+ import { runIndexPass } from "./indexer.js";
12
+ async function resolveConfig(cwd, configPath) {
13
+ const workDir = cwd ?? process.cwd();
14
+ if (configPath) {
15
+ const resolved = path.resolve(configPath);
16
+ const cfg = loadConfig(resolved);
17
+ resolveApiKey(cfg, workDir);
18
+ await loadChunkersFromConfig(cfg, path.dirname(resolved));
19
+ return cfg;
20
+ }
21
+ const locations = [
22
+ path.join(workDir, "opencode-rag.json"),
23
+ path.join(workDir, ".opencode", "opencode-rag.json"),
24
+ path.join(workDir, ".opencode", "rag.json"),
25
+ ];
26
+ for (const loc of locations) {
27
+ if (existsSync(loc)) {
28
+ const cfg = loadConfig(loc);
29
+ resolveApiKey(cfg, workDir);
30
+ await loadChunkersFromConfig(cfg, path.dirname(loc));
31
+ return cfg;
32
+ }
33
+ }
34
+ return { ...DEFAULT_CONFIG };
35
+ }
36
+ async function probeDimension(embedder) {
37
+ try {
38
+ const probe = await embedder.embed(["dimension-probe"], "query");
39
+ if (probe && probe[0] && probe[0].length > 0 && typeof probe[0][0] === "number") {
40
+ return probe[0].length;
41
+ }
42
+ }
43
+ catch {
44
+ // fallback to 384
45
+ }
46
+ return 384;
47
+ }
48
+ async function loadKeywordIndex(storePath) {
49
+ try {
50
+ const idx = await KeywordIndex.load(storePath);
51
+ return idx;
52
+ }
53
+ catch {
54
+ return new KeywordIndex(storePath);
55
+ }
56
+ }
57
+ function formatContextResults(results) {
58
+ if (results.length === 0)
59
+ return "No matching chunks found.";
60
+ const lines = [];
61
+ for (const r of results) {
62
+ const { filePath, startLine, endLine, language } = r.chunk.metadata;
63
+ lines.push(`#### ${filePath}:${startLine}-${endLine} (score: ${r.score.toFixed(3)})`);
64
+ lines.push("```" + language);
65
+ lines.push(r.chunk.content);
66
+ lines.push("```");
67
+ if (r.chunk.description) {
68
+ lines.push(`> ${r.chunk.description}`);
69
+ }
70
+ lines.push("");
71
+ }
72
+ return lines.join("\n");
73
+ }
74
+ export async function search(query, options = {}) {
75
+ const cfg = await resolveConfig(options.cwd, options.configPath);
76
+ const storePath = path.resolve(options.cwd ?? process.cwd(), cfg.vectorStore.path);
77
+ const embedder = createEmbedder(cfg);
78
+ const dimension = await probeDimension(embedder);
79
+ const store = new LanceDBStore(storePath, dimension);
80
+ const keywordIndex = await loadKeywordIndex(storePath);
81
+ return retrieve(query, embedder, store, {
82
+ topK: options.topK ?? cfg.retrieval.topK,
83
+ minScore: options.minScore ?? cfg.retrieval.minScore,
84
+ keywordIndex,
85
+ keywordWeight: options.keywordWeight ?? cfg.retrieval.hybridSearch?.keywordWeight ?? 0.4,
86
+ queryPrefix: cfg.embedding.queryPrefix,
87
+ });
88
+ }
89
+ export async function indexWorkspace(cwd, options = {}) {
90
+ const workDir = cwd ?? process.cwd();
91
+ const cfg = await resolveConfig(workDir, options.configPath);
92
+ const storePath = path.resolve(workDir, cfg.vectorStore.path);
93
+ const embedder = createEmbedder(cfg);
94
+ const dimension = await probeDimension(embedder);
95
+ const store = new LanceDBStore(storePath, dimension);
96
+ const keywordIndex = await loadKeywordIndex(storePath);
97
+ const descriptionConfig = cfg.description;
98
+ const descriptionProvider = descriptionConfig?.enabled
99
+ ? createDescriptionProvider(descriptionConfig)
100
+ : undefined;
101
+ if (options.onProgress) {
102
+ options.onProgress(`Indexing ${workDir}...`);
103
+ }
104
+ const stats = await runIndexPass({
105
+ cwd: workDir,
106
+ storePath,
107
+ config: cfg,
108
+ store,
109
+ embedder,
110
+ force: options.force ?? false,
111
+ keywordIndex,
112
+ descriptionProvider,
113
+ });
114
+ return stats;
115
+ }
116
+ export async function getContext(query, options = {}) {
117
+ const results = await search(query, options);
118
+ return {
119
+ chunks: results,
120
+ text: formatContextResults(results),
121
+ };
122
+ }
123
+ export { validateConfig } from "./core/config.js";
124
+ export { scanWorkspace, getIndexStatusSummary } from "./indexer.js";
125
+ //# sourceMappingURL=api.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"api.js","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,UAAU,EAAE,cAAc,EAA+C,MAAM,kBAAkB,CAAC;AAC3G,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAEpD,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAgF,MAAM,cAAc,CAAC;AAwB1H,KAAK,UAAU,aAAa,CAAC,GAAY,EAAE,UAAmB;IAC5D,MAAM,OAAO,GAAG,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAErC,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC1C,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACjC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC5B,MAAM,sBAAsB,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC1D,OAAO,GAAG,CAAC;IACb,CAAC;IAED,MAAM,SAAS,GAAG;QAChB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,mBAAmB,CAAC;QACpD,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC;KAC5C,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACpB,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;YAC5B,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC5B,MAAM,sBAAsB,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YACrD,OAAO,GAAG,CAAC;QACb,CAAC;IACH,CAAC;IAED,OAAO,EAAE,GAAG,cAAc,EAAE,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,QAA2B;IACvD,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC,CAAC;QACjE,IAAI,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;YAChF,OAAQ,KAAK,CAAC,CAAC,CAAc,CAAC,MAAM,CAAC;QACvC,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,kBAAkB;IACpB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,SAAiB;IAC/C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC/C,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;IACrC,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAuB;IACnD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,2BAA2B,CAAC;IAE7D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QACpE,KAAK,CAAC,IAAI,CAAC,QAAQ,QAAQ,IAAI,SAAS,IAAI,OAAO,YAAY,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACtF,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,IAAI,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QACzC,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAa,EACb,UAAyB,EAAE;IAE3B,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IACjE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAEnF,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IAErD,MAAM,YAAY,GAAG,MAAM,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAEvD,OAAO,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE;QACtC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI;QACxC,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC,SAAS,CAAC,QAAQ;QACpD,YAAY;QACZ,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,aAAa,IAAI,GAAG;QACxF,WAAW,EAAE,GAAG,CAAC,SAAS,CAAC,WAAW;KACb,CAAC,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,GAAY,EACZ,UAAwB,EAAE;IAE1B,MAAM,OAAO,GAAG,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACrC,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAE9D,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IAErD,MAAM,YAAY,GAAG,MAAM,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAEvD,MAAM,iBAAiB,GAAG,GAAG,CAAC,WAAW,CAAC;IAC1C,MAAM,mBAAmB,GAAG,iBAAiB,EAAE,OAAO;QACpD,CAAC,CAAC,yBAAyB,CAAC,iBAAiB,CAAC;QAC9C,CAAC,CAAC,SAAS,CAAC;IAEd,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,OAAO,CAAC,UAAU,CAAC,YAAY,OAAO,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC;QAC/B,GAAG,EAAE,OAAO;QACZ,SAAS;QACT,MAAM,EAAE,GAAG;QACX,KAAK;QACL,QAAQ;QACR,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,KAAK;QAC7B,YAAY;QACZ,mBAAmB;KACpB,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAa,EACb,UAAyB,EAAE;IAE3B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC7C,OAAO;QACL,MAAM,EAAE,OAAO;QACf,IAAI,EAAE,oBAAoB,CAAC,OAAO,CAAC;KACpC,CAAC;AACJ,CAAC;AAED,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC"}
@@ -4,7 +4,9 @@ export declare abstract class TreeSitterChunker implements Chunker {
4
4
  abstract readonly fileExtensions: string[];
5
5
  abstract readonly grammarName: string;
6
6
  abstract readonly nodeTypes: Set<string>;
7
+ readonly wasmFilePath?: string;
7
8
  private parser;
9
+ private parserPromise;
8
10
  withNodeTypes(types: Set<string>): Chunker;
9
11
  private getParser;
10
12
  chunk(filePath: string, content: string): Promise<Chunk[]>;
@@ -1,8 +1,10 @@
1
1
  import { Parser } from "web-tree-sitter";
2
- import { loadLanguage, walkTree } from "./grammar.js";
2
+ import { loadLanguage, loadLanguageFromPath, walkTree } from "./grammar.js";
3
3
  import { uuid } from "./uuid.js";
4
4
  export class TreeSitterChunker {
5
+ wasmFilePath;
5
6
  parser = null;
7
+ parserPromise;
6
8
  withNodeTypes(types) {
7
9
  const original = this;
8
10
  return {
@@ -29,13 +31,22 @@ export class TreeSitterChunker {
29
31
  },
30
32
  };
31
33
  }
32
- async getParser() {
33
- if (!this.parser) {
34
- const lang = await loadLanguage(this.grammarName);
35
- this.parser = new Parser();
36
- this.parser.setLanguage(lang);
34
+ getParser() {
35
+ if (this.parser)
36
+ return Promise.resolve(this.parser);
37
+ if (!this.parserPromise) {
38
+ this.parserPromise = (async () => {
39
+ const lang = this.wasmFilePath
40
+ ? await loadLanguageFromPath(this.grammarName, this.wasmFilePath)
41
+ : await loadLanguage(this.grammarName);
42
+ const parser = new Parser();
43
+ parser.setLanguage(lang);
44
+ this.parser = parser;
45
+ this.parserPromise = undefined;
46
+ return parser;
47
+ })();
37
48
  }
38
- return this.parser;
49
+ return this.parserPromise;
39
50
  }
40
51
  async chunk(filePath, content) {
41
52
  if (content.trim().length === 0)
@@ -1 +1 @@
1
- {"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/chunker/base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAgB,MAAM,cAAc,CAAC;AAEpE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,MAAM,OAAgB,iBAAiB;IAM7B,MAAM,GAAkB,IAAI,CAAC;IAErC,aAAa,CAAC,KAAkB;QAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC;QACtB,OAAO;YACL,IAAI,QAAQ,KAAK,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC5C,IAAI,cAAc,KAAK,OAAO,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC;YACxD,KAAK,CAAC,KAAK,CAAC,QAAgB,EAAE,OAAe;gBAC3C,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;oBAAE,OAAO,EAAE,CAAC;gBAC3C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,CAAC;gBAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACnC,IAAI,CAAC,IAAI;oBAAE,OAAO,EAAE,CAAC;gBACrB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;gBACtD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAa,EAAE,EAAE,CAAC,CAAC;oBACnC,EAAE,EAAE,IAAI,EAAE;oBACV,OAAO,EAAE,IAAI,CAAC,IAAI;oBAClB,QAAQ,EAAE;wBACR,QAAQ;wBACR,SAAS,EAAE,IAAI,CAAC,SAAS;wBACzB,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;qBAC5B;iBACF,CAAC,CAAC,CAAC;YACN,CAAC;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAClD,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAgB,EAAE,OAAe;QAC3C,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAE3C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QAErB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAE/D,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAa,EAAE,EAAE,CAAC,CAAC;YACnC,EAAE,EAAE,IAAI,EAAE;YACV,OAAO,EAAE,IAAI,CAAC,IAAI;YAClB,QAAQ,EAAE;gBACR,QAAQ;gBACR,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACxB;SACF,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
1
+ {"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/chunker/base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,QAAQ,EAAgB,MAAM,cAAc,CAAC;AAE1F,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,MAAM,OAAgB,iBAAiB;IAM5B,YAAY,CAAU;IAEvB,MAAM,GAAkB,IAAI,CAAC;IAC7B,aAAa,CAA8B;IAEnD,aAAa,CAAC,KAAkB;QAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC;QACtB,OAAO;YACL,IAAI,QAAQ,KAAK,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC5C,IAAI,cAAc,KAAK,OAAO,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC;YACxD,KAAK,CAAC,KAAK,CAAC,QAAgB,EAAE,OAAe;gBAC3C,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;oBAAE,OAAO,EAAE,CAAC;gBAC3C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,CAAC;gBAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACnC,IAAI,CAAC,IAAI;oBAAE,OAAO,EAAE,CAAC;gBACrB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;gBACtD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAa,EAAE,EAAE,CAAC,CAAC;oBACnC,EAAE,EAAE,IAAI,EAAE;oBACV,OAAO,EAAE,IAAI,CAAC,IAAI;oBAClB,QAAQ,EAAE;wBACR,QAAQ;wBACR,SAAS,EAAE,IAAI,CAAC,SAAS;wBACzB,OAAO,EAAE,IAAI,CAAC,OAAO;wBACrB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;qBAC5B;iBACF,CAAC,CAAC,CAAC;YACN,CAAC;SACF,CAAC;IACJ,CAAC;IAEO,SAAS;QACf,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACxB,IAAI,CAAC,aAAa,GAAG,CAAC,KAAK,IAAI,EAAE;gBAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY;oBAC5B,CAAC,CAAC,MAAM,oBAAoB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,YAAY,CAAC;oBACjE,CAAC,CAAC,MAAM,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBACzC,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;gBAC5B,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBACzB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;gBACrB,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC;gBAC/B,OAAO,MAAM,CAAC;YAChB,CAAC,CAAC,EAAE,CAAC;QACP,CAAC;QACD,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAgB,EAAE,OAAe;QAC3C,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAE3C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QAErB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAE/D,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAa,EAAE,EAAE,CAAC,CAAC;YACnC,EAAE,EAAE,IAAI,EAAE;YACV,OAAO,EAAE,IAAI,CAAC,IAAI;YAClB,QAAQ,EAAE;gBACR,QAAQ;gBACR,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACxB;SACF,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
@@ -0,0 +1,8 @@
1
+ import { TreeSitterChunker } from "./base.js";
2
+ export declare class BashChunker extends TreeSitterChunker {
3
+ readonly language = "bash";
4
+ readonly fileExtensions: string[];
5
+ readonly grammarName = "bash";
6
+ readonly nodeTypes: Set<string>;
7
+ }
8
+ export declare const bashChunker: BashChunker;
@@ -0,0 +1,11 @@
1
+ import { TreeSitterChunker } from "./base.js";
2
+ export class BashChunker extends TreeSitterChunker {
3
+ language = "bash";
4
+ fileExtensions = [".sh", ".bash", ".zsh"];
5
+ grammarName = "bash";
6
+ nodeTypes = new Set([
7
+ "function_definition",
8
+ ]);
9
+ }
10
+ export const bashChunker = new BashChunker();
11
+ //# sourceMappingURL=bash.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"bash.js","sourceRoot":"","sources":["../../src/chunker/bash.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C,MAAM,OAAO,WAAY,SAAQ,iBAAiB;IACvC,QAAQ,GAAG,MAAM,CAAC;IAClB,cAAc,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAC1C,WAAW,GAAG,MAAM,CAAC;IACrB,SAAS,GAAG,IAAI,GAAG,CAAC;QAC3B,qBAAqB;KACtB,CAAC,CAAC;CACJ;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC"}
@@ -0,0 +1,8 @@
1
+ import { TreeSitterChunker } from "./base.js";
2
+ export declare class DockerfileChunker extends TreeSitterChunker {
3
+ readonly language = "dockerfile";
4
+ readonly fileExtensions: string[];
5
+ readonly grammarName = "dockerfile";
6
+ readonly nodeTypes: Set<string>;
7
+ }
8
+ export declare const dockerfileChunker: DockerfileChunker;
@@ -0,0 +1,29 @@
1
+ import { TreeSitterChunker } from "./base.js";
2
+ export class DockerfileChunker extends TreeSitterChunker {
3
+ language = "dockerfile";
4
+ fileExtensions = [".dockerfile", "dockerfile", ".containerfile", "containerfile"];
5
+ grammarName = "dockerfile";
6
+ nodeTypes = new Set([
7
+ "from_instruction",
8
+ "run_instruction",
9
+ "cmd_instruction",
10
+ "entrypoint_instruction",
11
+ "env_instruction",
12
+ "arg_instruction",
13
+ "workdir_instruction",
14
+ "copy_instruction",
15
+ "add_instruction",
16
+ "expose_instruction",
17
+ "volume_instruction",
18
+ "user_instruction",
19
+ "label_instruction",
20
+ "healthcheck_instruction",
21
+ "shell_instruction",
22
+ "stopsignal_instruction",
23
+ "onbuild_instruction",
24
+ "maintainer_instruction",
25
+ "cross_build_instruction",
26
+ ]);
27
+ }
28
+ export const dockerfileChunker = new DockerfileChunker();
29
+ //# sourceMappingURL=dockerfile.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"dockerfile.js","sourceRoot":"","sources":["../../src/chunker/dockerfile.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C,MAAM,OAAO,iBAAkB,SAAQ,iBAAiB;IAC7C,QAAQ,GAAG,YAAY,CAAC;IACxB,cAAc,GAAG,CAAC,aAAa,EAAE,YAAY,EAAE,gBAAgB,EAAE,eAAe,CAAC,CAAC;IAClF,WAAW,GAAG,YAAY,CAAC;IAC3B,SAAS,GAAG,IAAI,GAAG,CAAC;QAC3B,kBAAkB;QAClB,iBAAiB;QACjB,iBAAiB;QACjB,wBAAwB;QACxB,iBAAiB;QACjB,iBAAiB;QACjB,qBAAqB;QACrB,kBAAkB;QAClB,iBAAiB;QACjB,oBAAoB;QACpB,oBAAoB;QACpB,kBAAkB;QAClB,mBAAmB;QACnB,yBAAyB;QACzB,mBAAmB;QACnB,wBAAwB;QACxB,qBAAqB;QACrB,wBAAwB;QACxB,yBAAyB;KAC1B,CAAC,CAAC;CACJ;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,iBAAiB,EAAE,CAAC"}
@@ -18,6 +18,14 @@ import { rustChunker } from "./rust.js";
18
18
  import { rubyChunker } from "./ruby.js";
19
19
  import { kotlinChunker } from "./kotlin.js";
20
20
  import { swiftChunker } from "./swift.js";
21
+ import { bashChunker } from "./bash.js";
22
+ import { phpChunker } from "./php.js";
23
+ import { powershellChunker } from "./powershell.js";
24
+ import { iniChunker } from "./ini.js";
25
+ import { yamlChunker } from "./yaml.js";
26
+ import { tomlChunker } from "./toml.js";
27
+ import { dockerfileChunker } from "./dockerfile.js";
28
+ import { sqlChunker } from "./sql.js";
21
29
  import { texChunker } from "./tex.js";
22
30
  import { fallbackChunker } from "./fallback.js";
23
31
  import { pdfChunker } from "./pdf.js";
@@ -25,4 +33,4 @@ export declare function registerChunker(chunker: Chunker, extensions?: string[])
25
33
  export declare function getChunker(filePath: string): Chunker;
26
34
  export declare function getRegisteredExtensions(): string[];
27
35
  export declare function chunkFile(filePath: string, content: string, nodeTypesOverrides?: Record<string, string[]>): Promise<Chunk[]>;
28
- export { typescriptChunker, pythonChunker, javaChunker, goChunker, markdownChunker, cChunker, cppChunker, csharpChunker, javascriptChunker, razorChunker, jsonChunker, htmlChunker, cssChunker, xmlChunker, slnChunker, rustChunker, rubyChunker, kotlinChunker, swiftChunker, texChunker, pdfChunker, fallbackChunker };
36
+ export { typescriptChunker, pythonChunker, javaChunker, goChunker, markdownChunker, cChunker, cppChunker, csharpChunker, javascriptChunker, razorChunker, jsonChunker, htmlChunker, cssChunker, xmlChunker, slnChunker, rustChunker, rubyChunker, kotlinChunker, swiftChunker, bashChunker, phpChunker, powershellChunker, iniChunker, yamlChunker, tomlChunker, dockerfileChunker, sqlChunker, texChunker, pdfChunker, fallbackChunker };
@@ -18,6 +18,14 @@ import { rustChunker } from "./rust.js";
18
18
  import { rubyChunker } from "./ruby.js";
19
19
  import { kotlinChunker } from "./kotlin.js";
20
20
  import { swiftChunker } from "./swift.js";
21
+ import { bashChunker } from "./bash.js";
22
+ import { phpChunker } from "./php.js";
23
+ import { powershellChunker } from "./powershell.js";
24
+ import { iniChunker } from "./ini.js";
25
+ import { yamlChunker } from "./yaml.js";
26
+ import { tomlChunker } from "./toml.js";
27
+ import { dockerfileChunker } from "./dockerfile.js";
28
+ import { sqlChunker } from "./sql.js";
21
29
  import { texChunker } from "./tex.js";
22
30
  import { fallbackChunker } from "./fallback.js";
23
31
  import { pdfChunker } from "./pdf.js";
@@ -45,6 +53,14 @@ const chunkers = [
45
53
  rubyChunker,
46
54
  kotlinChunker,
47
55
  swiftChunker,
56
+ bashChunker,
57
+ phpChunker,
58
+ powershellChunker,
59
+ iniChunker,
60
+ yamlChunker,
61
+ tomlChunker,
62
+ dockerfileChunker,
63
+ sqlChunker,
48
64
  texChunker,
49
65
  pdfChunker,
50
66
  docxChunker,
@@ -74,8 +90,13 @@ export function registerChunker(chunker, extensions) {
74
90
  }
75
91
  }
76
92
  export function getChunker(filePath) {
77
- const ext = filePath.slice(filePath.lastIndexOf(".")).toLowerCase();
78
- return extensionMap.get(ext) ?? fallbackChunker;
93
+ const dotIdx = filePath.lastIndexOf(".");
94
+ const ext = dotIdx >= 0 ? filePath.slice(dotIdx).toLowerCase() : "";
95
+ if (ext && extensionMap.has(ext)) {
96
+ return extensionMap.get(ext);
97
+ }
98
+ const basename = filePath.toLowerCase();
99
+ return extensionMap.get(basename) ?? fallbackChunker;
79
100
  }
80
101
  export function getRegisteredExtensions() {
81
102
  return [...extensionMap.keys()].sort();
@@ -150,5 +171,5 @@ export async function chunkFile(filePath, content, nodeTypesOverrides) {
150
171
  }
151
172
  return splitOversized(chunks, filePath);
152
173
  }
153
- export { typescriptChunker, pythonChunker, javaChunker, goChunker, markdownChunker, cChunker, cppChunker, csharpChunker, javascriptChunker, razorChunker, jsonChunker, htmlChunker, cssChunker, xmlChunker, slnChunker, rustChunker, rubyChunker, kotlinChunker, swiftChunker, texChunker, pdfChunker, fallbackChunker };
174
+ export { typescriptChunker, pythonChunker, javaChunker, goChunker, markdownChunker, cChunker, cppChunker, csharpChunker, javascriptChunker, razorChunker, jsonChunker, htmlChunker, cssChunker, xmlChunker, slnChunker, rustChunker, rubyChunker, kotlinChunker, swiftChunker, bashChunker, phpChunker, powershellChunker, iniChunker, yamlChunker, tomlChunker, dockerfileChunker, sqlChunker, texChunker, pdfChunker, fallbackChunker };
154
175
  //# sourceMappingURL=factory.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"factory.js","sourceRoot":"","sources":["../../src/chunker/factory.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,MAAM,QAAQ,GAAc;IAC1B,iBAAiB;IACjB,aAAa;IACb,WAAW;IACX,SAAS;IACT,eAAe;IACf,QAAQ;IACR,UAAU;IACV,aAAa;IACb,iBAAiB;IACjB,YAAY;IACZ,WAAW;IACX,WAAW;IACX,UAAU;IACV,UAAU;IACV,UAAU;IACV,WAAW;IACX,WAAW;IACX,aAAa;IACb,YAAY;IACZ,UAAU;IACV,UAAU;IACV,WAAW;IACX,UAAU;IACV,YAAY;CACb,CAAC;AAEF,MAAM,YAAY,GAAG,IAAI,GAAG,EAAmB,CAAC;AAEhD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;IAC/B,IAAI,gBAAgB,IAAI,OAAO,EAAE,CAAC;QAChC,MAAM,EAAE,GAAG,OAAwD,CAAC;QACpE,KAAK,MAAM,GAAG,IAAI,EAAE,CAAC,cAAc,EAAE,CAAC;YACpC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,OAAgB,EAChB,UAAqB;IAErB,MAAM,IAAI,GAAG,UAAU,IAAI,CAAC,gBAAgB,IAAI,OAAO;QACrD,CAAC,CAAE,OAAyD,CAAC,cAAc;QAC3E,CAAC,CAAC,EAAE,CAAC,CAAC;IAER,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CACV,+BAA+B,KAAK,sDAAsD,OAAO,CAAC,QAAQ,GAAG,CAC9G,CAAC;YACF,SAAS;QACX,CAAC;QACD,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,QAAgB;IACzC,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IACpE,OAAO,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,eAAe,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,uBAAuB;IACrC,OAAO,CAAC,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AACzC,CAAC;AAED,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B,SAAS,cAAc,CAAC,MAAe,EAAE,QAAgB;IACvD,MAAM,MAAM,GAAY,EAAE,CAAC;IAE3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,KAAK,CAAC,MAAM,IAAI,eAAe,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;YAC/E,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,SAAS;QACX,CAAC;QAED,MAAM,SAAS,GAAY,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAa,EAAE,CAAC;QAChC,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;YACvB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAEhC,IACE,YAAY,CAAC,MAAM,GAAG,CAAC;gBACvB,CAAC,YAAY,CAAC,MAAM,IAAI,eAAe,IAAI,gBAAgB,GAAG,OAAO,GAAG,eAAe,CAAC,EACxF,CAAC;gBACD,SAAS,CAAC,IAAI,CAAC;oBACb,EAAE,EAAE,IAAI,EAAE;oBACV,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;oBAChC,QAAQ,EAAE;wBACR,QAAQ;wBACR,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS,GAAG,UAAU;wBAChD,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS,GAAG,CAAC,GAAG,CAAC;wBACzC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ;qBAClC;iBACF,CAAC,CAAC;gBACH,YAAY,GAAG,EAAE,CAAC;gBAClB,gBAAgB,GAAG,CAAC,CAAC;gBACrB,UAAU,GAAG,CAAC,CAAC;YACjB,CAAC;YAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,gBAAgB,IAAI,OAAO,CAAC;QAC9B,CAAC;QAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,SAAS,CAAC,IAAI,CAAC;gBACb,EAAE,EAAE,IAAI,EAAE;gBACV,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChC,QAAQ,EAAE;oBACR,QAAQ;oBACR,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS,GAAG,UAAU;oBAChD,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC;oBACpD,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ;iBAClC;aACF,CAAC,CAAC;QACL,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAgB,EAChB,OAAe,EACf,kBAA6C;IAE7C,IAAI,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAEnC,IAAI,kBAAkB,IAAI,OAAO,YAAY,iBAAiB,EAAE,CAAC;QAC/D,MAAM,aAAa,GAAG,kBAAkB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3D,IAAI,aAAa,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9C,OAAO,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAEtD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;AAC1C,CAAC;AAED,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,WAAW,EAAE,SAAS,EAAE,eAAe,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,iBAAiB,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,eAAe,EAAE,CAAC"}
1
+ {"version":3,"file":"factory.js","sourceRoot":"","sources":["../../src/chunker/factory.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,MAAM,QAAQ,GAAc;IAC1B,iBAAiB;IACjB,aAAa;IACb,WAAW;IACX,SAAS;IACT,eAAe;IACf,QAAQ;IACR,UAAU;IACV,aAAa;IACb,iBAAiB;IACjB,YAAY;IACZ,WAAW;IACX,WAAW;IACX,UAAU;IACV,UAAU;IACV,UAAU;IACV,WAAW;IACX,WAAW;IACX,aAAa;IACb,YAAY;IACZ,WAAW;IACX,UAAU;IACV,iBAAiB;IACjB,UAAU;IACV,WAAW;IACX,WAAW;IACX,iBAAiB;IACjB,UAAU;IACV,UAAU;IACV,UAAU;IACV,WAAW;IACX,UAAU;IACV,YAAY;CACb,CAAC;AAEF,MAAM,YAAY,GAAG,IAAI,GAAG,EAAmB,CAAC;AAEhD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;IAC/B,IAAI,gBAAgB,IAAI,OAAO,EAAE,CAAC;QAChC,MAAM,EAAE,GAAG,OAAwD,CAAC;QACpE,KAAK,MAAM,GAAG,IAAI,EAAE,CAAC,cAAc,EAAE,CAAC;YACpC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,OAAgB,EAChB,UAAqB;IAErB,MAAM,IAAI,GAAG,UAAU,IAAI,CAAC,gBAAgB,IAAI,OAAO;QACrD,CAAC,CAAE,OAAyD,CAAC,cAAc;QAC3E,CAAC,CAAC,EAAE,CAAC,CAAC;IAER,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CACV,+BAA+B,KAAK,sDAAsD,OAAO,CAAC,QAAQ,GAAG,CAC9G,CAAC;YACF,SAAS;QACX,CAAC;QACD,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,QAAgB;IACzC,MAAM,MAAM,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACpE,IAAI,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,YAAY,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;IAChC,CAAC;IACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;IACxC,OAAO,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,eAAe,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,uBAAuB;IACrC,OAAO,CAAC,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AACzC,CAAC;AAED,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B,SAAS,cAAc,CAAC,MAAe,EAAE,QAAgB;IACvD,MAAM,MAAM,GAAY,EAAE,CAAC;IAE3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,KAAK,CAAC,MAAM,IAAI,eAAe,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;YAC/E,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,SAAS;QACX,CAAC;QAED,MAAM,SAAS,GAAY,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAa,EAAE,CAAC;QAChC,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;YACvB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAEhC,IACE,YAAY,CAAC,MAAM,GAAG,CAAC;gBACvB,CAAC,YAAY,CAAC,MAAM,IAAI,eAAe,IAAI,gBAAgB,GAAG,OAAO,GAAG,eAAe,CAAC,EACxF,CAAC;gBACD,SAAS,CAAC,IAAI,CAAC;oBACb,EAAE,EAAE,IAAI,EAAE;oBACV,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;oBAChC,QAAQ,EAAE;wBACR,QAAQ;wBACR,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS,GAAG,UAAU;wBAChD,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS,GAAG,CAAC,GAAG,CAAC;wBACzC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ;qBAClC;iBACF,CAAC,CAAC;gBACH,YAAY,GAAG,EAAE,CAAC;gBAClB,gBAAgB,GAAG,CAAC,CAAC;gBACrB,UAAU,GAAG,CAAC,CAAC;YACjB,CAAC;YAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,gBAAgB,IAAI,OAAO,CAAC;QAC9B,CAAC;QAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,SAAS,CAAC,IAAI,CAAC;gBACb,EAAE,EAAE,IAAI,EAAE;gBACV,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChC,QAAQ,EAAE;oBACR,QAAQ;oBACR,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS,GAAG,UAAU;oBAChD,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC;oBACpD,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ;iBAClC;aACF,CAAC,CAAC;QACL,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAgB,EAChB,OAAe,EACf,kBAA6C;IAE7C,IAAI,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAEnC,IAAI,kBAAkB,IAAI,OAAO,YAAY,iBAAiB,EAAE,CAAC;QAC/D,MAAM,aAAa,GAAG,kBAAkB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3D,IAAI,aAAa,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9C,OAAO,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAEtD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;AAC1C,CAAC;AAED,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,WAAW,EAAE,SAAS,EAAE,eAAe,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,iBAAiB,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,iBAAiB,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,iBAAiB,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,eAAe,EAAE,CAAC"}
@@ -1,6 +1,7 @@
1
1
  import { Language, Node } from "web-tree-sitter";
2
2
  export declare function initParser(): Promise<void>;
3
3
  export declare function loadLanguage(lang: string): Promise<Language>;
4
+ export declare function loadLanguageFromPath(cacheKey: string, wasmPath: string): Promise<Language>;
4
5
  export interface AstNode {
5
6
  text: string;
6
7
  startLine: number;
@@ -1,6 +1,35 @@
1
1
  import { Parser, Language } from "web-tree-sitter";
2
- import { getWasmPath } from "tree-sitter-wasm";
3
2
  import { readFileSync } from "node:fs";
3
+ import { resolve, dirname } from "node:path";
4
+ import { fileURLToPath } from "node:url";
5
+ const PROJECT_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..");
6
+ const MS_WASM_DIR = resolve(PROJECT_ROOT, "node_modules", "@vscode", "tree-sitter-wasm", "wasm");
7
+ const SELF_WASM_DIR = resolve(PROJECT_ROOT, "wasm");
8
+ const MS_LANGUAGES = new Set([
9
+ "bash",
10
+ "c_sharp",
11
+ "cpp",
12
+ "css",
13
+ "go",
14
+ "ini",
15
+ "java",
16
+ "javascript",
17
+ "php",
18
+ "powershell",
19
+ "python",
20
+ "regex",
21
+ "ruby",
22
+ "rust",
23
+ "tsx",
24
+ "typescript",
25
+ ]);
26
+ function resolveWasmPath(lang) {
27
+ if (MS_LANGUAGES.has(lang)) {
28
+ const msName = lang.replace(/_/g, "-");
29
+ return resolve(MS_WASM_DIR, `tree-sitter-${msName}.wasm`);
30
+ }
31
+ return resolve(SELF_WASM_DIR, `tree-sitter-${lang}.wasm`);
32
+ }
4
33
  let initialized = false;
5
34
  let initPromise = null;
6
35
  export async function initParser() {
@@ -20,12 +49,23 @@ export async function loadLanguage(lang) {
20
49
  if (cached)
21
50
  return cached;
22
51
  await initParser();
23
- const wasmPath = getWasmPath(lang);
52
+ const wasmPath = resolveWasmPath(lang);
24
53
  const buffer = readFileSync(wasmPath);
25
54
  const language = await Language.load(buffer);
26
55
  grammarCache.set(lang, language);
27
56
  return language;
28
57
  }
58
+ export async function loadLanguageFromPath(cacheKey, wasmPath) {
59
+ const cached = grammarCache.get(cacheKey);
60
+ if (cached)
61
+ return cached;
62
+ await initParser();
63
+ const resolvedPath = resolve(wasmPath);
64
+ const buffer = readFileSync(resolvedPath);
65
+ const language = await Language.load(buffer);
66
+ grammarCache.set(cacheKey, language);
67
+ return language;
68
+ }
29
69
  export function walkTree(node, nodeTypes, source, maxDepth = 10, depth = 0) {
30
70
  const results = [];
31
71
  if (nodeTypes.has(node.type) && depth > 0) {
@@ -1 +1 @@
1
- {"version":3,"file":"grammar.js","sourceRoot":"","sources":["../../src/chunker/grammar.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAQ,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,WAAW,EAA0B,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAEvC,IAAI,WAAW,GAAG,KAAK,CAAC;AACxB,IAAI,WAAW,GAAyB,IAAI,CAAC;AAE7C,MAAM,CAAC,KAAK,UAAU,UAAU;IAC9B,IAAI,WAAW;QAAE,OAAO;IACxB,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC;IACpC,WAAW,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;QACpC,WAAW,GAAG,IAAI,CAAC;QACnB,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC,CAAC,CAAC;IACH,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,MAAM,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAC;AAEjD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY;IAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACtC,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAE1B,MAAM,UAAU,EAAE,CAAC;IACnB,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAyB,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7C,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACjC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAWD,MAAM,UAAU,QAAQ,CACtB,IAAU,EACV,SAAsB,EACtB,MAAc,EACd,WAAmB,EAAE,EACrB,QAAgB,CAAC;IAEjB,MAAM,OAAO,GAAc,EAAE,CAAC;IAE9B,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC1C,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;YAClD,SAAS,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC;YACrC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,GAAG,CAAC;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;QACrB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,OAAO,CAAC,IAAI,CACV,GAAG,QAAQ,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAC3D,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
1
+ {"version":3,"file":"grammar.js","sourceRoot":"","sources":["../../src/chunker/grammar.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAQ,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAElF,MAAM,WAAW,GAAG,OAAO,CACzB,YAAY,EACZ,cAAc,EACd,SAAS,EACT,kBAAkB,EAClB,MAAM,CACP,CAAC;AAEF,MAAM,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;AAEpD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC3B,MAAM;IACN,SAAS;IACT,KAAK;IACL,KAAK;IACL,IAAI;IACJ,KAAK;IACL,MAAM;IACN,YAAY;IACZ,KAAK;IACL,YAAY;IACZ,QAAQ;IACR,OAAO;IACP,MAAM;IACN,MAAM;IACN,KAAK;IACL,YAAY;CACb,CAAC,CAAC;AAEH,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACvC,OAAO,OAAO,CAAC,WAAW,EAAE,eAAe,MAAM,OAAO,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,OAAO,CAAC,aAAa,EAAE,eAAe,IAAI,OAAO,CAAC,CAAC;AAC5D,CAAC;AAED,IAAI,WAAW,GAAG,KAAK,CAAC;AACxB,IAAI,WAAW,GAAyB,IAAI,CAAC;AAE7C,MAAM,CAAC,KAAK,UAAU,UAAU;IAC9B,IAAI,WAAW;QAAE,OAAO;IACxB,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC;IACpC,WAAW,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;QACpC,WAAW,GAAG,IAAI,CAAC;QACnB,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC,CAAC,CAAC;IACH,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,MAAM,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAC;AAEjD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY;IAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACtC,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAE1B,MAAM,UAAU,EAAE,CAAC;IACnB,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7C,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACjC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,QAAgB,EAChB,QAAgB;IAEhB,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC1C,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAE1B,MAAM,UAAU,EAAE,CAAC;IACnB,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7C,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACrC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAWD,MAAM,UAAU,QAAQ,CACtB,IAAU,EACV,SAAsB,EACtB,MAAc,EACd,WAAmB,EAAE,EACrB,QAAgB,CAAC;IAEjB,MAAM,OAAO,GAAc,EAAE,CAAC;IAE9B,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC1C,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC;YAClD,SAAS,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC;YACrC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,GAAG,CAAC;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;QACrB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,OAAO,CAAC,IAAI,CACV,GAAG,QAAQ,CAAC,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAC3D,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
@@ -0,0 +1,8 @@
1
+ import { TreeSitterChunker } from "./base.js";
2
+ export declare class IniChunker extends TreeSitterChunker {
3
+ readonly language = "ini";
4
+ readonly fileExtensions: string[];
5
+ readonly grammarName = "ini";
6
+ readonly nodeTypes: Set<string>;
7
+ }
8
+ export declare const iniChunker: IniChunker;
@@ -0,0 +1,11 @@
1
+ import { TreeSitterChunker } from "./base.js";
2
+ export class IniChunker extends TreeSitterChunker {
3
+ language = "ini";
4
+ fileExtensions = [".ini", ".cfg"];
5
+ grammarName = "ini";
6
+ nodeTypes = new Set([
7
+ "section",
8
+ ]);
9
+ }
10
+ export const iniChunker = new IniChunker();
11
+ //# sourceMappingURL=ini.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ini.js","sourceRoot":"","sources":["../../src/chunker/ini.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C,MAAM,OAAO,UAAW,SAAQ,iBAAiB;IACtC,QAAQ,GAAG,KAAK,CAAC;IACjB,cAAc,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,WAAW,GAAG,KAAK,CAAC;IACpB,SAAS,GAAG,IAAI,GAAG,CAAC;QAC3B,SAAS;KACV,CAAC,CAAC;CACJ;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,UAAU,EAAE,CAAC"}
@@ -1,7 +1,8 @@
1
- import type { Chunker, Chunk } from "../core/interfaces.js";
2
- export declare class MarkdownChunker implements Chunker {
1
+ import { TreeSitterChunker } from "./base.js";
2
+ export declare class MarkdownChunker extends TreeSitterChunker {
3
3
  readonly language = "markdown";
4
4
  readonly fileExtensions: string[];
5
- chunk(filePath: string, content: string): Promise<Chunk[]>;
5
+ readonly grammarName = "markdown";
6
+ readonly nodeTypes: Set<string>;
6
7
  }
7
8
  export declare const markdownChunker: MarkdownChunker;