opencode-rag-plugin 1.12.17 → 1.14.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +15 -13
- package/dist/api.d.ts +54 -2
- package/dist/api.js +41 -1
- package/dist/api.js.map +1 -1
- package/dist/chunker/base.js +2 -0
- package/dist/chunker/base.js.map +1 -1
- package/dist/chunker/bash.d.ts +5 -0
- package/dist/chunker/bash.js +5 -0
- package/dist/chunker/bash.js.map +1 -1
- package/dist/chunker/c.d.ts +6 -0
- package/dist/chunker/c.js +6 -0
- package/dist/chunker/c.js.map +1 -1
- package/dist/chunker/cpp.d.ts +6 -0
- package/dist/chunker/cpp.js +6 -0
- package/dist/chunker/cpp.js.map +1 -1
- package/dist/chunker/csharp.d.ts +6 -0
- package/dist/chunker/csharp.js +6 -0
- package/dist/chunker/csharp.js.map +1 -1
- package/dist/chunker/css.d.ts +6 -0
- package/dist/chunker/css.js +6 -0
- package/dist/chunker/css.js.map +1 -1
- package/dist/chunker/doc.d.ts +18 -0
- package/dist/chunker/doc.js +18 -0
- package/dist/chunker/doc.js.map +1 -1
- package/dist/chunker/dockerfile.d.ts +6 -0
- package/dist/chunker/dockerfile.js +6 -0
- package/dist/chunker/dockerfile.js.map +1 -1
- package/dist/chunker/docx.d.ts +18 -0
- package/dist/chunker/docx.js +18 -0
- package/dist/chunker/docx.js.map +1 -1
- package/dist/chunker/excel.d.ts +19 -0
- package/dist/chunker/excel.js +19 -0
- package/dist/chunker/excel.js.map +1 -1
- package/dist/chunker/go.d.ts +5 -0
- package/dist/chunker/go.js +5 -0
- package/dist/chunker/go.js.map +1 -1
- package/dist/chunker/grammar.d.ts +62 -0
- package/dist/chunker/grammar.js +133 -0
- package/dist/chunker/grammar.js.map +1 -1
- package/dist/chunker/html.d.ts +5 -0
- package/dist/chunker/html.js +5 -0
- package/dist/chunker/html.js.map +1 -1
- package/dist/chunker/image.d.ts +31 -0
- package/dist/chunker/image.js +50 -4
- package/dist/chunker/image.js.map +1 -1
- package/dist/chunker/ini.d.ts +5 -0
- package/dist/chunker/ini.js +5 -0
- package/dist/chunker/ini.js.map +1 -1
- package/dist/chunker/java.d.ts +5 -0
- package/dist/chunker/java.js +5 -0
- package/dist/chunker/java.js.map +1 -1
- package/dist/chunker/javascript.d.ts +6 -0
- package/dist/chunker/javascript.js +6 -0
- package/dist/chunker/javascript.js.map +1 -1
- package/dist/chunker/json.d.ts +5 -0
- package/dist/chunker/json.js +5 -0
- package/dist/chunker/json.js.map +1 -1
- package/dist/chunker/kotlin.d.ts +6 -0
- package/dist/chunker/kotlin.js +6 -0
- package/dist/chunker/kotlin.js.map +1 -1
- package/dist/chunker/loader.d.ts +11 -0
- package/dist/chunker/loader.js +20 -0
- package/dist/chunker/loader.js.map +1 -1
- package/dist/chunker/markdown.d.ts +5 -0
- package/dist/chunker/markdown.js +5 -0
- package/dist/chunker/markdown.js.map +1 -1
- package/dist/chunker/pdf.d.ts +19 -0
- package/dist/chunker/pdf.js +30 -0
- package/dist/chunker/pdf.js.map +1 -1
- package/dist/chunker/php.d.ts +5 -0
- package/dist/chunker/php.js +5 -0
- package/dist/chunker/php.js.map +1 -1
- package/dist/chunker/powershell.d.ts +5 -0
- package/dist/chunker/powershell.js +5 -0
- package/dist/chunker/powershell.js.map +1 -1
- package/dist/chunker/python.d.ts +5 -0
- package/dist/chunker/python.js +5 -0
- package/dist/chunker/python.js.map +1 -1
- package/dist/chunker/razor.d.ts +12 -0
- package/dist/chunker/razor.js +25 -0
- package/dist/chunker/razor.js.map +1 -1
- package/dist/chunker/ruby.d.ts +5 -0
- package/dist/chunker/ruby.js +5 -0
- package/dist/chunker/ruby.js.map +1 -1
- package/dist/chunker/rust.d.ts +6 -0
- package/dist/chunker/rust.js +6 -0
- package/dist/chunker/rust.js.map +1 -1
- package/dist/chunker/sln.d.ts +11 -0
- package/dist/chunker/sln.js +11 -0
- package/dist/chunker/sln.js.map +1 -1
- package/dist/chunker/sql.d.ts +5 -0
- package/dist/chunker/sql.js +5 -0
- package/dist/chunker/sql.js.map +1 -1
- package/dist/chunker/ssl.d.ts +12 -0
- package/dist/chunker/ssl.js +21 -2
- package/dist/chunker/ssl.js.map +1 -1
- package/dist/chunker/swift.d.ts +5 -0
- package/dist/chunker/swift.js +5 -0
- package/dist/chunker/swift.js.map +1 -1
- package/dist/chunker/tex.d.ts +12 -0
- package/dist/chunker/tex.js +12 -0
- package/dist/chunker/tex.js.map +1 -1
- package/dist/chunker/toml.d.ts +5 -0
- package/dist/chunker/toml.js +5 -0
- package/dist/chunker/toml.js.map +1 -1
- package/dist/chunker/typescript.d.ts +6 -0
- package/dist/chunker/typescript.js +6 -0
- package/dist/chunker/typescript.js.map +1 -1
- package/dist/chunker/xml.d.ts +5 -0
- package/dist/chunker/xml.js +5 -0
- package/dist/chunker/xml.js.map +1 -1
- package/dist/chunker/yaml.d.ts +5 -0
- package/dist/chunker/yaml.js +5 -0
- package/dist/chunker/yaml.js.map +1 -1
- package/dist/cli/commands/clear.d.ts +12 -0
- package/dist/cli/commands/clear.js +81 -0
- package/dist/cli/commands/clear.js.map +1 -0
- package/dist/cli/commands/describe-image.d.ts +14 -0
- package/dist/cli/commands/describe-image.js +68 -0
- package/dist/cli/commands/describe-image.js.map +1 -0
- package/dist/cli/commands/dump.d.ts +13 -0
- package/dist/cli/commands/dump.js +55 -0
- package/dist/cli/commands/dump.js.map +1 -0
- package/dist/cli/commands/eval.d.ts +32 -0
- package/dist/cli/commands/eval.js +158 -0
- package/dist/cli/commands/eval.js.map +1 -0
- package/dist/cli/commands/index-command.d.ts +17 -0
- package/dist/cli/commands/index-command.js +167 -0
- package/dist/cli/commands/index-command.js.map +1 -0
- package/dist/cli/commands/index.d.ts +16 -0
- package/dist/cli/commands/index.js +17 -0
- package/dist/cli/commands/index.js.map +1 -0
- package/dist/cli/commands/init-helpers.d.ts +101 -0
- package/dist/cli/commands/init-helpers.js +397 -0
- package/dist/cli/commands/init-helpers.js.map +1 -0
- package/dist/cli/commands/init.d.ts +19 -0
- package/dist/cli/commands/init.js +251 -0
- package/dist/cli/commands/init.js.map +1 -0
- package/dist/cli/commands/list.d.ts +13 -0
- package/dist/cli/commands/list.js +46 -0
- package/dist/cli/commands/list.js.map +1 -0
- package/dist/cli/commands/mcp.d.ts +14 -0
- package/dist/cli/commands/mcp.js +35 -0
- package/dist/cli/commands/mcp.js.map +1 -0
- package/dist/cli/commands/query.d.ts +14 -0
- package/dist/cli/commands/query.js +86 -0
- package/dist/cli/commands/query.js.map +1 -0
- package/dist/cli/commands/show.d.ts +13 -0
- package/dist/cli/commands/show.js +51 -0
- package/dist/cli/commands/show.js.map +1 -0
- package/dist/cli/commands/status.d.ts +14 -0
- package/dist/cli/commands/status.js +104 -0
- package/dist/cli/commands/status.js.map +1 -0
- package/dist/cli/commands/ui.d.ts +14 -0
- package/dist/cli/commands/ui.js +69 -0
- package/dist/cli/commands/ui.js.map +1 -0
- package/dist/cli/commands/update.d.ts +14 -0
- package/dist/cli/commands/update.js +76 -0
- package/dist/cli/commands/update.js.map +1 -0
- package/dist/cli/format.d.ts +112 -0
- package/dist/cli/format.js +184 -0
- package/dist/cli/format.js.map +1 -0
- package/dist/cli/helpers.d.ts +51 -0
- package/dist/cli/helpers.js +86 -0
- package/dist/cli/helpers.js.map +1 -0
- package/dist/cli/index.d.ts +28 -0
- package/dist/cli/index.js +104 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/progress.d.ts +39 -0
- package/dist/cli/progress.js +137 -0
- package/dist/cli/progress.js.map +1 -0
- package/dist/cli/types.d.ts +40 -0
- package/dist/cli/types.js +3 -0
- package/dist/cli/types.js.map +1 -0
- package/dist/cli.d.ts +11 -4
- package/dist/cli.js +9 -1253
- package/dist/cli.js.map +1 -1
- package/dist/content/doc.d.ts +1 -1
- package/dist/content/doc.js +1 -1
- package/dist/content/doc.js.map +1 -1
- package/dist/content/docx.d.ts +1 -1
- package/dist/content/docx.js +1 -1
- package/dist/content/docx.js.map +1 -1
- package/dist/content/excel.d.ts +1 -1
- package/dist/content/excel.js +1 -1
- package/dist/content/excel.js.map +1 -1
- package/dist/content/pdf.d.ts +1 -1
- package/dist/content/pdf.js +1 -1
- package/dist/content/pdf.js.map +1 -1
- package/dist/content/reader.d.ts +3 -1
- package/dist/content/reader.js +42 -10
- package/dist/content/reader.js.map +1 -1
- package/dist/core/bootstrap.d.ts +18 -0
- package/dist/core/bootstrap.js +9 -15
- package/dist/core/bootstrap.js.map +1 -1
- package/dist/core/config.d.ts +144 -0
- package/dist/core/config.js +65 -14
- package/dist/core/config.js.map +1 -1
- package/dist/core/doc-progress.d.ts +15 -0
- package/dist/core/doc-progress.js +61 -0
- package/dist/core/doc-progress.js.map +1 -0
- package/dist/core/fileLogger.d.ts +7 -0
- package/dist/core/fileLogger.js +2 -0
- package/dist/core/fileLogger.js.map +1 -1
- package/dist/core/interfaces.d.ts +52 -1
- package/dist/core/manifest.d.ts +25 -0
- package/dist/core/manifest.js +15 -1
- package/dist/core/manifest.js.map +1 -1
- package/dist/core/provider-defaults.d.ts +8 -0
- package/dist/core/provider-defaults.js +3 -0
- package/dist/core/provider-defaults.js.map +1 -1
- package/dist/core/rag-injection-flag.d.ts +3 -0
- package/dist/core/rag-injection-flag.js +2 -0
- package/dist/core/rag-injection-flag.js.map +1 -1
- package/dist/core/resolve-api-key.d.ts +1 -0
- package/dist/core/resolve-api-key.js +1 -0
- package/dist/core/resolve-api-key.js.map +1 -1
- package/dist/core/runtime-overrides.d.ts +4 -0
- package/dist/core/runtime-overrides.js +4 -1
- package/dist/core/runtime-overrides.js.map +1 -1
- package/dist/describer/anthropic.d.ts +22 -2
- package/dist/describer/anthropic.js +42 -110
- package/dist/describer/anthropic.js.map +1 -1
- package/dist/describer/describer.d.ts +21 -4
- package/dist/describer/describer.js +52 -111
- package/dist/describer/describer.js.map +1 -1
- package/dist/describer/gemini.d.ts +1 -2
- package/dist/describer/gemini.js +20 -113
- package/dist/describer/gemini.js.map +1 -1
- package/dist/describer/shared.d.ts +3 -0
- package/dist/describer/shared.js +19 -0
- package/dist/describer/shared.js.map +1 -0
- package/dist/embedder/factory.d.ts +25 -0
- package/dist/embedder/factory.js +25 -0
- package/dist/embedder/factory.js.map +1 -1
- package/dist/embedder/health.d.ts +10 -0
- package/dist/embedder/health.js +26 -14
- package/dist/embedder/health.js.map +1 -1
- package/dist/embedder/http.d.ts +47 -0
- package/dist/embedder/http.js +61 -3
- package/dist/embedder/http.js.map +1 -1
- package/dist/eval/index.d.ts +6 -0
- package/dist/eval/index.js.map +1 -1
- package/dist/eval/run-token-test.js +1 -5
- package/dist/eval/run-token-test.js.map +1 -1
- package/dist/eval/session-logger.d.ts +5 -0
- package/dist/eval/session-logger.js +4 -0
- package/dist/eval/session-logger.js.map +1 -1
- package/dist/eval/storage.d.ts +9 -0
- package/dist/eval/storage.js +9 -5
- package/dist/eval/storage.js.map +1 -1
- package/dist/eval/token-analysis.d.ts +8 -0
- package/dist/eval/token-analysis.js +6 -2
- package/dist/eval/token-analysis.js.map +1 -1
- package/dist/eval/token-counter.d.ts +1 -3
- package/dist/eval/token-counter.js +4 -5
- package/dist/eval/token-counter.js.map +1 -1
- package/dist/eval/types.d.ts +6 -0
- package/dist/eval/types.js +2 -0
- package/dist/eval/types.js.map +1 -1
- package/dist/index.d.ts +11 -0
- package/dist/index.js +12 -3
- package/dist/index.js.map +1 -1
- package/dist/indexer/description-stage.js +11 -2
- package/dist/indexer/description-stage.js.map +1 -1
- package/dist/indexer/embed-stage.js +0 -8
- package/dist/indexer/embed-stage.js.map +1 -1
- package/dist/indexer/pipeline.d.ts +27 -3
- package/dist/indexer/pipeline.js +314 -143
- package/dist/indexer/pipeline.js.map +1 -1
- package/dist/indexer/stats.d.ts +35 -0
- package/dist/indexer/stats.js +9 -0
- package/dist/indexer/stats.js.map +1 -1
- package/dist/indexer/watch.d.ts +25 -0
- package/dist/indexer/watch.js +21 -0
- package/dist/indexer/watch.js.map +1 -1
- package/dist/indexer/worker.d.ts +90 -2
- package/dist/indexer/worker.js +109 -24
- package/dist/indexer/worker.js.map +1 -1
- package/dist/indexer.d.ts +11 -1
- package/dist/indexer.js +9 -0
- package/dist/indexer.js.map +1 -1
- package/dist/mcp/cli.d.ts +3 -0
- package/dist/mcp/cli.js +1 -0
- package/dist/mcp/cli.js.map +1 -1
- package/dist/mcp/handlers.d.ts +41 -0
- package/dist/mcp/handlers.js +40 -1
- package/dist/mcp/handlers.js.map +1 -1
- package/dist/mcp/server.d.ts +8 -0
- package/dist/mcp/server.js +18 -1
- package/dist/mcp/server.js.map +1 -1
- package/dist/opencode/create-read-tool.js +1 -0
- package/dist/opencode/create-read-tool.js.map +1 -1
- package/dist/opencode/tools.d.ts +8 -17
- package/dist/opencode/tools.js +74 -109
- package/dist/opencode/tools.js.map +1 -1
- package/dist/plugin-entry.d.ts +6 -0
- package/dist/plugin-entry.js +6 -0
- package/dist/plugin-entry.js.map +1 -1
- package/dist/plugin.d.ts +37 -0
- package/dist/plugin.js +265 -40
- package/dist/plugin.js.map +1 -1
- package/dist/retriever/keyword-index.js +9 -9
- package/dist/retriever/keyword-index.js.map +1 -1
- package/dist/retriever/retriever.js +14 -4
- package/dist/retriever/retriever.js.map +1 -1
- package/dist/tui.d.ts +5 -0
- package/dist/tui.js +100 -12
- package/dist/tui.js.map +1 -1
- package/dist/types/opencode-plugin.d.ts +4 -1
- package/dist/updater.d.ts +23 -1
- package/dist/updater.js +55 -16
- package/dist/updater.js.map +1 -1
- package/dist/vectorstore/factory.d.ts +13 -0
- package/dist/vectorstore/factory.js +13 -0
- package/dist/vectorstore/factory.js.map +1 -1
- package/dist/vectorstore/lancedb.d.ts +78 -1
- package/dist/vectorstore/lancedb.js +195 -21
- package/dist/vectorstore/lancedb.js.map +1 -1
- package/dist/vectorstore/memory.d.ts +5 -0
- package/dist/vectorstore/memory.js +11 -0
- package/dist/vectorstore/memory.js.map +1 -1
- package/dist/watcher.d.ts +24 -0
- package/dist/watcher.js +17 -35
- package/dist/watcher.js.map +1 -1
- package/dist/web/api.d.ts +37 -0
- package/dist/web/api.js +54 -0
- package/dist/web/api.js.map +1 -1
- package/dist/web/server.d.ts +15 -0
- package/dist/web/server.js +14 -0
- package/dist/web/server.js.map +1 -1
- package/dist/web/static.d.ts +8 -0
- package/dist/web/static.js +8 -0
- package/dist/web/static.js.map +1 -1
- package/package.json +11 -15
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anthropic.js","sourceRoot":"","sources":["../../src/describer/anthropic.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,MAAM,MAAM,SAAS,CAAC;AAW7B,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEnE,MAAM,OAAO,4BAA4B;IACtB,MAAM,CAAoB;IAE3C,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,KAAY;QACpC,MAAM,QAAQ,GAAuB;YACnC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,CAAC,KAAK,CAAC,EAAE;SACnD,CAAC;QAEF,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,KAAK,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,MAAe
|
|
1
|
+
{"version":3,"file":"anthropic.js","sourceRoot":"","sources":["../../src/describer/anthropic.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,MAAM,MAAM,SAAS,CAAC;AAW7B,mDAAmD;AACnD,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEnE;;;;;GAKG;AACH,MAAM,OAAO,4BAA4B;IACtB,MAAM,CAAoB;IAE3C;;OAEG;IACH,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,kBAAkB;IAClB,KAAK,CAAC,mBAAmB,CAAC,KAAY;QACpC,MAAM,QAAQ,GAAuB;YACnC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,CAAC,KAAK,CAAC,EAAE;SACnD,CAAC;QAEF,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,KAAK,CAAC,CAAC;IACpE,CAAC;IAED,kBAAkB;IAClB,KAAK,CAAC,yBAAyB,CAAC,MAAe,EAAE,QAAgC;QAC/E,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC;QACtD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,2CAA2C,KAAK,yBAAyB,WAAW,GAAG,CAAC,CAAC;QAErG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;QAClC,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACnB,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;YACxC,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,6BAA6B,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,OAAO,OAAO,EAAE,CAAC,CAAC;YAC3I,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;gBACnD,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC3B,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,8BAA8B,KAAK,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;YACjF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,IAAI,CAAC,wCAAwC,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjL,CAAC;YACD,SAAS,EAAE,CAAC;YACZ,IAAI,SAAS,GAAG,EAAE,KAAK,CAAC,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;gBAChD,OAAO,CAAC,GAAG,CAAC,yBAAyB,SAAS,IAAI,KAAK,EAAE,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC,CAAC,CACH,CACF,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,uCAAuC,MAAM,CAAC,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC;QAC3E,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;;;;OASG;IACK,KAAK,CAAC,WAAW,CACvB,QAA4B,EAC5B,SAAiB;QAEjB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;QACxC,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;QAE9C,MAAM,IAAI,GAA4B;YACpC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;YACxB,UAAU,EAAE,IAAI;YAChB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,gBAAgB,EAAE,CAAC;SAC9I,CAAC;QAEF,MAAM,OAAO,GAA2B;YACtC,WAAW,EAAE,MAAM;YACnB,mBAAmB,EAAE,YAAY;YACjC,cAAc,EAAE,kBAAkB;SACnC,CAAC;QAEF,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;QAC3C,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,IAAI,CAAC;QAE9D,IAAI,SAA4B,CAAC;QACjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC;YACrD,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAC7B,GAAG,OAAO,WAAW,EACrB,IAAI,EACJ,OAAO,EACP,SAAS,EACT,IAAI,CAAC,MAAM,CAAC,KAAK,CAClB,CAAC;YAEF,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;gBAChB,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAsB,CAAC;gBAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC;gBACrC,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;gBACrB,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAChF,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,IAAI,KAAK,CACrB,iCAAiC,QAAQ,CAAC,MAAM,MAAM,IAAI,EAAE,CAC7D,CAAC;YAEF,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;gBACrE,MAAM,KAAK,CAAC;YACd,CAAC;YAED,SAAS,GAAG,KAAK,CAAC;YAClB,MAAM,OAAO,GAAG,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YACxD,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;QAED,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAC9E,CAAC;CACF"}
|
|
@@ -1,12 +1,29 @@
|
|
|
1
1
|
import type { Chunk, DescriptionProvider } from "../core/interfaces.js";
|
|
2
2
|
import type { DescriptionConfig } from "../core/config.js";
|
|
3
|
+
/**
|
|
4
|
+
* Description provider that works with any OpenAI-compatible chat API (including Ollama).
|
|
5
|
+
*
|
|
6
|
+
* Supports Bearer-token authentication, optional proxy, and retry with exponential backoff.
|
|
7
|
+
* For Ollama, uses the `/api/chat` endpoint and sends additional options like `num_ctx` and `think`.
|
|
8
|
+
*/
|
|
3
9
|
export declare class LLMDescriptionProvider implements DescriptionProvider {
|
|
4
10
|
private readonly config;
|
|
11
|
+
/**
|
|
12
|
+
* @param config - Configuration for the LLM provider, including base URL, model, API key, proxy, and retry settings.
|
|
13
|
+
*/
|
|
5
14
|
constructor(config: DescriptionConfig);
|
|
15
|
+
/** @inheritdoc */
|
|
6
16
|
generateDescription(chunk: Chunk): Promise<string>;
|
|
7
|
-
|
|
8
|
-
|
|
17
|
+
/** @inheritdoc */
|
|
18
|
+
generateBatchDescriptions(chunks: Chunk[], logDebug?: (msg: string) => void): Promise<Map<string, string>>;
|
|
19
|
+
/**
|
|
20
|
+
* Sends a chat completion request to the LLM API with retry and exponential backoff.
|
|
21
|
+
* For Ollama, uses the `/api/chat` endpoint with streaming disabled; otherwise uses the standard `/v1/chat/completions` endpoint.
|
|
22
|
+
*
|
|
23
|
+
* @param messages - The conversation messages including system prompt and user content.
|
|
24
|
+
* @param timeoutMs - Request timeout in milliseconds.
|
|
25
|
+
* @returns The trimmed response text extracted from the API response.
|
|
26
|
+
* @throws When all retry attempts are exhausted or the response is empty.
|
|
27
|
+
*/
|
|
9
28
|
private chatRequest;
|
|
10
29
|
}
|
|
11
|
-
export declare function buildBatchUserMessage(chunks: Chunk[]): string;
|
|
12
|
-
export declare function parseBatchResponse(text: string, chunks: Chunk[]): Map<string, string>;
|
|
@@ -1,11 +1,23 @@
|
|
|
1
1
|
import { postJson } from "../embedder/http.js";
|
|
2
|
+
import { buildUserMessage, sleep } from "./shared.js";
|
|
2
3
|
import pLimit from "p-limit";
|
|
4
|
+
/** HTTP status codes that are safe to retry on. */
|
|
3
5
|
const RETRYABLE_STATUSES = new Set([404, 408, 429, 500, 502, 503, 504]);
|
|
6
|
+
/**
|
|
7
|
+
* Description provider that works with any OpenAI-compatible chat API (including Ollama).
|
|
8
|
+
*
|
|
9
|
+
* Supports Bearer-token authentication, optional proxy, and retry with exponential backoff.
|
|
10
|
+
* For Ollama, uses the `/api/chat` endpoint and sends additional options like `num_ctx` and `think`.
|
|
11
|
+
*/
|
|
4
12
|
export class LLMDescriptionProvider {
|
|
5
13
|
config;
|
|
14
|
+
/**
|
|
15
|
+
* @param config - Configuration for the LLM provider, including base URL, model, API key, proxy, and retry settings.
|
|
16
|
+
*/
|
|
6
17
|
constructor(config) {
|
|
7
18
|
this.config = config;
|
|
8
19
|
}
|
|
20
|
+
/** @inheritdoc */
|
|
9
21
|
async generateDescription(chunk) {
|
|
10
22
|
const messages = [
|
|
11
23
|
{ role: "system", content: this.config.systemPrompt },
|
|
@@ -13,54 +25,44 @@ export class LLMDescriptionProvider {
|
|
|
13
25
|
];
|
|
14
26
|
return this.chatRequest(messages, this.config.timeoutMs ?? 60000);
|
|
15
27
|
}
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
}
|
|
21
|
-
const batchMaxChunks = this.config.batchMaxChunks ?? 25;
|
|
22
|
-
const batchConcurrency = this.config.batchConcurrency ?? 3;
|
|
23
|
-
const batches = [];
|
|
24
|
-
for (let i = 0; i < chunks.length; i += batchMaxChunks) {
|
|
25
|
-
batches.push(chunks.slice(i, i + batchMaxChunks));
|
|
26
|
-
}
|
|
28
|
+
/** @inheritdoc */
|
|
29
|
+
async generateBatchDescriptions(chunks, logDebug) {
|
|
30
|
+
const concurrency = this.config.batchConcurrency ?? 3;
|
|
31
|
+
const total = chunks.length;
|
|
32
|
+
process.stderr.write(`\nGenerating descriptions for ${total} chunks via ${this.config.provider}/${this.config.model} (concurrency: ${concurrency})...\n`);
|
|
27
33
|
const result = new Map();
|
|
28
|
-
const
|
|
29
|
-
|
|
34
|
+
const limit = pLimit(concurrency);
|
|
35
|
+
let completed = 0;
|
|
36
|
+
await Promise.all(chunks.map((chunk) => limit(async () => {
|
|
37
|
+
const userMsg = buildUserMessage(chunk);
|
|
38
|
+
(logDebug ?? console.debug)(`[describer] REQUEST chunk ${chunk.id} (${chunk.metadata.filePath}:${chunk.metadata.startLine}):\n${userMsg}`);
|
|
30
39
|
try {
|
|
31
|
-
const
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
40
|
+
const desc = await this.generateDescription(chunk);
|
|
41
|
+
result.set(chunk.id, desc);
|
|
42
|
+
(logDebug ?? console.debug)(`[describer] RESPONSE chunk ${chunk.id}: ${desc}`);
|
|
43
|
+
}
|
|
44
|
+
catch (err) {
|
|
45
|
+
console.warn(`[describer] Failed to describe chunk ${chunk.id} (${chunk.metadata.filePath}:${chunk.metadata.startLine}): ${err instanceof Error ? err.message : String(err)}`);
|
|
35
46
|
}
|
|
36
|
-
|
|
37
|
-
|
|
47
|
+
completed++;
|
|
48
|
+
if (completed % 25 === 0 || completed === total) {
|
|
49
|
+
process.stderr.write(`\rDescriptions: ${completed}/${total}`);
|
|
50
|
+
(logDebug ?? console.debug)(`[describer] Progress: ${completed}/${total}`);
|
|
38
51
|
}
|
|
39
52
|
})));
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
const fallbackLimit = pLimit(batchConcurrency);
|
|
43
|
-
await Promise.all(missingChunks.map((chunk) => fallbackLimit(async () => {
|
|
44
|
-
try {
|
|
45
|
-
const desc = await this.generateDescription(chunk);
|
|
46
|
-
result.set(chunk.id, desc);
|
|
47
|
-
}
|
|
48
|
-
catch {
|
|
49
|
-
// skip — caller will fall back to content
|
|
50
|
-
}
|
|
51
|
-
})));
|
|
52
|
-
}
|
|
53
|
+
process.stderr.write(`\rDescriptions: ${result.size}/${total} done.\n`);
|
|
54
|
+
(logDebug ?? console.debug)(`[describer] Descriptions generated: ${result.size}/${total}`);
|
|
53
55
|
return result;
|
|
54
56
|
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
57
|
+
/**
|
|
58
|
+
* Sends a chat completion request to the LLM API with retry and exponential backoff.
|
|
59
|
+
* For Ollama, uses the `/api/chat` endpoint with streaming disabled; otherwise uses the standard `/v1/chat/completions` endpoint.
|
|
60
|
+
*
|
|
61
|
+
* @param messages - The conversation messages including system prompt and user content.
|
|
62
|
+
* @param timeoutMs - Request timeout in milliseconds.
|
|
63
|
+
* @returns The trimmed response text extracted from the API response.
|
|
64
|
+
* @throws When all retry attempts are exhausted or the response is empty.
|
|
65
|
+
*/
|
|
64
66
|
async chatRequest(messages, timeoutMs) {
|
|
65
67
|
const baseUrl = this.config.baseUrl.replace(/\/+$/, "");
|
|
66
68
|
const isOllama = this.config.provider === "ollama";
|
|
@@ -95,76 +97,15 @@ export class LLMDescriptionProvider {
|
|
|
95
97
|
throw lastError ?? new Error("Description LLM request failed: unknown error");
|
|
96
98
|
}
|
|
97
99
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
parts.push(`Language: ${chunk.metadata.language}`);
|
|
108
|
-
}
|
|
109
|
-
parts.push(`Lines: ${chunk.metadata.startLine}-${chunk.metadata.endLine}`);
|
|
110
|
-
parts.push("");
|
|
111
|
-
parts.push("```" + (chunk.metadata.language || ""));
|
|
112
|
-
parts.push(chunk.content);
|
|
113
|
-
parts.push("```");
|
|
114
|
-
return parts.join("\n");
|
|
115
|
-
}
|
|
116
|
-
export function buildBatchUserMessage(chunks) {
|
|
117
|
-
const first = chunks[0];
|
|
118
|
-
const parts = [];
|
|
119
|
-
if (first.metadata.filePath) {
|
|
120
|
-
parts.push(`File: ${first.metadata.filePath}`);
|
|
121
|
-
}
|
|
122
|
-
if (first.metadata.language) {
|
|
123
|
-
parts.push(`Language: ${first.metadata.language}`);
|
|
124
|
-
}
|
|
125
|
-
parts.push(`Chunks: ${chunks.length}`);
|
|
126
|
-
parts.push("");
|
|
127
|
-
for (let i = 0; i < chunks.length; i++) {
|
|
128
|
-
const chunk = chunks[i];
|
|
129
|
-
const lang = chunk.metadata.language || "";
|
|
130
|
-
parts.push(`=== CHUNK ${i} (lines ${chunk.metadata.startLine}-${chunk.metadata.endLine}) ===`);
|
|
131
|
-
parts.push("```" + lang);
|
|
132
|
-
parts.push(chunk.content);
|
|
133
|
-
parts.push("```");
|
|
134
|
-
parts.push("");
|
|
135
|
-
}
|
|
136
|
-
return parts.join("\n");
|
|
137
|
-
}
|
|
138
|
-
export function parseBatchResponse(text, chunks) {
|
|
139
|
-
const result = new Map();
|
|
140
|
-
const chunkPattern = /^CHUNK\s+(\d+)\s*[):-]?\s*/i;
|
|
141
|
-
const lines = text.split("\n");
|
|
142
|
-
let currentIndex = null;
|
|
143
|
-
let currentDesc = [];
|
|
144
|
-
for (const line of lines) {
|
|
145
|
-
const match = chunkPattern.exec(line.trim());
|
|
146
|
-
if (match) {
|
|
147
|
-
if (currentIndex !== null && currentDesc.length > 0) {
|
|
148
|
-
const desc = currentDesc.join(" ").trim();
|
|
149
|
-
if (desc.length > 0 && currentIndex >= 0 && currentIndex < chunks.length) {
|
|
150
|
-
result.set(chunks[currentIndex].id, desc);
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
currentIndex = parseInt(match[1], 10);
|
|
154
|
-
currentDesc = [line.slice(match[0].length).trim()];
|
|
155
|
-
}
|
|
156
|
-
else if (currentIndex !== null) {
|
|
157
|
-
currentDesc.push(line.trim());
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
if (currentIndex !== null && currentDesc.length > 0) {
|
|
161
|
-
const desc = currentDesc.join(" ").trim();
|
|
162
|
-
if (desc.length > 0 && currentIndex >= 0 && currentIndex < chunks.length) {
|
|
163
|
-
result.set(chunks[currentIndex].id, desc);
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
return result;
|
|
167
|
-
}
|
|
100
|
+
/**
|
|
101
|
+
* Extracts the response text from a chat completion response.
|
|
102
|
+
* For Ollama, reads from `message.content`; for OpenAI-compatible APIs, reads from `choices[0].message.content`.
|
|
103
|
+
*
|
|
104
|
+
* @param json - The parsed chat response object.
|
|
105
|
+
* @param isOllama - Whether the response is from an Ollama API (different response shape).
|
|
106
|
+
* @returns The trimmed response text.
|
|
107
|
+
* @throws If the response contains no usable text content.
|
|
108
|
+
*/
|
|
168
109
|
function extractResponseText(json, isOllama) {
|
|
169
110
|
if (isOllama) {
|
|
170
111
|
const content = json.message?.content;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"describer.js","sourceRoot":"","sources":["../../src/describer/describer.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,MAAM,MAAM,SAAS,CAAC;AAY7B,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAExE,MAAM,OAAO,sBAAsB;IAChB,MAAM,CAAoB;IAE3C,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,KAAY;QACpC,MAAM,QAAQ,GAAkB;YAC9B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE;YACrD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,CAAC,KAAK,CAAC,EAAE;SACnD,CAAC;QAEF,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,KAAK,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,MAAe
|
|
1
|
+
{"version":3,"file":"describer.js","sourceRoot":"","sources":["../../src/describer/describer.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,MAAM,MAAM,SAAS,CAAC;AAY7B,mDAAmD;AACnD,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAExE;;;;;GAKG;AACH,MAAM,OAAO,sBAAsB;IAChB,MAAM,CAAoB;IAE3C;;OAEG;IACH,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,kBAAkB;IAClB,KAAK,CAAC,mBAAmB,CAAC,KAAY;QACpC,MAAM,QAAQ,GAAkB;YAC9B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE;YACrD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,CAAC,KAAK,CAAC,EAAE;SACnD,CAAC;QAEF,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,KAAK,CAAC,CAAC;IACpE,CAAC;IAED,kBAAkB;IAClB,KAAK,CAAC,yBAAyB,CAAC,MAAe,EAAE,QAAgC;QAC/E,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC;QACtD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,KAAK,eAAe,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,kBAAkB,WAAW,QAAQ,CAAC,CAAC;QAC1J,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;QAClC,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACnB,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;YACxC,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,6BAA6B,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,OAAO,OAAO,EAAE,CAAC,CAAC;YAC3I,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;gBACnD,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC3B,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,8BAA8B,KAAK,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;YACjF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,IAAI,CAAC,wCAAwC,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjL,CAAC;YACD,SAAS,EAAE,CAAC;YACZ,IAAI,SAAS,GAAG,EAAE,KAAK,CAAC,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;gBAChD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,SAAS,IAAI,KAAK,EAAE,CAAC,CAAC;gBAC9D,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,yBAAyB,SAAS,IAAI,KAAK,EAAE,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC,CAAC,CACH,CACF,CAAC;QAEF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,MAAM,CAAC,IAAI,IAAI,KAAK,UAAU,CAAC,CAAC;QACxE,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,uCAAuC,MAAM,CAAC,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC;QAC3F,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,WAAW,CACvB,QAAuB,EACvB,SAAiB;QAEjB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC;QAEnD,MAAM,GAAG,GAAG,QAAQ;YAClB,CAAC,CAAC,GAAG,OAAO,OAAO;YACnB,CAAC,CAAC,GAAG,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,mBAAmB,CAAC;QAEzE,MAAM,IAAI,GAAG,QAAQ;YACnB,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,KAAK,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE;YACpI,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC;QAE3C,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACvB,OAAO,CAAC,aAAa,GAAG,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACzD,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;QAC3C,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,IAAI,CAAC;QAE9D,IAAI,SAA4B,CAAC;QACjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC;YACrD,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAElF,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;gBAChB,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAiB,CAAC;gBACrD,OAAO,mBAAmB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,IAAI,KAAK,CACrB,mCAAmC,QAAQ,CAAC,MAAM,MAAM,IAAI,EAAE,CAC/D,CAAC;YAEF,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;gBACrE,MAAM,KAAK,CAAC;YACd,CAAC;YAED,SAAS,GAAG,KAAK,CAAC;YAClB,MAAM,OAAO,GAAG,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YACxD,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;QAED,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IAChF,CAAC;CACF;AAID;;;;;;;;GAQG;AACH,SAAS,mBAAmB,CAAC,IAAkB,EAAE,QAAiB;IAChE,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC;QACtC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7D,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;QACxB,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;IACpD,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7D,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC;IAED,MAAM,IAAI,KAAK,CACb,4CAA4C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CACnE,CAAC;AACJ,CAAC"}
|
|
@@ -4,7 +4,6 @@ export declare class GeminiDescriptionProvider implements DescriptionProvider {
|
|
|
4
4
|
private readonly config;
|
|
5
5
|
constructor(config: DescriptionConfig);
|
|
6
6
|
generateDescription(chunk: Chunk): Promise<string>;
|
|
7
|
-
generateBatchDescriptions(chunks: Chunk[]): Promise<Map<string, string>>;
|
|
8
|
-
private executeBatch;
|
|
7
|
+
generateBatchDescriptions(chunks: Chunk[], logDebug?: (msg: string) => void): Promise<Map<string, string>>;
|
|
9
8
|
private chatRequest;
|
|
10
9
|
}
|
package/dist/describer/gemini.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { postJson } from "../embedder/http.js";
|
|
2
|
+
import { buildUserMessage, sleep } from "./shared.js";
|
|
2
3
|
import pLimit from "p-limit";
|
|
3
4
|
const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]);
|
|
4
5
|
export class GeminiDescriptionProvider {
|
|
@@ -15,56 +16,32 @@ export class GeminiDescriptionProvider {
|
|
|
15
16
|
];
|
|
16
17
|
return this.chatRequest(contents, this.config.timeoutMs ?? 60000);
|
|
17
18
|
}
|
|
18
|
-
async generateBatchDescriptions(chunks) {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
}
|
|
23
|
-
const batchMaxChunks = this.config.batchMaxChunks ?? 25;
|
|
24
|
-
const batchConcurrency = this.config.batchConcurrency ?? 3;
|
|
25
|
-
const batches = [];
|
|
26
|
-
for (let i = 0; i < chunks.length; i += batchMaxChunks) {
|
|
27
|
-
batches.push(chunks.slice(i, i + batchMaxChunks));
|
|
28
|
-
}
|
|
19
|
+
async generateBatchDescriptions(chunks, logDebug) {
|
|
20
|
+
const concurrency = this.config.batchConcurrency ?? 3;
|
|
21
|
+
const total = chunks.length;
|
|
22
|
+
console.log(`[describer] Generating descriptions for ${total} chunks (concurrency: ${concurrency})`);
|
|
29
23
|
const result = new Map();
|
|
30
|
-
const
|
|
31
|
-
|
|
24
|
+
const limit = pLimit(concurrency);
|
|
25
|
+
let completed = 0;
|
|
26
|
+
await Promise.all(chunks.map((chunk) => limit(async () => {
|
|
27
|
+
const userMsg = buildUserMessage(chunk);
|
|
28
|
+
(logDebug ?? console.debug)(`[describer] REQUEST chunk ${chunk.id} (${chunk.metadata.filePath}:${chunk.metadata.startLine}):\n${userMsg}`);
|
|
32
29
|
try {
|
|
33
|
-
const
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
30
|
+
const desc = await this.generateDescription(chunk);
|
|
31
|
+
result.set(chunk.id, desc);
|
|
32
|
+
(logDebug ?? console.debug)(`[describer] RESPONSE chunk ${chunk.id}: ${desc}`);
|
|
33
|
+
}
|
|
34
|
+
catch (err) {
|
|
35
|
+
console.warn(`[describer] Failed to describe chunk ${chunk.id} (${chunk.metadata.filePath}:${chunk.metadata.startLine}): ${err instanceof Error ? err.message : String(err)}`);
|
|
37
36
|
}
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
completed++;
|
|
38
|
+
if (completed % 25 === 0 || completed === total) {
|
|
39
|
+
console.log(`[describer] Progress: ${completed}/${total}`);
|
|
40
40
|
}
|
|
41
41
|
})));
|
|
42
|
-
|
|
43
|
-
if (missingChunks.length > 0) {
|
|
44
|
-
const fallbackLimit = pLimit(batchConcurrency);
|
|
45
|
-
await Promise.all(missingChunks.map((chunk) => fallbackLimit(async () => {
|
|
46
|
-
try {
|
|
47
|
-
const desc = await this.generateDescription(chunk);
|
|
48
|
-
result.set(chunk.id, desc);
|
|
49
|
-
}
|
|
50
|
-
catch {
|
|
51
|
-
// skip — caller will fall back to content
|
|
52
|
-
}
|
|
53
|
-
})));
|
|
54
|
-
}
|
|
42
|
+
console.log(`[describer] Descriptions generated: ${result.size}/${total}`);
|
|
55
43
|
return result;
|
|
56
44
|
}
|
|
57
|
-
async executeBatch(chunks) {
|
|
58
|
-
const contents = [
|
|
59
|
-
{
|
|
60
|
-
role: "user",
|
|
61
|
-
parts: [{ text: buildBatchUserMessage(chunks) }],
|
|
62
|
-
},
|
|
63
|
-
];
|
|
64
|
-
const timeoutMs = this.config.batchTimeoutMs ?? 120000;
|
|
65
|
-
const responseText = await this.chatRequest(contents, timeoutMs);
|
|
66
|
-
return parseBatchResponse(responseText, chunks);
|
|
67
|
-
}
|
|
68
45
|
async chatRequest(contents, timeoutMs) {
|
|
69
46
|
const baseUrl = this.config.baseUrl.replace(/\/+$/, "");
|
|
70
47
|
const apiKey = this.config.apiKey ?? "";
|
|
@@ -108,74 +85,4 @@ export class GeminiDescriptionProvider {
|
|
|
108
85
|
throw lastError ?? new Error("Gemini LLM request failed: unknown error");
|
|
109
86
|
}
|
|
110
87
|
}
|
|
111
|
-
function sleep(ms) {
|
|
112
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
113
|
-
}
|
|
114
|
-
function buildUserMessage(chunk) {
|
|
115
|
-
const parts = [];
|
|
116
|
-
if (chunk.metadata.filePath) {
|
|
117
|
-
parts.push(`File: ${chunk.metadata.filePath}`);
|
|
118
|
-
}
|
|
119
|
-
if (chunk.metadata.language) {
|
|
120
|
-
parts.push(`Language: ${chunk.metadata.language}`);
|
|
121
|
-
}
|
|
122
|
-
parts.push(`Lines: ${chunk.metadata.startLine}-${chunk.metadata.endLine}`);
|
|
123
|
-
parts.push("");
|
|
124
|
-
parts.push("```" + (chunk.metadata.language || ""));
|
|
125
|
-
parts.push(chunk.content);
|
|
126
|
-
parts.push("```");
|
|
127
|
-
return parts.join("\n");
|
|
128
|
-
}
|
|
129
|
-
function buildBatchUserMessage(chunks) {
|
|
130
|
-
const first = chunks[0];
|
|
131
|
-
const parts = [];
|
|
132
|
-
if (first.metadata.filePath) {
|
|
133
|
-
parts.push(`File: ${first.metadata.filePath}`);
|
|
134
|
-
}
|
|
135
|
-
if (first.metadata.language) {
|
|
136
|
-
parts.push(`Language: ${first.metadata.language}`);
|
|
137
|
-
}
|
|
138
|
-
parts.push(`Chunks: ${chunks.length}`);
|
|
139
|
-
parts.push("");
|
|
140
|
-
for (let i = 0; i < chunks.length; i++) {
|
|
141
|
-
const chunk = chunks[i];
|
|
142
|
-
const lang = chunk.metadata.language || "";
|
|
143
|
-
parts.push(`=== CHUNK ${i} (lines ${chunk.metadata.startLine}-${chunk.metadata.endLine}) ===`);
|
|
144
|
-
parts.push("```" + lang);
|
|
145
|
-
parts.push(chunk.content);
|
|
146
|
-
parts.push("```");
|
|
147
|
-
parts.push("");
|
|
148
|
-
}
|
|
149
|
-
return parts.join("\n");
|
|
150
|
-
}
|
|
151
|
-
function parseBatchResponse(text, chunks) {
|
|
152
|
-
const result = new Map();
|
|
153
|
-
const chunkPattern = /^CHUNK\s+(\d+)\s*[):-]?\s*/i;
|
|
154
|
-
const lines = text.split("\n");
|
|
155
|
-
let currentIndex = null;
|
|
156
|
-
let currentDesc = [];
|
|
157
|
-
for (const line of lines) {
|
|
158
|
-
const match = chunkPattern.exec(line.trim());
|
|
159
|
-
if (match) {
|
|
160
|
-
if (currentIndex !== null && currentDesc.length > 0) {
|
|
161
|
-
const desc = currentDesc.join(" ").trim();
|
|
162
|
-
if (desc.length > 0 && currentIndex >= 0 && currentIndex < chunks.length) {
|
|
163
|
-
result.set(chunks[currentIndex].id, desc);
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
currentIndex = parseInt(match[1], 10);
|
|
167
|
-
currentDesc = [line.slice(match[0].length).trim()];
|
|
168
|
-
}
|
|
169
|
-
else if (currentIndex !== null) {
|
|
170
|
-
currentDesc.push(line.trim());
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
if (currentIndex !== null && currentDesc.length > 0) {
|
|
174
|
-
const desc = currentDesc.join(" ").trim();
|
|
175
|
-
if (desc.length > 0 && currentIndex >= 0 && currentIndex < chunks.length) {
|
|
176
|
-
result.set(chunks[currentIndex].id, desc);
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
return result;
|
|
180
|
-
}
|
|
181
88
|
//# sourceMappingURL=gemini.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gemini.js","sourceRoot":"","sources":["../../src/describer/gemini.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,MAAM,MAAM,SAAS,CAAC;AAe7B,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEnE,MAAM,OAAO,yBAAyB;IACnB,MAAM,CAAoB;IAE3C,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,KAAY;QACpC,MAAM,QAAQ,GAAoB;YAChC;gBACE,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;aAC3C;SACF,CAAC;QAEF,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,KAAK,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,MAAe
|
|
1
|
+
{"version":3,"file":"gemini.js","sourceRoot":"","sources":["../../src/describer/gemini.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,MAAM,MAAM,SAAS,CAAC;AAe7B,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEnE,MAAM,OAAO,yBAAyB;IACnB,MAAM,CAAoB;IAE3C,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,KAAY;QACpC,MAAM,QAAQ,GAAoB;YAChC;gBACE,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;aAC3C;SACF,CAAC;QAEF,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,KAAK,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,MAAe,EAAE,QAAgC;QAC/E,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC;QACtD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,2CAA2C,KAAK,yBAAyB,WAAW,GAAG,CAAC,CAAC;QAErG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;QAClC,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACnB,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;YACxC,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,6BAA6B,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,OAAO,OAAO,EAAE,CAAC,CAAC;YAC3I,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;gBACnD,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC3B,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,8BAA8B,KAAK,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;YACjF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,IAAI,CAAC,wCAAwC,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACjL,CAAC;YACD,SAAS,EAAE,CAAC;YACZ,IAAI,SAAS,GAAG,EAAE,KAAK,CAAC,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;gBAChD,OAAO,CAAC,GAAG,CAAC,yBAAyB,SAAS,IAAI,KAAK,EAAE,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC,CAAC,CACH,CACF,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,uCAAuC,MAAM,CAAC,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC;QAC3E,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,QAAyB,EACzB,SAAiB;QAEjB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAChC,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;QAE9C,MAAM,QAAQ,GAA4B,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC;QACnE,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;QAED,MAAM,IAAI,GAA4B;YACpC,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;SAC9C,CAAC;QAEF,MAAM,OAAO,GAA2B;YACtC,cAAc,EAAE,kBAAkB;SACnC,CAAC;QAEF,MAAM,GAAG,GAAG,MAAM;YAChB,CAAC,CAAC,GAAG,OAAO,WAAW,KAAK,wBAAwB,MAAM,EAAE;YAC5D,CAAC,CAAC,GAAG,OAAO,WAAW,KAAK,kBAAkB,CAAC;QAEjD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;QAC3C,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,IAAI,CAAC;QAE9D,IAAI,SAA4B,CAAC;QACjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC;YACrD,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;YAE/D,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;gBAChB,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAmB,CAAC;gBACvD,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC;gBAC7D,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;gBACrB,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC7E,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,IAAI,KAAK,CACrB,8BAA8B,QAAQ,CAAC,MAAM,MAAM,IAAI,EAAE,CAC1D,CAAC;YAEF,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;gBACrE,MAAM,KAAK,CAAC;YACd,CAAC;YAED,SAAS,GAAG,KAAK,CAAC;YAClB,MAAM,OAAO,GAAG,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YACxD,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;QAED,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC3E,CAAC;CACF"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export function buildUserMessage(chunk) {
|
|
2
|
+
const parts = [];
|
|
3
|
+
if (chunk.metadata.filePath) {
|
|
4
|
+
parts.push(`File: ${chunk.metadata.filePath}`);
|
|
5
|
+
}
|
|
6
|
+
if (chunk.metadata.language) {
|
|
7
|
+
parts.push(`Language: ${chunk.metadata.language}`);
|
|
8
|
+
}
|
|
9
|
+
parts.push(`Lines: ${chunk.metadata.startLine}-${chunk.metadata.endLine}`);
|
|
10
|
+
parts.push("");
|
|
11
|
+
parts.push("```" + (chunk.metadata.language || ""));
|
|
12
|
+
parts.push(chunk.content);
|
|
13
|
+
parts.push("```");
|
|
14
|
+
return parts.join("\n");
|
|
15
|
+
}
|
|
16
|
+
export function sleep(ms) {
|
|
17
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=shared.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/describer/shared.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,gBAAgB,CAAC,KAAY;IAC3C,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,SAAS,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,aAAa,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,QAAQ,CAAC,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;IACpD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC1B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAElB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"}
|
|
@@ -1,4 +1,29 @@
|
|
|
1
1
|
import type { EmbeddingProvider } from "../core/interfaces.js";
|
|
2
2
|
import type { RagConfig } from "../core/config.js";
|
|
3
|
+
/**
|
|
4
|
+
* Create an embedding provider instance based on the application configuration.
|
|
5
|
+
*
|
|
6
|
+
* Dispatches to the correct provider class (Ollama, Cohere, or OpenAI-compatible)
|
|
7
|
+
* depending on the `provider` field in `config.embedding`. Throws if the provider
|
|
8
|
+
* is unknown or if a required API key is missing.
|
|
9
|
+
*
|
|
10
|
+
* @param config - Full application configuration, including embedding settings
|
|
11
|
+
* @returns An initialized EmbeddingProvider instance
|
|
12
|
+
* @throws If the provider is unsupported or a required apiKey is not set
|
|
13
|
+
*/
|
|
3
14
|
export declare function createEmbedder(config: RagConfig): EmbeddingProvider;
|
|
15
|
+
/**
|
|
16
|
+
* Embed a list of texts in batches with optional concurrency control.
|
|
17
|
+
*
|
|
18
|
+
* Splits the input texts into chunks of `batchSize` and embeds them sequentially
|
|
19
|
+
* (or concurrently when `concurrency > 1`). When concurrency is limited, uses
|
|
20
|
+
* `p-limit` to cap the number of in-flight requests.
|
|
21
|
+
*
|
|
22
|
+
* @param embedder - The embedding provider to use
|
|
23
|
+
* @param texts - Array of text strings to embed
|
|
24
|
+
* @param batchSize - Number of texts per batch (default 10)
|
|
25
|
+
* @param purpose - Optional hint for query vs. document embedding
|
|
26
|
+
* @param concurrency - Maximum number of concurrent batch requests (default 1)
|
|
27
|
+
* @returns A promise resolving to a flat array of embedding vectors (one per input text)
|
|
28
|
+
*/
|
|
4
29
|
export declare function embedBatch(embedder: EmbeddingProvider, texts: string[], batchSize?: number, purpose?: "query" | "document", concurrency?: number): Promise<number[][]>;
|
package/dist/embedder/factory.js
CHANGED
|
@@ -3,6 +3,17 @@ import { OllamaProvider } from "./ollama.js";
|
|
|
3
3
|
import { OpenAIProvider } from "./openai.js";
|
|
4
4
|
import { CohereProvider } from "./cohere.js";
|
|
5
5
|
import pLimit from "p-limit";
|
|
6
|
+
/**
|
|
7
|
+
* Create an embedding provider instance based on the application configuration.
|
|
8
|
+
*
|
|
9
|
+
* Dispatches to the correct provider class (Ollama, Cohere, or OpenAI-compatible)
|
|
10
|
+
* depending on the `provider` field in `config.embedding`. Throws if the provider
|
|
11
|
+
* is unknown or if a required API key is missing.
|
|
12
|
+
*
|
|
13
|
+
* @param config - Full application configuration, including embedding settings
|
|
14
|
+
* @returns An initialized EmbeddingProvider instance
|
|
15
|
+
* @throws If the provider is unsupported or a required apiKey is not set
|
|
16
|
+
*/
|
|
6
17
|
export function createEmbedder(config) {
|
|
7
18
|
const { provider, baseUrl, model, apiKey, proxy, timeoutMs } = config.embedding;
|
|
8
19
|
const effectiveTimeoutMs = timeoutMs ?? 30000;
|
|
@@ -23,6 +34,20 @@ export function createEmbedder(config) {
|
|
|
23
34
|
}
|
|
24
35
|
throw new Error(`Unknown embedding provider: ${provider}`);
|
|
25
36
|
}
|
|
37
|
+
/**
|
|
38
|
+
* Embed a list of texts in batches with optional concurrency control.
|
|
39
|
+
*
|
|
40
|
+
* Splits the input texts into chunks of `batchSize` and embeds them sequentially
|
|
41
|
+
* (or concurrently when `concurrency > 1`). When concurrency is limited, uses
|
|
42
|
+
* `p-limit` to cap the number of in-flight requests.
|
|
43
|
+
*
|
|
44
|
+
* @param embedder - The embedding provider to use
|
|
45
|
+
* @param texts - Array of text strings to embed
|
|
46
|
+
* @param batchSize - Number of texts per batch (default 10)
|
|
47
|
+
* @param purpose - Optional hint for query vs. document embedding
|
|
48
|
+
* @param concurrency - Maximum number of concurrent batch requests (default 1)
|
|
49
|
+
* @returns A promise resolving to a flat array of embedding vectors (one per input text)
|
|
50
|
+
*/
|
|
26
51
|
export async function embedBatch(embedder, texts, batchSize = 10, purpose, concurrency = 1) {
|
|
27
52
|
if (texts.length === 0)
|
|
28
53
|
return [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"factory.js","sourceRoot":"","sources":["../../src/embedder/factory.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,MAAM,MAAM,SAAS,CAAC;AAE7B,MAAM,UAAU,cAAc,CAAC,MAAiB;IAC9C,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC;IAChF,MAAM,kBAAkB,GAAG,SAAS,IAAI,KAAK,CAAC;IAE9C,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1B,OAAO,IAAI,cAAc,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACrG,CAAC;IAED,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1B,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QACD,OAAO,IAAI,cAAc,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,CAAC,CAAC;IAC/E,CAAC;IAED,IAAI,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,8BAA8B,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,IAAI,cAAc,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,QAA2B,EAC3B,KAAe,EACf,YAAoB,EAAE,EACtB,OAA8B,EAC9B,cAAsB,CAAC;IAEvB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAElC,MAAM,OAAO,GAAyC,EAAE,CAAC;IACzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACjD,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,IAAI,WAAW,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAC9D,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;QAC9B,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;IAClC,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACpB,KAAK,CAAC,KAAK,IAAI,EAAE;QACf,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC9D,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC;IAC5C,CAAC,CAAC,CACH,CACF,CAAC;IAEF,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,YAAY,EAAE,CAAC;QAC1C,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
1
|
+
{"version":3,"file":"factory.js","sourceRoot":"","sources":["../../src/embedder/factory.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,MAAM,MAAM,SAAS,CAAC;AAE7B;;;;;;;;;;GAUG;AACH,MAAM,UAAU,cAAc,CAAC,MAAiB;IAC9C,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC;IAChF,MAAM,kBAAkB,GAAG,SAAS,IAAI,KAAK,CAAC;IAE9C,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1B,OAAO,IAAI,cAAc,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACrG,CAAC;IAED,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1B,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QACD,OAAO,IAAI,cAAc,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,CAAC,CAAC;IAC/E,CAAC;IAED,IAAI,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,8BAA8B,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,IAAI,cAAc,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAC;AAC7D,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,QAA2B,EAC3B,KAAe,EACf,YAAoB,EAAE,EACtB,OAA8B,EAC9B,cAAsB,CAAC;IAEvB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAElC,MAAM,OAAO,GAAyC,EAAE,CAAC;IACzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACjD,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,IAAI,WAAW,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAC9D,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;QAC9B,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;IAClC,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACpB,KAAK,CAAC,KAAK,IAAI,EAAE;QACf,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC9D,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC;IAC5C,CAAC,CAAC,CACH,CACF,CAAC;IAEF,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,YAAY,EAAE,CAAC;QAC1C,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|