abb-opencode-local-rag 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.de.md +416 -0
- package/README.es.md +416 -0
- package/README.fr.md +416 -0
- package/README.md +491 -0
- package/README.pt-BR.md +416 -0
- package/README.zh-CN.md +416 -0
- package/dist/bin/install-skills.d.ts +20 -0
- package/dist/bin/install-skills.d.ts.map +1 -0
- package/dist/bin/install-skills.js +195 -0
- package/dist/bin/install-skills.js.map +1 -0
- package/dist/chunker/index.d.ts +24 -0
- package/dist/chunker/index.d.ts.map +1 -0
- package/dist/chunker/index.js +2 -0
- package/dist/chunker/index.js.map +1 -0
- package/dist/chunker/semantic-chunker.d.ts +97 -0
- package/dist/chunker/semantic-chunker.d.ts.map +1 -0
- package/dist/chunker/semantic-chunker.js +294 -0
- package/dist/chunker/semantic-chunker.js.map +1 -0
- package/dist/chunker/sentence-splitter.d.ts +28 -0
- package/dist/chunker/sentence-splitter.d.ts.map +1 -0
- package/dist/chunker/sentence-splitter.js +219 -0
- package/dist/chunker/sentence-splitter.js.map +1 -0
- package/dist/cli/common.d.ts +65 -0
- package/dist/cli/common.d.ts.map +1 -0
- package/dist/cli/common.js +138 -0
- package/dist/cli/common.js.map +1 -0
- package/dist/cli/delete.d.ts +8 -0
- package/dist/cli/delete.d.ts.map +1 -0
- package/dist/cli/delete.js +173 -0
- package/dist/cli/delete.js.map +1 -0
- package/dist/cli/file-collection.d.ts +2 -0
- package/dist/cli/file-collection.d.ts.map +1 -0
- package/dist/cli/file-collection.js +53 -0
- package/dist/cli/file-collection.js.map +1 -0
- package/dist/cli/ingest.d.ts +100 -0
- package/dist/cli/ingest.d.ts.map +1 -0
- package/dist/cli/ingest.js +363 -0
- package/dist/cli/ingest.js.map +1 -0
- package/dist/cli/list.d.ts +35 -0
- package/dist/cli/list.d.ts.map +1 -0
- package/dist/cli/list.js +210 -0
- package/dist/cli/list.js.map +1 -0
- package/dist/cli/options.d.ts +100 -0
- package/dist/cli/options.d.ts.map +1 -0
- package/dist/cli/options.js +241 -0
- package/dist/cli/options.js.map +1 -0
- package/dist/cli/query.d.ts +24 -0
- package/dist/cli/query.d.ts.map +1 -0
- package/dist/cli/query.js +191 -0
- package/dist/cli/query.js.map +1 -0
- package/dist/cli/read-neighbors.d.ts +11 -0
- package/dist/cli/read-neighbors.d.ts.map +1 -0
- package/dist/cli/read-neighbors.js +224 -0
- package/dist/cli/read-neighbors.js.map +1 -0
- package/dist/cli/status.d.ts +8 -0
- package/dist/cli/status.d.ts.map +1 -0
- package/dist/cli/status.js +80 -0
- package/dist/cli/status.js.map +1 -0
- package/dist/cli/sync.d.ts +8 -0
- package/dist/cli/sync.d.ts.map +1 -0
- package/dist/cli/sync.js +244 -0
- package/dist/cli/sync.js.map +1 -0
- package/dist/cli-main.d.ts +12 -0
- package/dist/cli-main.d.ts.map +1 -0
- package/dist/cli-main.js +63 -0
- package/dist/cli-main.js.map +1 -0
- package/dist/embedder/index.d.ts +85 -0
- package/dist/embedder/index.d.ts.map +1 -0
- package/dist/embedder/index.js +284 -0
- package/dist/embedder/index.js.map +1 -0
- package/dist/features/list.d.ts +37 -0
- package/dist/features/list.d.ts.map +1 -0
- package/dist/features/list.js +40 -0
- package/dist/features/list.js.map +1 -0
- package/dist/features/sync.d.ts +207 -0
- package/dist/features/sync.d.ts.map +1 -0
- package/dist/features/sync.js +380 -0
- package/dist/features/sync.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +53 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest/compute.d.ts +86 -0
- package/dist/ingest/compute.d.ts.map +1 -0
- package/dist/ingest/compute.js +177 -0
- package/dist/ingest/compute.js.map +1 -0
- package/dist/ingest/file.d.ts +27 -0
- package/dist/ingest/file.d.ts.map +1 -0
- package/dist/ingest/file.js +67 -0
- package/dist/ingest/file.js.map +1 -0
- package/dist/ingest/visual.d.ts +45 -0
- package/dist/ingest/visual.d.ts.map +1 -0
- package/dist/ingest/visual.js +234 -0
- package/dist/ingest/visual.js.map +1 -0
- package/dist/parser/docx-parser.d.ts +12 -0
- package/dist/parser/docx-parser.d.ts.map +1 -0
- package/dist/parser/docx-parser.js +328 -0
- package/dist/parser/docx-parser.js.map +1 -0
- package/dist/parser/html-parser.d.ts +18 -0
- package/dist/parser/html-parser.d.ts.map +1 -0
- package/dist/parser/html-parser.js +102 -0
- package/dist/parser/html-parser.js.map +1 -0
- package/dist/parser/index.d.ts +214 -0
- package/dist/parser/index.d.ts.map +1 -0
- package/dist/parser/index.js +454 -0
- package/dist/parser/index.js.map +1 -0
- package/dist/parser/pdf-extract.d.ts +81 -0
- package/dist/parser/pdf-extract.d.ts.map +1 -0
- package/dist/parser/pdf-extract.js +112 -0
- package/dist/parser/pdf-extract.js.map +1 -0
- package/dist/parser/pdf-filter.d.ts +117 -0
- package/dist/parser/pdf-filter.d.ts.map +1 -0
- package/dist/parser/pdf-filter.js +528 -0
- package/dist/parser/pdf-filter.js.map +1 -0
- package/dist/parser/title-extractor.d.ts +69 -0
- package/dist/parser/title-extractor.d.ts.map +1 -0
- package/dist/parser/title-extractor.js +145 -0
- package/dist/parser/title-extractor.js.map +1 -0
- package/dist/pdf-visual/captioner.d.ts +16 -0
- package/dist/pdf-visual/captioner.d.ts.map +1 -0
- package/dist/pdf-visual/captioner.js +63 -0
- package/dist/pdf-visual/captioner.js.map +1 -0
- package/dist/pdf-visual/captioners/fast.d.ts +7 -0
- package/dist/pdf-visual/captioners/fast.d.ts.map +1 -0
- package/dist/pdf-visual/captioners/fast.js +103 -0
- package/dist/pdf-visual/captioners/fast.js.map +1 -0
- package/dist/pdf-visual/captioners/quality.d.ts +7 -0
- package/dist/pdf-visual/captioners/quality.d.ts.map +1 -0
- package/dist/pdf-visual/captioners/quality.js +127 -0
- package/dist/pdf-visual/captioners/quality.js.map +1 -0
- package/dist/pdf-visual/captioners/shared.d.ts +44 -0
- package/dist/pdf-visual/captioners/shared.d.ts.map +1 -0
- package/dist/pdf-visual/captioners/shared.js +104 -0
- package/dist/pdf-visual/captioners/shared.js.map +1 -0
- package/dist/pdf-visual/detector.d.ts +9 -0
- package/dist/pdf-visual/detector.d.ts.map +1 -0
- package/dist/pdf-visual/detector.js +234 -0
- package/dist/pdf-visual/detector.js.map +1 -0
- package/dist/pdf-visual/index.d.ts +13 -0
- package/dist/pdf-visual/index.d.ts.map +1 -0
- package/dist/pdf-visual/index.js +45 -0
- package/dist/pdf-visual/index.js.map +1 -0
- package/dist/pdf-visual/renderer.d.ts +9 -0
- package/dist/pdf-visual/renderer.d.ts.map +1 -0
- package/dist/pdf-visual/renderer.js +177 -0
- package/dist/pdf-visual/renderer.js.map +1 -0
- package/dist/pdf-visual/types.d.ts +62 -0
- package/dist/pdf-visual/types.d.ts.map +1 -0
- package/dist/pdf-visual/types.js +32 -0
- package/dist/pdf-visual/types.js.map +1 -0
- package/dist/server/error-utils.d.ts +79 -0
- package/dist/server/error-utils.d.ts.map +1 -0
- package/dist/server/error-utils.js +148 -0
- package/dist/server/error-utils.js.map +1 -0
- package/dist/server/index.d.ts +258 -0
- package/dist/server/index.d.ts.map +1 -0
- package/dist/server/index.js +1104 -0
- package/dist/server/index.js.map +1 -0
- package/dist/server/list-scanner.d.ts +52 -0
- package/dist/server/list-scanner.d.ts.map +1 -0
- package/dist/server/list-scanner.js +72 -0
- package/dist/server/list-scanner.js.map +1 -0
- package/dist/server/tool-definitions.d.ts +8 -0
- package/dist/server/tool-definitions.d.ts.map +1 -0
- package/dist/server/tool-definitions.js +181 -0
- package/dist/server/tool-definitions.js.map +1 -0
- package/dist/server/tool-input.d.ts +37 -0
- package/dist/server/tool-input.d.ts.map +1 -0
- package/dist/server/tool-input.js +216 -0
- package/dist/server/tool-input.js.map +1 -0
- package/dist/server/types.d.ts +331 -0
- package/dist/server/types.d.ts.map +1 -0
- package/dist/server/types.js +3 -0
- package/dist/server/types.js.map +1 -0
- package/dist/server-main.d.ts +46 -0
- package/dist/server-main.d.ts.map +1 -0
- package/dist/server-main.js +242 -0
- package/dist/server-main.js.map +1 -0
- package/dist/utils/base-dirs.d.ts +212 -0
- package/dist/utils/base-dirs.d.ts.map +1 -0
- package/dist/utils/base-dirs.js +422 -0
- package/dist/utils/base-dirs.js.map +1 -0
- package/dist/utils/errors.d.ts +24 -0
- package/dist/utils/errors.d.ts.map +1 -0
- package/dist/utils/errors.js +53 -0
- package/dist/utils/errors.js.map +1 -0
- package/dist/utils/limits.d.ts +26 -0
- package/dist/utils/limits.d.ts.map +1 -0
- package/dist/utils/limits.js +28 -0
- package/dist/utils/limits.js.map +1 -0
- package/dist/utils/list-sources.d.ts +47 -0
- package/dist/utils/list-sources.d.ts.map +1 -0
- package/dist/utils/list-sources.js +50 -0
- package/dist/utils/list-sources.js.map +1 -0
- package/dist/utils/raw-data-utils.d.ts +131 -0
- package/dist/utils/raw-data-utils.d.ts.map +1 -0
- package/dist/utils/raw-data-utils.js +255 -0
- package/dist/utils/raw-data-utils.js.map +1 -0
- package/dist/utils/scan.d.ts +126 -0
- package/dist/utils/scan.d.ts.map +1 -0
- package/dist/utils/scan.js +221 -0
- package/dist/utils/scan.js.map +1 -0
- package/dist/utils/scope-match.d.ts +43 -0
- package/dist/utils/scope-match.d.ts.map +1 -0
- package/dist/utils/scope-match.js +87 -0
- package/dist/utils/scope-match.js.map +1 -0
- package/dist/utils/sensitive-path.d.ts +23 -0
- package/dist/utils/sensitive-path.d.ts.map +1 -0
- package/dist/utils/sensitive-path.js +91 -0
- package/dist/utils/sensitive-path.js.map +1 -0
- package/dist/utils/sync-path-key.d.ts +20 -0
- package/dist/utils/sync-path-key.d.ts.map +1 -0
- package/dist/utils/sync-path-key.js +33 -0
- package/dist/utils/sync-path-key.js.map +1 -0
- package/dist/vectordb/index.d.ts +168 -0
- package/dist/vectordb/index.d.ts.map +1 -0
- package/dist/vectordb/index.js +619 -0
- package/dist/vectordb/index.js.map +1 -0
- package/dist/vectordb/search-filters.d.ts +39 -0
- package/dist/vectordb/search-filters.d.ts.map +1 -0
- package/dist/vectordb/search-filters.js +136 -0
- package/dist/vectordb/search-filters.js.map +1 -0
- package/dist/vectordb/types.d.ts +196 -0
- package/dist/vectordb/types.d.ts.map +1 -0
- package/dist/vectordb/types.js +224 -0
- package/dist/vectordb/types.js.map +1 -0
- package/package.json +105 -0
- package/skills/mcp-local-rag/SKILL.md +308 -0
- package/skills/mcp-local-rag/references/cli-reference.md +175 -0
- package/skills/mcp-local-rag/references/html-ingestion.md +78 -0
- package/skills/mcp-local-rag/references/query-optimization.md +57 -0
- package/skills/mcp-local-rag/references/result-refinement.md +56 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sentence-splitter.js","sourceRoot":"","sources":["../../src/chunker/sentence-splitter.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,sBAAsB;AACtB,6EAA6E;AAI7E,+CAA+C;AAC/C,YAAY;AACZ,+CAA+C;AAE/C;;GAEG;AACH,MAAM,sBAAsB,GAAG,wBAAwB,CAAA;AAEvD;;GAEG;AACH,MAAM,uBAAuB,GAAG,yBAAyB,CAAA;AA+BzD,+CAA+C;AAC/C,mBAAmB;AACnB,+CAA+C;AAE/C,SAAS,QAAQ,CACf,IAAY,EACZ,WAAmB;IAEnB,MAAM,MAAM,GAAoB,EAAE,CAAA;IAClC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACrE,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAC9F,CAAA;IACD,MAAM,aAAa,GAAG,CAAC,GAAG,MAAM,CAAC,CAAA;IACjC,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,KAAK,MAAM,WAAW,IAAI,CAAC,GAAG,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC;QAChF,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,WAAW,CAAC,KAAK,CAAC,CAAA;QACnD,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7C,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;gBAAE,SAAQ;YACvC,aAAa,CAAC,IAAI,CAAC;gBACjB,KAAK,EAAE,QAAQ,GAAG,KAAK,CAAC,KAAK;gBAC7B,GAAG,EAAE,QAAQ,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;aAC9C,CAAC,CAAA;QACJ,CAAC;QACD,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAA;IAC5B,CAAC;IACD,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;IAE7D,IAAI,UAAU,GAAG,EAAE,CAAA;IACnB,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,MAAM,QAAQ,GAAqB,EAAE,CAAA;IACrC,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,KAAa,EAAE,GAAW,EAAE,MAAe,EAAQ,EAAE;QAClF,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAA;QACrC,UAAU,IAAI,KAAK,CAAA;QACnB,QAAQ,CAAC,IAAI,CAAC;YACZ,WAAW;YACX,SAAS,EAAE,UAAU,CAAC,MAAM;YAC5B,WAAW,EAAE,WAAW,GAAG,KAAK;YAChC,SAAS,EAAE,WAAW,GAAG,GAAG;YAC5B,MAAM;SACP,CAAC,CAAA;IACJ,CAAC,CAAA;IACD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;QACrD,IAAI,KAAK,CAAC,KAAK,GAAG,MAAM;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;QAC5F,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;QAClD,MAAM,iBAAiB,GAAG,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC;YACjD,CAAC,CAAC,sBAAsB;YACxB,CAAC,CAAC,uBAAuB,CAAA;QAC3B,MAAM,WAAW,GAAG,GAAG,iBAAiB,GAAG,KAAK,GAAG,iBAAiB,EAAE,CAAA;QACtE,MAAM,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAA;QACrC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QAClD,MAAM,GAAG,KAAK,CAAC,GAAG,CAAA;IACpB,CAAC;IACD,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM;QAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IAC/E,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAA;AAC3D,CAAC;AAED,SAAS,cAAc,CAAC,MAAkB,EAAE,MAAc,EAAE,QAAgB;IAC1E,IAAI,GAAG,GAAG,CAAC,CAAA;IACX,IAAI,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAA;IACrC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;QAC3C,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAmB,CAAA;QACzD,IAAI,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YACjC,IAAI,GAAG,MAAM,GAAG,CAAC,CAAA;QACnB,CAAC;aAAM,IAAI,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;YACtC,GAAG,GAAG,MAAM,GAAG,CAAC,CAAA;QAClB,CAAC;aAAM,IAAI,MAAM,KAAK,OAAO,CAAC,SAAS,EAAE,CAAC;YACxC,OAAO,OAAO,CAAC,SAAS,CAAA;QAC1B,CAAC;aAAM,CAAC;YACN,OAAO,OAAO,CAAC,MAAM;gBACnB,CAAC,CAAC,OAAO,CAAC,WAAW,GAAG,CAAC,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;gBACtD,CAAC,CAAC,OAAO,CAAC,WAAW,CAAA;QACzB,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,MAAuB;IACxD,IAAI,QAAQ,GAAG,IAAI,CAAA;IACnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IACrE,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,KAAa,EAAE,GAAW;IAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IACpC,MAAM,YAAY,GAAG,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,CAAA;IACtE,MAAM,UAAU,GAAG,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,MAAM,CAAA;IACjD,OAAO,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;AACtE,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY,EAAE,WAAmB,EAAE,SAAiB;IAC9E,IAAI,WAAW,IAAI,SAAS;QAAE,OAAO,EAAE,CAAA;IACvC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC,EAAE,WAAW,CAAC,CAAA;IACpF,6GAA6G;IAC7G,MAAM,kBAAkB,GAAG,gCAAgC,CAAA;IAC3D,MAAM,eAAe,GAA4B,EAAE,CAAA;IACnD,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;QAC7D,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,CAAA;QAClC,IAAI,cAAc,KAAK,SAAS;YAAE,SAAQ;QAC1C,eAAe,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAA;QAC9C,MAAM,GAAG,cAAc,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;IAC3C,CAAC;IACD,eAAe,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;IAElD,MAAM,KAAK,GAAmB,EAAE,CAAA;IAChC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,GAAW,EAAQ,EAAE;QACtD,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,CAAA;QACnD,IAAI,CAAC,KAAK;YAAE,OAAM;QAClB,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,GAAG,KAAK,CAAA;QACxC,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAA;QACxF,IAAI,CAAC,QAAQ;YAAE,OAAM;QACrB,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,KAAK;YACb,WAAW,EAAE,cAAc,CAAC,MAAM,EAAE,YAAY,EAAE,WAAW,CAAC;YAC9D,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC;SACzD,CAAC,CAAA;IACJ,CAAC,CAAA;IAED,KAAK,MAAM,CAAC,cAAc,EAAE,YAAY,CAAC,IAAI,eAAe,EAAE,CAAC;QAC7D,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,EAAE,YAAY,CAAC,CAAA;QACrE,IAAI,CAAC,KAAK;YAAE,SAAQ;QACpB,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,GAAG,KAAK,CAAA;QACxC,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;QAC7D,IAAI,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YAChC,UAAU,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;YACpC,SAAQ;QACV,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YACnD,UAAU,CACR,YAAY,GAAG,OAAO,CAAC,KAAK,EAC5B,YAAY,GAAG,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CACtD,CAAA;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,+CAA+C;AAC/C,iCAAiC;AACjC,+CAA+C;AAE/C,4CAA4C;AAC5C,qEAAqE;AACrE,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAA;AAExE;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,CAAA;IACX,CAAC;IACD,OAAO,kBAAkB,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY,EAAE,YAAwC;IAClF,IAAI,WAAW,GAAG,CAAC,CAAA;IAEnB,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,MAAM,YAAY,GAChB,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC;YAC7B,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC;YAC3B,KAAK,CAAC,KAAK,IAAI,CAAC;YAChB,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG;YACvB,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,CAAA;QAC1B,MAAM,wBAAwB,GAAG,KAAK,CAAC,KAAK,IAAI,WAAW,CAAA;QAE3D,IAAI,CAAC,YAAY,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CACb,yBAAyB,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,GAAG,qBAAqB,IAAI,CAAC,MAAM,EAAE,CACrF,CAAA;QACH,CAAC;QACD,WAAW,GAAG,KAAK,CAAC,GAAG,CAAA;IACzB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CACpC,IAAY,EACZ,YAAY,GAA+B,EAAE;IAE7C,oBAAoB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IACxC,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,kBAAkB,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IACjD,CAAC;IAED,MAAM,KAAK,GAAmB,EAAE,CAAA;IAChC,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;QAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;QAC5D,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,yBAAyB,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,GAAG,eAAe,CAAC,CAAA;QACpF,CAAC;QACD,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,UAAU;YAChB,MAAM,EAAE,IAAI;YACZ,WAAW,EAAE,KAAK,CAAC,KAAK;YACxB,SAAS,EAAE,KAAK,CAAC,GAAG;SACrB,CAAC,CAAA;QACF,MAAM,GAAG,KAAK,CAAC,GAAG,CAAA;IACpB,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;IAE5D,OAAO,KAAK,CAAA;AACd,CAAC"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { Embedder } from '../embedder/index.js';
|
|
2
|
+
import { type BaseDirsConfig, type BaseDirsConfigWarning } from '../utils/base-dirs.js';
|
|
3
|
+
import { VectorStore } from '../vectordb/index.js';
|
|
4
|
+
import { type ResolvedGlobalConfig } from './options.js';
|
|
5
|
+
/**
|
|
6
|
+
* Render an unknown caught value for a CLI failure site: every link of the
|
|
7
|
+
* `.cause` chain (via {@link getCauseChain}) followed by its stack, joined so
|
|
8
|
+
* the operator sees the full diagnostic. Deeper links are prefixed
|
|
9
|
+
* `Caused by: `; the outermost link is not.
|
|
10
|
+
*
|
|
11
|
+
* The CLI is operator-facing, so — unlike the MCP client boundary, which never
|
|
12
|
+
* leaks the cause chain — the full chain IS printed (Contract-Delta CLI row:
|
|
13
|
+
* `message` → `message` + cause chain on stderr). Callers keep their own
|
|
14
|
+
* operation prefix and exit-code policy; this function only produces the
|
|
15
|
+
* rendered reason. Single source for the CLI catch-block error-rendering
|
|
16
|
+
* pattern, replacing the former message-only renderer.
|
|
17
|
+
*/
|
|
18
|
+
export declare function formatCliError(error: unknown): string;
|
|
19
|
+
/**
|
|
20
|
+
* Create an uninitialized VectorStore from resolved global config.
|
|
21
|
+
* Callers are responsible for calling initialize() before use.
|
|
22
|
+
*/
|
|
23
|
+
export declare function createVectorStore(config: ResolvedGlobalConfig): VectorStore;
|
|
24
|
+
/**
|
|
25
|
+
* Create an uninitialized Embedder from resolved global config.
|
|
26
|
+
* Callers are responsible for managing the Embedder lifecycle.
|
|
27
|
+
*/
|
|
28
|
+
export declare function createEmbedder(config: ResolvedGlobalConfig): Embedder;
|
|
29
|
+
/**
|
|
30
|
+
* Result of {@link resolveCliBaseDirsOrExit}. Resolution warnings travel with
|
|
31
|
+
* the config so subcommands can render them per their own UI contract (CLI
|
|
32
|
+
* subcommands generally write them to stderr).
|
|
33
|
+
*/
|
|
34
|
+
export interface CliBaseDirsResolution {
|
|
35
|
+
config: BaseDirsConfig;
|
|
36
|
+
warnings: BaseDirsConfigWarning[];
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Resolve effective base directories for a CLI subcommand using the shared
|
|
40
|
+
* resolver, surfacing any configuration error as a process-level failure.
|
|
41
|
+
*
|
|
42
|
+
* Inputs (single source of truth for CLI precedence — kept here so per-
|
|
43
|
+
* subcommand entry points don't each replicate the env-fallback chain):
|
|
44
|
+
* - `cliRoots`: repeated `--base-dir` flag values in CLI order. When non-
|
|
45
|
+
* empty, REPLACES env roots — no merge.
|
|
46
|
+
* - `process.env['BASE_DIRS']`: JSON array, used only when CLI roots are
|
|
47
|
+
* absent.
|
|
48
|
+
* - `process.env['BASE_DIR']`: single path, used only when CLI roots and
|
|
49
|
+
* `BASE_DIRS` are absent.
|
|
50
|
+
* - `process.cwd()`: final fallback.
|
|
51
|
+
*
|
|
52
|
+
* Failure mode: a `BaseDirsConfigError` (invalid `BASE_DIRS` JSON, missing
|
|
53
|
+
* directory, not-a-directory, ...) is reported to stderr and exits with
|
|
54
|
+
* code 1. This is intentional: the resolver explicitly does NOT fall back
|
|
55
|
+
* (see §Technical Decisions → Resolution order in the multi-base-dirs
|
|
56
|
+
* plan), so CLI consumers should fail fast rather than silently degrading
|
|
57
|
+
* to `cwd`.
|
|
58
|
+
*
|
|
59
|
+
* Warnings (`base-dirs-overrides-base-dir`, `nested-root-pruned`) are
|
|
60
|
+
* returned to the caller rather than written here, so each subcommand can
|
|
61
|
+
* decide its own rendering (JSON-output subcommands like `list` may need
|
|
62
|
+
* to keep stderr clean even when warnings are present).
|
|
63
|
+
*/
|
|
64
|
+
export declare function resolveCliBaseDirsOrExit(cliRoots: string[]): Promise<CliBaseDirsResolution>;
|
|
65
|
+
//# sourceMappingURL=common.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../src/cli/common.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAG3B,MAAM,uBAAuB,CAAA;AAG9B,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAAE,KAAK,oBAAoB,EAA6C,MAAM,cAAc,CAAA;AAEnG;;;;;;;;;;;;GAYG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAQrD;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,oBAAoB,GAAG,WAAW,CAK3E;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,oBAAoB,GAAG,QAAQ,CAerE;AAED;;;;GAIG;AACH,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,cAAc,CAAA;IACtB,QAAQ,EAAE,qBAAqB,EAAE,CAAA;CAClC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAsB,wBAAwB,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAqDjG"}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
// Shared CLI component helpers — factory functions for VectorStore and Embedder
|
|
2
|
+
// plus base-directory resolution shared by every subcommand that scans files.
|
|
3
|
+
import { Embedder } from '../embedder/index.js';
|
|
4
|
+
import { parseBaseDirsEnv, resolveBaseDirs, } from '../utils/base-dirs.js';
|
|
5
|
+
import { getCauseChain } from '../utils/errors.js';
|
|
6
|
+
import { checkSensitivePath } from '../utils/sensitive-path.js';
|
|
7
|
+
import { VectorStore } from '../vectordb/index.js';
|
|
8
|
+
import { resolveDevice, resolveDtype, validatePath } from './options.js';
|
|
9
|
+
/**
|
|
10
|
+
* Render an unknown caught value for a CLI failure site: every link of the
|
|
11
|
+
* `.cause` chain (via {@link getCauseChain}) followed by its stack, joined so
|
|
12
|
+
* the operator sees the full diagnostic. Deeper links are prefixed
|
|
13
|
+
* `Caused by: `; the outermost link is not.
|
|
14
|
+
*
|
|
15
|
+
* The CLI is operator-facing, so — unlike the MCP client boundary, which never
|
|
16
|
+
* leaks the cause chain — the full chain IS printed (Contract-Delta CLI row:
|
|
17
|
+
* `message` → `message` + cause chain on stderr). Callers keep their own
|
|
18
|
+
* operation prefix and exit-code policy; this function only produces the
|
|
19
|
+
* rendered reason. Single source for the CLI catch-block error-rendering
|
|
20
|
+
* pattern, replacing the former message-only renderer.
|
|
21
|
+
*/
|
|
22
|
+
export function formatCliError(error) {
|
|
23
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
24
|
+
return getCauseChain(err)
|
|
25
|
+
.map((link, index) => {
|
|
26
|
+
const header = index === 0 ? '' : 'Caused by: ';
|
|
27
|
+
return `${header}${link.stack || `${link.name}: ${link.message}`}`;
|
|
28
|
+
})
|
|
29
|
+
.join('\n');
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Create an uninitialized VectorStore from resolved global config.
|
|
33
|
+
* Callers are responsible for calling initialize() before use.
|
|
34
|
+
*/
|
|
35
|
+
export function createVectorStore(config) {
|
|
36
|
+
return new VectorStore({
|
|
37
|
+
dbPath: config.dbPath,
|
|
38
|
+
tableName: 'chunks',
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Create an uninitialized Embedder from resolved global config.
|
|
43
|
+
* Callers are responsible for managing the Embedder lifecycle.
|
|
44
|
+
*/
|
|
45
|
+
export function createEmbedder(config) {
|
|
46
|
+
const embedderConfig = {
|
|
47
|
+
modelPath: config.modelName,
|
|
48
|
+
batchSize: 16,
|
|
49
|
+
cacheDir: config.cacheDir,
|
|
50
|
+
device: resolveDevice(process.env['RAG_DEVICE']),
|
|
51
|
+
};
|
|
52
|
+
// Set dtype only when RAG_DTYPE resolves to a defined value, mirroring the
|
|
53
|
+
// server path — an unset RAG_DTYPE leaves config.dtype undefined so the
|
|
54
|
+
// embedder applies its fp32 default.
|
|
55
|
+
const dtype = resolveDtype(process.env['RAG_DTYPE']);
|
|
56
|
+
if (dtype !== undefined) {
|
|
57
|
+
embedderConfig.dtype = dtype;
|
|
58
|
+
}
|
|
59
|
+
return new Embedder(embedderConfig);
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Resolve effective base directories for a CLI subcommand using the shared
|
|
63
|
+
* resolver, surfacing any configuration error as a process-level failure.
|
|
64
|
+
*
|
|
65
|
+
* Inputs (single source of truth for CLI precedence — kept here so per-
|
|
66
|
+
* subcommand entry points don't each replicate the env-fallback chain):
|
|
67
|
+
* - `cliRoots`: repeated `--base-dir` flag values in CLI order. When non-
|
|
68
|
+
* empty, REPLACES env roots — no merge.
|
|
69
|
+
* - `process.env['BASE_DIRS']`: JSON array, used only when CLI roots are
|
|
70
|
+
* absent.
|
|
71
|
+
* - `process.env['BASE_DIR']`: single path, used only when CLI roots and
|
|
72
|
+
* `BASE_DIRS` are absent.
|
|
73
|
+
* - `process.cwd()`: final fallback.
|
|
74
|
+
*
|
|
75
|
+
* Failure mode: a `BaseDirsConfigError` (invalid `BASE_DIRS` JSON, missing
|
|
76
|
+
* directory, not-a-directory, ...) is reported to stderr and exits with
|
|
77
|
+
* code 1. This is intentional: the resolver explicitly does NOT fall back
|
|
78
|
+
* (see §Technical Decisions → Resolution order in the multi-base-dirs
|
|
79
|
+
* plan), so CLI consumers should fail fast rather than silently degrading
|
|
80
|
+
* to `cwd`.
|
|
81
|
+
*
|
|
82
|
+
* Warnings (`base-dirs-overrides-base-dir`, `nested-root-pruned`) are
|
|
83
|
+
* returned to the caller rather than written here, so each subcommand can
|
|
84
|
+
* decide its own rendering (JSON-output subcommands like `list` may need
|
|
85
|
+
* to keep stderr clean even when warnings are present).
|
|
86
|
+
*/
|
|
87
|
+
export async function resolveCliBaseDirsOrExit(cliRoots) {
|
|
88
|
+
// Screen the raw env-supplied paths before the resolver realpath-
|
|
89
|
+
// normalizes them, so a literal `BASE_DIR=/etc` is rejected with the
|
|
90
|
+
// env var as the attribution surface.
|
|
91
|
+
if (cliRoots.length === 0) {
|
|
92
|
+
if (process.env['BASE_DIRS'] !== undefined && process.env['BASE_DIRS'].length > 0) {
|
|
93
|
+
const parsed = parseBaseDirsEnv(process.env['BASE_DIRS']);
|
|
94
|
+
if (parsed.ok) {
|
|
95
|
+
for (const raw of parsed.value) {
|
|
96
|
+
const sensitive = checkSensitivePath(raw, 'BASE_DIRS');
|
|
97
|
+
if (sensitive) {
|
|
98
|
+
console.error(sensitive);
|
|
99
|
+
process.exit(1);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
// Malformed BASE_DIRS surfaces below via resolveBaseDirs.
|
|
104
|
+
}
|
|
105
|
+
else if (process.env['BASE_DIR'] !== undefined && process.env['BASE_DIR'].trim().length > 0) {
|
|
106
|
+
const sensitive = checkSensitivePath(process.env['BASE_DIR'], 'BASE_DIR');
|
|
107
|
+
if (sensitive) {
|
|
108
|
+
console.error(sensitive);
|
|
109
|
+
process.exit(1);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
const result = await resolveBaseDirs({
|
|
114
|
+
cliRoots,
|
|
115
|
+
envBaseDirs: process.env['BASE_DIRS'],
|
|
116
|
+
envBaseDir: process.env['BASE_DIR'],
|
|
117
|
+
cwd: process.cwd(),
|
|
118
|
+
});
|
|
119
|
+
if (!result.ok) {
|
|
120
|
+
console.error(result.error.message);
|
|
121
|
+
process.exit(1);
|
|
122
|
+
}
|
|
123
|
+
// Apply the sensitive-path policy uniformly to every effective root
|
|
124
|
+
// (CLI, env, or cwd). Pre-multi-root code validated `BASE_DIR` here; the
|
|
125
|
+
// same policy must continue to apply to `BASE_DIRS` entries and to CLI
|
|
126
|
+
// roots that pre-validation in the subcommand may have missed (e.g.
|
|
127
|
+
// realpath-resolved targets of symlinks). Reported under `--base-dir`
|
|
128
|
+
// because that is the flag the user most directly controls.
|
|
129
|
+
for (const root of result.config.baseDirs) {
|
|
130
|
+
const sensitive = validatePath(root, '--base-dir');
|
|
131
|
+
if (sensitive) {
|
|
132
|
+
console.error(sensitive);
|
|
133
|
+
process.exit(1);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return { config: result.config, warnings: result.warnings };
|
|
137
|
+
}
|
|
138
|
+
//# sourceMappingURL=common.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"common.js","sourceRoot":"","sources":["../../src/cli/common.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,8EAA8E;AAE9E,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EAGL,gBAAgB,EAChB,eAAe,GAChB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAA6B,aAAa,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAEnG;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,cAAc,CAAC,KAAc;IAC3C,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;IACrE,OAAO,aAAa,CAAC,GAAG,CAAC;SACtB,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACnB,MAAM,MAAM,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAA;QAC/C,OAAO,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,OAAO,EAAE,EAAE,CAAA;IACpE,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAA;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAA4B;IAC5D,OAAO,IAAI,WAAW,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,SAAS,EAAE,QAAQ;KACpB,CAAC,CAAA;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,MAA4B;IACzD,MAAM,cAAc,GAA8C;QAChE,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,SAAS,EAAE,EAAE;QACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;KACjD,CAAA;IACD,2EAA2E;IAC3E,wEAAwE;IACxE,qCAAqC;IACrC,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAA;IACpD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,cAAc,CAAC,KAAK,GAAG,KAAK,CAAA;IAC9B,CAAC;IACD,OAAO,IAAI,QAAQ,CAAC,cAAc,CAAC,CAAA;AACrC,CAAC;AAYD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,QAAkB;IAC/D,kEAAkE;IAClE,qEAAqE;IACrE,sCAAsC;IACtC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClF,MAAM,MAAM,GAAG,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAA;YACzD,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;gBACd,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;oBAC/B,MAAM,SAAS,GAAG,kBAAkB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAA;oBACtD,IAAI,SAAS,EAAE,CAAC;wBACd,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;wBACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;oBACjB,CAAC;gBACH,CAAC;YACH,CAAC;YACD,0DAA0D;QAC5D,CAAC;aAAM,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9F,MAAM,SAAS,GAAG,kBAAkB,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC,CAAA;YACzE,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;gBACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC;QACnC,QAAQ;QACR,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;QACrC,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;QACnC,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;KACnB,CAAC,CAAA;IAEF,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;QACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,oEAAoE;IACpE,yEAAyE;IACzE,uEAAuE;IACvE,oEAAoE;IACpE,sEAAsE;IACtE,4DAA4D;IAC5D,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;QAClD,IAAI,SAAS,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAA;AAC7D,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { GlobalOptions } from './options.js';
|
|
2
|
+
/**
|
|
3
|
+
* Run the delete CLI subcommand.
|
|
4
|
+
* @param args - Arguments after "delete"
|
|
5
|
+
* @param globalOptions - Global options parsed before the subcommand
|
|
6
|
+
*/
|
|
7
|
+
export declare function runDelete(args: string[], globalOptions?: GlobalOptions): Promise<void>;
|
|
8
|
+
//# sourceMappingURL=delete.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../../src/cli/delete.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAkFjD;;;;GAIG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,aAAa,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CAsGhG"}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
// CLI delete subcommand — delete ingested content by file path or source URL
|
|
2
|
+
import { unlink } from 'node:fs/promises';
|
|
3
|
+
import { resolve } from 'node:path';
|
|
4
|
+
import { checkRawDataArtifacts, generateMetaJsonPath, generateRawDataPath, isEnoent, isPathInRawDataDirLexical, } from '../utils/raw-data-utils.js';
|
|
5
|
+
import { createVectorStore, formatCliError } from './common.js';
|
|
6
|
+
import { resolveGlobalConfig, validatePath } from './options.js';
|
|
7
|
+
// ============================================
|
|
8
|
+
// Help
|
|
9
|
+
// ============================================
|
|
10
|
+
const HELP_TEXT = `Usage: mcp-local-rag [global-options] delete [--source <url>] [<file-path>]
|
|
11
|
+
|
|
12
|
+
Delete ingested content by file path or source URL.
|
|
13
|
+
|
|
14
|
+
Either <file-path> or --source is required (not both).
|
|
15
|
+
|
|
16
|
+
Arguments:
|
|
17
|
+
<file-path> File path of ingested content to delete
|
|
18
|
+
|
|
19
|
+
Options:
|
|
20
|
+
--source <url> Delete by source URL (for content ingested via ingest_data)
|
|
21
|
+
-h, --help Show this help
|
|
22
|
+
|
|
23
|
+
Global options (must appear before "delete"):
|
|
24
|
+
--db-path <path> LanceDB database path
|
|
25
|
+
--cache-dir <path> Model cache directory
|
|
26
|
+
--model-name <name> Embedding model`;
|
|
27
|
+
/**
|
|
28
|
+
* Parse delete-specific CLI arguments.
|
|
29
|
+
* Accepts a positional <file-path>, --source <url>, and -h/--help.
|
|
30
|
+
* Unknown flags or conflicting args cause exit(1).
|
|
31
|
+
*/
|
|
32
|
+
function parseArgs(args) {
|
|
33
|
+
let help = false;
|
|
34
|
+
let source;
|
|
35
|
+
let filePath;
|
|
36
|
+
let i = 0;
|
|
37
|
+
while (i < args.length) {
|
|
38
|
+
const arg = args[i];
|
|
39
|
+
if (arg === '-h' || arg === '--help') {
|
|
40
|
+
help = true;
|
|
41
|
+
i++;
|
|
42
|
+
}
|
|
43
|
+
else if (arg === '--source') {
|
|
44
|
+
const value = args[++i];
|
|
45
|
+
if (value === undefined || value.startsWith('-')) {
|
|
46
|
+
console.error('Missing value for --source');
|
|
47
|
+
console.error(HELP_TEXT);
|
|
48
|
+
process.exit(1);
|
|
49
|
+
}
|
|
50
|
+
source = value;
|
|
51
|
+
i++;
|
|
52
|
+
}
|
|
53
|
+
else if (arg.startsWith('-')) {
|
|
54
|
+
console.error(`Unknown option: ${arg}`);
|
|
55
|
+
console.error(HELP_TEXT);
|
|
56
|
+
process.exit(1);
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
// Positional argument: file-path
|
|
60
|
+
filePath = arg;
|
|
61
|
+
i++;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
const result = { help };
|
|
65
|
+
if (source !== undefined)
|
|
66
|
+
result.source = source;
|
|
67
|
+
if (filePath !== undefined)
|
|
68
|
+
result.filePath = filePath;
|
|
69
|
+
return result;
|
|
70
|
+
}
|
|
71
|
+
// ============================================
|
|
72
|
+
// Main Entry Point
|
|
73
|
+
// ============================================
|
|
74
|
+
/**
|
|
75
|
+
* Run the delete CLI subcommand.
|
|
76
|
+
* @param args - Arguments after "delete"
|
|
77
|
+
* @param globalOptions - Global options parsed before the subcommand
|
|
78
|
+
*/
|
|
79
|
+
export async function runDelete(args, globalOptions = {}) {
|
|
80
|
+
// Parse CLI options
|
|
81
|
+
const parsed = parseArgs(args);
|
|
82
|
+
// Handle --help
|
|
83
|
+
if (parsed.help) {
|
|
84
|
+
console.error(HELP_TEXT);
|
|
85
|
+
process.exit(0);
|
|
86
|
+
}
|
|
87
|
+
// Validate: either file-path or --source required, not both
|
|
88
|
+
if (!parsed.filePath && !parsed.source) {
|
|
89
|
+
console.error('Either <file-path> or --source is required');
|
|
90
|
+
console.error(HELP_TEXT);
|
|
91
|
+
process.exit(1);
|
|
92
|
+
}
|
|
93
|
+
if (parsed.filePath && parsed.source) {
|
|
94
|
+
console.error('Cannot specify both <file-path> and --source');
|
|
95
|
+
console.error(HELP_TEXT);
|
|
96
|
+
process.exit(1);
|
|
97
|
+
}
|
|
98
|
+
// Resolve global config
|
|
99
|
+
const globalConfig = resolveGlobalConfig(globalOptions);
|
|
100
|
+
const vectorStore = createVectorStore(globalConfig);
|
|
101
|
+
try {
|
|
102
|
+
await vectorStore.initialize();
|
|
103
|
+
// Determine target file path
|
|
104
|
+
let targetPath;
|
|
105
|
+
if (parsed.source) {
|
|
106
|
+
// Generate raw-data path from source URL
|
|
107
|
+
targetPath = generateRawDataPath(globalConfig.dbPath, parsed.source);
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
// DB key is the resolve()'d ingest path, so look up by resolve() (never
|
|
111
|
+
// realpath) — realpath stays in validatePath/validateFilePath.
|
|
112
|
+
targetPath = resolve(parsed.filePath);
|
|
113
|
+
// Validate path (reject sensitive system directories)
|
|
114
|
+
const pathError = validatePath(targetPath, '<file-path>');
|
|
115
|
+
if (pathError) {
|
|
116
|
+
console.error(pathError);
|
|
117
|
+
process.exitCode = 1;
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
// Delete chunks from VectorStore
|
|
122
|
+
const removedChunks = await vectorStore.deleteChunks(targetPath);
|
|
123
|
+
// Optimize immediately after the DB delete: a later raw-data unlink failure
|
|
124
|
+
// (re-thrown below for non-ENOENT) must not skip compaction once the rows
|
|
125
|
+
// are already gone.
|
|
126
|
+
await vectorStore.optimize();
|
|
127
|
+
let rawDataExisted = false;
|
|
128
|
+
let metaExisted = false;
|
|
129
|
+
// Clean up physical raw-data files if applicable.
|
|
130
|
+
if (isPathInRawDataDirLexical(targetPath, globalConfig.dbPath)) {
|
|
131
|
+
// Pre-unlink existence (shared with the MCP server delete path).
|
|
132
|
+
const artifacts = await checkRawDataArtifacts(targetPath);
|
|
133
|
+
rawDataExisted = artifacts.rawDataExisted;
|
|
134
|
+
metaExisted = artifacts.metaExisted;
|
|
135
|
+
try {
|
|
136
|
+
await unlink(targetPath);
|
|
137
|
+
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
// Ignore ENOENT (file already deleted / never existed)
|
|
140
|
+
if (!isEnoent(error)) {
|
|
141
|
+
throw error;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
try {
|
|
145
|
+
await unlink(generateMetaJsonPath(targetPath));
|
|
146
|
+
}
|
|
147
|
+
catch (error) {
|
|
148
|
+
// Ignore ENOENT
|
|
149
|
+
if (!isEnoent(error)) {
|
|
150
|
+
throw error;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
// Output result JSON to stdout
|
|
155
|
+
const result = {
|
|
156
|
+
filePath: targetPath,
|
|
157
|
+
deleted: true,
|
|
158
|
+
removedChunks,
|
|
159
|
+
existed: removedChunks > 0 || rawDataExisted || metaExisted,
|
|
160
|
+
timestamp: new Date().toISOString(),
|
|
161
|
+
};
|
|
162
|
+
process.stdout.write(JSON.stringify(result));
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
const reason = formatCliError(error);
|
|
166
|
+
console.error(`Error: ${reason}`);
|
|
167
|
+
process.exitCode = 1;
|
|
168
|
+
}
|
|
169
|
+
finally {
|
|
170
|
+
await vectorStore.close();
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
//# sourceMappingURL=delete.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"delete.js","sourceRoot":"","sources":["../../src/cli/delete.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAE7E,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EACL,qBAAqB,EACrB,oBAAoB,EACpB,mBAAmB,EACnB,QAAQ,EACR,yBAAyB,GAC1B,MAAM,4BAA4B,CAAA;AACnC,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE/D,OAAO,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAEhE,+CAA+C;AAC/C,OAAO;AACP,+CAA+C;AAE/C,MAAM,SAAS,GAAG;;;;;;;;;;;;;;;;yCAgBuB,CAAA;AAYzC;;;;GAIG;AACH,SAAS,SAAS,CAAC,IAAc;IAC/B,IAAI,IAAI,GAAG,KAAK,CAAA;IAChB,IAAI,MAA0B,CAAA;IAC9B,IAAI,QAA4B,CAAA;IAEhC,IAAI,CAAC,GAAG,CAAC,CAAA;IACT,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAE,CAAA;QAEpB,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,QAAQ,EAAE,CAAC;YACrC,IAAI,GAAG,IAAI,CAAA;YACX,CAAC,EAAE,CAAA;QACL,CAAC;aAAM,IAAI,GAAG,KAAK,UAAU,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;YACvB,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjD,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAA;gBAC3C,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;gBACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACjB,CAAC;YACD,MAAM,GAAG,KAAK,CAAA;YACd,CAAC,EAAE,CAAA;QACL,CAAC;aAAM,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAA;YACvC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;aAAM,CAAC;YACN,iCAAiC;YACjC,QAAQ,GAAG,GAAG,CAAA;YACd,CAAC,EAAE,CAAA;QACL,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAe,EAAE,IAAI,EAAE,CAAA;IACnC,IAAI,MAAM,KAAK,SAAS;QAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAA;IAChD,IAAI,QAAQ,KAAK,SAAS;QAAE,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAA;IACtD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,+CAA+C;AAC/C,mBAAmB;AACnB,+CAA+C;AAE/C;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAc,EAAE,aAAa,GAAkB,EAAE;IAC/E,oBAAoB;IACpB,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;IAE9B,gBAAgB;IAChB,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,4DAA4D;IAC5D,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACvC,OAAO,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAA;QAC3D,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACrC,OAAO,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAA;QAC7D,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,wBAAwB;IACxB,MAAM,YAAY,GAAG,mBAAmB,CAAC,aAAa,CAAC,CAAA;IACvD,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAEnD,IAAI,CAAC;QACH,MAAM,WAAW,CAAC,UAAU,EAAE,CAAA;QAE9B,6BAA6B;QAC7B,IAAI,UAAkB,CAAA;QAEtB,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,yCAAyC;YACzC,UAAU,GAAG,mBAAmB,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;QACtE,CAAC;aAAM,CAAC;YACN,wEAAwE;YACxE,+DAA+D;YAC/D,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,QAAS,CAAC,CAAA;YAEtC,sDAAsD;YACtD,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU,EAAE,aAAa,CAAC,CAAA;YACzD,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;gBACxB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;gBACpB,OAAM;YACR,CAAC;QACH,CAAC;QAED,iCAAiC;QACjC,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,YAAY,CAAC,UAAU,CAAC,CAAA;QAChE,4EAA4E;QAC5E,0EAA0E;QAC1E,oBAAoB;QACpB,MAAM,WAAW,CAAC,QAAQ,EAAE,CAAA;QAE5B,IAAI,cAAc,GAAG,KAAK,CAAA;QAC1B,IAAI,WAAW,GAAG,KAAK,CAAA;QAEvB,kDAAkD;QAClD,IAAI,yBAAyB,CAAC,UAAU,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/D,iEAAiE;YACjE,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,CAAA;YACzD,cAAc,GAAG,SAAS,CAAC,cAAc,CAAA;YACzC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAA;YAEnC,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,UAAU,CAAC,CAAA;YAC1B,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACxB,uDAAuD;gBACvD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBACrB,MAAM,KAAK,CAAA;gBACb,CAAC;YACH,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC,CAAA;YAChD,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACxB,gBAAgB;gBAChB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBACrB,MAAM,KAAK,CAAA;gBACb,CAAC;YACH,CAAC;QACH,CAAC;QAED,+BAA+B;QAC/B,MAAM,MAAM,GAAG;YACb,QAAQ,EAAE,UAAU;YACpB,OAAO,EAAE,IAAI;YACb,aAAa;YACb,OAAO,EAAE,aAAa,GAAG,CAAC,IAAI,cAAc,IAAI,WAAW;YAC3D,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAA;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;IAC9C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAA;QACpC,OAAO,CAAC,KAAK,CAAC,UAAU,MAAM,EAAE,CAAC,CAAA;QACjC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;IACtB,CAAC;YAAS,CAAC;QACT,MAAM,WAAW,CAAC,KAAK,EAAE,CAAA;IAC3B,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-collection.d.ts","sourceRoot":"","sources":["../../src/cli/file-collection.ts"],"names":[],"mappings":"AAeA,wBAAsB,YAAY,CAChC,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,YAAY,EAAE,MAAM,EAAE,GACrB,OAAO,CAAC,MAAM,EAAE,CAAC,CA0DnB"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// CLI ingest file collection.
|
|
2
|
+
//
|
|
3
|
+
// Resolves the positional ingest path into the concrete list of supported
|
|
4
|
+
// files to ingest. Single-file mode validates the extension; directory mode
|
|
5
|
+
// confirms the path is inside a configured root and delegates the bounded
|
|
6
|
+
// BFS walk to `bfsCollectSupportedFiles`. Out-of-root directory targets exit
|
|
7
|
+
// the process (preserved from the original inline implementation).
|
|
8
|
+
import { realpath, stat } from 'node:fs/promises';
|
|
9
|
+
import { extname, resolve, sep } from 'node:path';
|
|
10
|
+
import { SUPPORTED_EXTENSIONS } from '../parser/index.js';
|
|
11
|
+
import { MAX_SCAN_DEPTH } from '../utils/limits.js';
|
|
12
|
+
import { bfsCollectSupportedFiles } from '../utils/scan.js';
|
|
13
|
+
export async function collectFiles(targetPath, baseDirs, excludePaths) {
|
|
14
|
+
const resolved = resolve(targetPath);
|
|
15
|
+
const info = await stat(resolved);
|
|
16
|
+
if (info.isFile()) {
|
|
17
|
+
const ext = extname(resolved).toLowerCase();
|
|
18
|
+
if (!SUPPORTED_EXTENSIONS.has(ext)) {
|
|
19
|
+
console.error(`Unsupported file extension: ${ext} (supported: ${[...SUPPORTED_EXTENSIONS].join(', ')})`);
|
|
20
|
+
return [];
|
|
21
|
+
}
|
|
22
|
+
// Store the resolve()'d path (the DB key), not realpath — path policy:
|
|
23
|
+
// realpath only at the security boundary, resolve() everywhere user-facing.
|
|
24
|
+
return [resolved];
|
|
25
|
+
}
|
|
26
|
+
if (info.isDirectory()) {
|
|
27
|
+
// realpath both sides so a symlinked positional path still matches a
|
|
28
|
+
// root whose realpath agrees. baseDirs from resolveCliBaseDirsOrExit
|
|
29
|
+
// are already realpath-normalized with a trailing sep.
|
|
30
|
+
const realResolved = await realpath(resolved);
|
|
31
|
+
const realResolvedWithSep = realResolved.endsWith(sep) ? realResolved : realResolved + sep;
|
|
32
|
+
const insideAnyRoot = baseDirs.some((root) => realResolvedWithSep === root || realResolvedWithSep.startsWith(root));
|
|
33
|
+
if (!insideAnyRoot) {
|
|
34
|
+
console.error(`Error: ${targetPath} is not under any configured base directory. ` +
|
|
35
|
+
`Allowed roots: ${baseDirs.join(', ')}. ` +
|
|
36
|
+
`Provide a path inside one of the configured roots, or set BASE_DIRS / --base-dir to include the desired tree.`);
|
|
37
|
+
process.exit(1);
|
|
38
|
+
}
|
|
39
|
+
// `realResolved` is used ONLY for the in-root containment check (security).
|
|
40
|
+
// The walk uses the resolve()'d `resolved` so the stored DB keys match what
|
|
41
|
+
// `list`/`delete`/`read_chunk_neighbors` use (resolve(), not realpath).
|
|
42
|
+
const { files: collected, unreadableDirs, depthLimited, } = await bfsCollectSupportedFiles(resolved, excludePaths);
|
|
43
|
+
for (const { dirPath } of unreadableDirs) {
|
|
44
|
+
console.error(`Warning: cannot read directory: ${dirPath}`);
|
|
45
|
+
}
|
|
46
|
+
if (depthLimited) {
|
|
47
|
+
console.error(`Warning: some directories were skipped because they exceed the maximum depth (${MAX_SCAN_DEPTH})`);
|
|
48
|
+
}
|
|
49
|
+
return [...new Set(collected)].sort();
|
|
50
|
+
}
|
|
51
|
+
return [];
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=file-collection.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-collection.js","sourceRoot":"","sources":["../../src/cli/file-collection.ts"],"names":[],"mappings":"AAAA,8BAA8B;AAC9B,EAAE;AACF,0EAA0E;AAC1E,4EAA4E;AAC5E,0EAA0E;AAC1E,6EAA6E;AAC7E,mEAAmE;AAEnE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAA;AACjD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAA;AAEjD,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,wBAAwB,EAAE,MAAM,kBAAkB,CAAA;AAE3D,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,UAAkB,EAClB,QAA2B,EAC3B,YAAsB;IAEtB,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IACpC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAA;IAEjC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAA;QAC3C,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,KAAK,CACX,+BAA+B,GAAG,gBAAgB,CAAC,GAAG,oBAAoB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC1F,CAAA;YACD,OAAO,EAAE,CAAA;QACX,CAAC;QACD,uEAAuE;QACvE,4EAA4E;QAC5E,OAAO,CAAC,QAAQ,CAAC,CAAA;IACnB,CAAC;IAED,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;QACvB,qEAAqE;QACrE,qEAAqE;QACrE,uDAAuD;QACvD,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAC7C,MAAM,mBAAmB,GAAG,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,YAAY,GAAG,GAAG,CAAA;QAC1F,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI,CACjC,CAAC,IAAI,EAAE,EAAE,CAAC,mBAAmB,KAAK,IAAI,IAAI,mBAAmB,CAAC,UAAU,CAAC,IAAI,CAAC,CAC/E,CAAA;QACD,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,OAAO,CAAC,KAAK,CACX,UAAU,UAAU,+CAA+C;gBACjE,kBAAkB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;gBACzC,+GAA+G,CAClH,CAAA;YACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;QAED,4EAA4E;QAC5E,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,EACJ,KAAK,EAAE,SAAS,EAChB,cAAc,EACd,YAAY,GACb,GAAG,MAAM,wBAAwB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAA;QAE1D,KAAK,MAAM,EAAE,OAAO,EAAE,IAAI,cAAc,EAAE,CAAC;YACzC,OAAO,CAAC,KAAK,CAAC,mCAAmC,OAAO,EAAE,CAAC,CAAA;QAC7D,CAAC;QAED,IAAI,YAAY,EAAE,CAAC;YACjB,OAAO,CAAC,KAAK,CACX,iFAAiF,cAAc,GAAG,CACnG,CAAA;QACH,CAAC;QAED,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IACvC,CAAC;IAED,OAAO,EAAE,CAAA;AACX,CAAC"}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { SemanticChunker } from '../chunker/index.js';
|
|
2
|
+
import type { Embedder } from '../embedder/index.js';
|
|
3
|
+
import { DocumentParser } from '../parser/index.js';
|
|
4
|
+
import { type QualityProfile } from '../pdf-visual/types.js';
|
|
5
|
+
import type { BaseDirsConfig, BaseDirsConfigWarning } from '../utils/base-dirs.js';
|
|
6
|
+
import type { VectorStore } from '../vectordb/index.js';
|
|
7
|
+
import type { GlobalOptions, ResolvedGlobalConfig } from './options.js';
|
|
8
|
+
interface IngestConfig {
|
|
9
|
+
baseDirs: BaseDirsConfig;
|
|
10
|
+
baseDirsWarnings: BaseDirsConfigWarning[];
|
|
11
|
+
dbPath: string;
|
|
12
|
+
cacheDir: string;
|
|
13
|
+
modelName: string;
|
|
14
|
+
maxFileSize: number;
|
|
15
|
+
chunkMinLength?: number;
|
|
16
|
+
}
|
|
17
|
+
interface IngestCliOptions {
|
|
18
|
+
/**
|
|
19
|
+
* Collected `--base-dir` values in CLI order. Repeatable: each flag
|
|
20
|
+
* occurrence appends one entry. An empty array means the flag was not
|
|
21
|
+
* provided (resolver then falls through to env / cwd).
|
|
22
|
+
*/
|
|
23
|
+
baseDirs?: string[] | undefined;
|
|
24
|
+
maxFileSize?: number | undefined;
|
|
25
|
+
chunkMinLength?: number | undefined;
|
|
26
|
+
visual?: boolean | undefined;
|
|
27
|
+
images?: boolean | undefined;
|
|
28
|
+
/**
|
|
29
|
+
* Visual-quality profile selector. Only meaningful when `visual` is true;
|
|
30
|
+
* silently ignored otherwise (mirrors the existing `--visual` precedent
|
|
31
|
+
* of silently coercing for non-PDF files). Defaults to `'fast'`.
|
|
32
|
+
*/
|
|
33
|
+
visualQuality?: QualityProfile | undefined;
|
|
34
|
+
}
|
|
35
|
+
interface ParsedArgs {
|
|
36
|
+
positional: string | undefined;
|
|
37
|
+
options: IngestCliOptions;
|
|
38
|
+
help: boolean;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Parse ingest-specific CLI arguments into options and a positional path.
|
|
42
|
+
* Flags: --base-dir, --max-file-size, -h/--help
|
|
43
|
+
* Unknown flags (including global flags passed after subcommand) cause an error.
|
|
44
|
+
*/
|
|
45
|
+
export declare function parseArgs(args: string[]): ParsedArgs;
|
|
46
|
+
/**
|
|
47
|
+
* Resolve ingest config by merging global config with ingest-specific options.
|
|
48
|
+
*
|
|
49
|
+
* Base directories are resolved via the shared CLI resolver
|
|
50
|
+
* ({@link resolveCliBaseDirsOrExit}) which applies the documented precedence
|
|
51
|
+
* (CLI roots > `BASE_DIRS` > `BASE_DIR` > `cwd`), realpath-normalizes every
|
|
52
|
+
* effective root, dedupes exact duplicates, and prunes nested roots. CLI
|
|
53
|
+
* roots are pre-validated against the sensitive-path policy here so the
|
|
54
|
+
* user sees `--base-dir`-attributed errors before the resolver touches the
|
|
55
|
+
* filesystem.
|
|
56
|
+
*
|
|
57
|
+
* Other ingest-specific values (maxFileSize, chunkMinLength) follow the
|
|
58
|
+
* existing CLI > env > defaults order and are validated against the same
|
|
59
|
+
* ranges as before.
|
|
60
|
+
*/
|
|
61
|
+
export declare function resolveConfig(globalConfig: ResolvedGlobalConfig, ingestOptions?: IngestCliOptions): Promise<IngestConfig>;
|
|
62
|
+
/**
|
|
63
|
+
* Options for `ingestSingleFile`. VLM configuration is required only when
|
|
64
|
+
* visual captions are enabled.
|
|
65
|
+
*/
|
|
66
|
+
export type IngestSingleFileOptions = {
|
|
67
|
+
visual?: false | undefined;
|
|
68
|
+
images?: false | undefined;
|
|
69
|
+
} | {
|
|
70
|
+
visual: false;
|
|
71
|
+
images: true;
|
|
72
|
+
} | {
|
|
73
|
+
visual: true;
|
|
74
|
+
images?: boolean | undefined;
|
|
75
|
+
profile: QualityProfile;
|
|
76
|
+
cacheDir: string;
|
|
77
|
+
device?: string | undefined;
|
|
78
|
+
};
|
|
79
|
+
/**
|
|
80
|
+
* Ingest a single file: hash, parse, chunk, embed, delete old chunks, insert new
|
|
81
|
+
* chunks. Returns the number of chunks inserted.
|
|
82
|
+
*
|
|
83
|
+
* When `options.visual === true` AND the file is a `.pdf`, routes through the
|
|
84
|
+
* visual-enrichment path: `parsePdfPages` + VLM captioning (`pdf-visual`
|
|
85
|
+
* orchestrator) + joined-text chunking. `pdf-visual` is loaded via dynamic
|
|
86
|
+
* `await import('../pdf-visual/index.js')` so the default (non-visual) path
|
|
87
|
+
* never pulls the VLM module into the bundle.
|
|
88
|
+
*
|
|
89
|
+
* Non-visual, non-PDF, and `visual: true` + non-PDF paths all use the default
|
|
90
|
+
* text-only branch and never load `pdf-visual`.
|
|
91
|
+
*/
|
|
92
|
+
export declare function ingestSingleFile(filePath: string, parser: DocumentParser, chunker: SemanticChunker, embedder: Embedder, vectorStore: VectorStore, options?: IngestSingleFileOptions): Promise<number>;
|
|
93
|
+
/**
|
|
94
|
+
* Run the ingest CLI subcommand.
|
|
95
|
+
* @param args - Arguments after "ingest" (e.g., option flags and file/directory path)
|
|
96
|
+
* @param globalOptions - Global options parsed before the subcommand
|
|
97
|
+
*/
|
|
98
|
+
export declare function runIngest(args: string[], globalOptions?: GlobalOptions): Promise<void>;
|
|
99
|
+
export {};
|
|
100
|
+
//# sourceMappingURL=ingest.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/cli/ingest.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AACrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAoB,KAAK,cAAc,EAAE,MAAM,wBAAwB,CAAA;AAC9E,OAAO,KAAK,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAElF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAQvD,OAAO,KAAK,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAA;AAevE,UAAU,YAAY;IACpB,QAAQ,EAAE,cAAc,CAAA;IACxB,gBAAgB,EAAE,qBAAqB,EAAE,CAAA;IACzC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAQD,UAAU,gBAAgB;IACxB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAA;IAC/B,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAChC,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACnC,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;IAC5B;;;;OAIG;IACH,aAAa,CAAC,EAAE,cAAc,GAAG,SAAS,CAAA;CAC3C;AAED,UAAU,UAAU;IAClB,UAAU,EAAE,MAAM,GAAG,SAAS,CAAA;IAC9B,OAAO,EAAE,gBAAgB,CAAA;IACzB,IAAI,EAAE,OAAO,CAAA;CACd;AAwCD;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,UAAU,CAuFpD;AAMD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,aAAa,CACjC,YAAY,EAAE,oBAAoB,EAClC,aAAa,GAAE,gBAAqB,GACnC,OAAO,CAAC,YAAY,CAAC,CAyDvB;AAMD;;;GAGG;AACH,MAAM,MAAM,uBAAuB,GAC/B;IAAE,MAAM,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC;IAAC,MAAM,CAAC,EAAE,KAAK,GAAG,SAAS,CAAA;CAAE,GAC1D;IAAE,MAAM,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,IAAI,CAAA;CAAE,GAC/B;IACE,MAAM,EAAE,IAAI,CAAA;IACZ,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;IAC5B,OAAO,EAAE,cAAc,CAAA;IACvB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAC5B,CAAA;AAEL;;;;;;;;;;;;GAYG;AACH,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,cAAc,EACtB,OAAO,EAAE,eAAe,EACxB,QAAQ,EAAE,QAAQ,EAClB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,uBAAuB,GAChC,OAAO,CAAC,MAAM,CAAC,CA+BjB;AAMD;;;;GAIG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,aAAa,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CAwIhG"}
|