@hazeljs/rag 0.2.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +192 -0
- package/README.md +504 -0
- package/dist/__tests__/graph/community-detector.test.d.ts +2 -0
- package/dist/__tests__/graph/community-detector.test.d.ts.map +1 -0
- package/dist/__tests__/graph/community-detector.test.js +87 -0
- package/dist/__tests__/graph/community-detector.test.js.map +1 -0
- package/dist/__tests__/graph/community-summarizer.test.d.ts +2 -0
- package/dist/__tests__/graph/community-summarizer.test.d.ts.map +1 -0
- package/dist/__tests__/graph/community-summarizer.test.js +131 -0
- package/dist/__tests__/graph/community-summarizer.test.js.map +1 -0
- package/dist/__tests__/graph/entity-extractor.test.d.ts +2 -0
- package/dist/__tests__/graph/entity-extractor.test.d.ts.map +1 -0
- package/dist/__tests__/graph/entity-extractor.test.js +129 -0
- package/dist/__tests__/graph/entity-extractor.test.js.map +1 -0
- package/dist/__tests__/graph/graph-rag-pipeline.test.d.ts +2 -0
- package/dist/__tests__/graph/graph-rag-pipeline.test.d.ts.map +1 -0
- package/dist/__tests__/graph/graph-rag-pipeline.test.js +158 -0
- package/dist/__tests__/graph/graph-rag-pipeline.test.js.map +1 -0
- package/dist/__tests__/graph/knowledge-graph.test.d.ts +2 -0
- package/dist/__tests__/graph/knowledge-graph.test.d.ts.map +1 -0
- package/dist/__tests__/graph/knowledge-graph.test.js +208 -0
- package/dist/__tests__/graph/knowledge-graph.test.js.map +1 -0
- package/dist/__tests__/loaders/base.loader.test.d.ts +2 -0
- package/dist/__tests__/loaders/base.loader.test.d.ts.map +1 -0
- package/dist/__tests__/loaders/base.loader.test.js +114 -0
- package/dist/__tests__/loaders/base.loader.test.js.map +1 -0
- package/dist/__tests__/loaders/csv-file.loader.test.d.ts +2 -0
- package/dist/__tests__/loaders/csv-file.loader.test.d.ts.map +1 -0
- package/dist/__tests__/loaders/csv-file.loader.test.js +98 -0
- package/dist/__tests__/loaders/csv-file.loader.test.js.map +1 -0
- package/dist/__tests__/loaders/directory.loader.test.d.ts +2 -0
- package/dist/__tests__/loaders/directory.loader.test.d.ts.map +1 -0
- package/dist/__tests__/loaders/directory.loader.test.js +154 -0
- package/dist/__tests__/loaders/directory.loader.test.js.map +1 -0
- package/dist/__tests__/loaders/html-file.loader.test.d.ts +2 -0
- package/dist/__tests__/loaders/html-file.loader.test.d.ts.map +1 -0
- package/dist/__tests__/loaders/html-file.loader.test.js +93 -0
- package/dist/__tests__/loaders/html-file.loader.test.js.map +1 -0
- package/dist/__tests__/loaders/json-file.loader.test.d.ts +2 -0
- package/dist/__tests__/loaders/json-file.loader.test.d.ts.map +1 -0
- package/dist/__tests__/loaders/json-file.loader.test.js +84 -0
- package/dist/__tests__/loaders/json-file.loader.test.js.map +1 -0
- package/dist/__tests__/loaders/markdown-file.loader.test.d.ts +2 -0
- package/dist/__tests__/loaders/markdown-file.loader.test.d.ts.map +1 -0
- package/dist/__tests__/loaders/markdown-file.loader.test.js +83 -0
- package/dist/__tests__/loaders/markdown-file.loader.test.js.map +1 -0
- package/dist/__tests__/loaders/text-file.loader.test.d.ts +2 -0
- package/dist/__tests__/loaders/text-file.loader.test.d.ts.map +1 -0
- package/dist/__tests__/loaders/text-file.loader.test.js +50 -0
- package/dist/__tests__/loaders/text-file.loader.test.js.map +1 -0
- package/dist/__tests__/rag-pipeline.test.d.ts +2 -0
- package/dist/__tests__/rag-pipeline.test.d.ts.map +1 -0
- package/dist/__tests__/rag-pipeline.test.js +210 -0
- package/dist/__tests__/rag-pipeline.test.js.map +1 -0
- package/dist/__tests__/retrieval/bm25.test.d.ts +2 -0
- package/dist/__tests__/retrieval/bm25.test.d.ts.map +1 -0
- package/dist/__tests__/retrieval/bm25.test.js +86 -0
- package/dist/__tests__/retrieval/bm25.test.js.map +1 -0
- package/dist/__tests__/retrieval/hybrid-search.test.d.ts +2 -0
- package/dist/__tests__/retrieval/hybrid-search.test.d.ts.map +1 -0
- package/dist/__tests__/retrieval/hybrid-search.test.js +85 -0
- package/dist/__tests__/retrieval/hybrid-search.test.js.map +1 -0
- package/dist/__tests__/retrieval/multi-query.test.d.ts +2 -0
- package/dist/__tests__/retrieval/multi-query.test.d.ts.map +1 -0
- package/dist/__tests__/retrieval/multi-query.test.js +90 -0
- package/dist/__tests__/retrieval/multi-query.test.js.map +1 -0
- package/dist/__tests__/text-splitters/recursive-text-splitter.test.d.ts +2 -0
- package/dist/__tests__/text-splitters/recursive-text-splitter.test.d.ts.map +1 -0
- package/dist/__tests__/text-splitters/recursive-text-splitter.test.js +97 -0
- package/dist/__tests__/text-splitters/recursive-text-splitter.test.js.map +1 -0
- package/dist/__tests__/utils/similarity.test.d.ts +2 -0
- package/dist/__tests__/utils/similarity.test.d.ts.map +1 -0
- package/dist/__tests__/utils/similarity.test.js +47 -0
- package/dist/__tests__/utils/similarity.test.js.map +1 -0
- package/dist/agentic/agentic-rag.service.d.ts +49 -0
- package/dist/agentic/agentic-rag.service.d.ts.map +1 -0
- package/dist/agentic/agentic-rag.service.js +149 -0
- package/dist/agentic/agentic-rag.service.js.map +1 -0
- package/dist/agentic/decorators/active-learning.decorator.d.ts +19 -0
- package/dist/agentic/decorators/active-learning.decorator.d.ts.map +1 -0
- package/dist/agentic/decorators/active-learning.decorator.js +98 -0
- package/dist/agentic/decorators/active-learning.decorator.js.map +1 -0
- package/dist/agentic/decorators/adaptive-retrieval.decorator.d.ts +17 -0
- package/dist/agentic/decorators/adaptive-retrieval.decorator.d.ts.map +1 -0
- package/dist/agentic/decorators/adaptive-retrieval.decorator.js +103 -0
- package/dist/agentic/decorators/adaptive-retrieval.decorator.js.map +1 -0
- package/dist/agentic/decorators/cached.decorator.d.ts +18 -0
- package/dist/agentic/decorators/cached.decorator.d.ts.map +1 -0
- package/dist/agentic/decorators/cached.decorator.js +93 -0
- package/dist/agentic/decorators/cached.decorator.js.map +1 -0
- package/dist/agentic/decorators/context-aware.decorator.d.ts +16 -0
- package/dist/agentic/decorators/context-aware.decorator.d.ts.map +1 -0
- package/dist/agentic/decorators/context-aware.decorator.js +169 -0
- package/dist/agentic/decorators/context-aware.decorator.js.map +1 -0
- package/dist/agentic/decorators/corrective-rag.decorator.d.ts +16 -0
- package/dist/agentic/decorators/corrective-rag.decorator.d.ts.map +1 -0
- package/dist/agentic/decorators/corrective-rag.decorator.js +142 -0
- package/dist/agentic/decorators/corrective-rag.decorator.js.map +1 -0
- package/dist/agentic/decorators/hyde.decorator.d.ts +15 -0
- package/dist/agentic/decorators/hyde.decorator.d.ts.map +1 -0
- package/dist/agentic/decorators/hyde.decorator.js +91 -0
- package/dist/agentic/decorators/hyde.decorator.js.map +1 -0
- package/dist/agentic/decorators/index.d.ts +16 -0
- package/dist/agentic/decorators/index.d.ts.map +1 -0
- package/dist/agentic/decorators/index.js +32 -0
- package/dist/agentic/decorators/index.js.map +1 -0
- package/dist/agentic/decorators/multi-hop.decorator.d.ts +15 -0
- package/dist/agentic/decorators/multi-hop.decorator.d.ts.map +1 -0
- package/dist/agentic/decorators/multi-hop.decorator.js +109 -0
- package/dist/agentic/decorators/multi-hop.decorator.js.map +1 -0
- package/dist/agentic/decorators/query-planner.decorator.d.ts +20 -0
- package/dist/agentic/decorators/query-planner.decorator.d.ts.map +1 -0
- package/dist/agentic/decorators/query-planner.decorator.js +213 -0
- package/dist/agentic/decorators/query-planner.decorator.js.map +1 -0
- package/dist/agentic/decorators/query-rewriter.decorator.d.ts +16 -0
- package/dist/agentic/decorators/query-rewriter.decorator.d.ts.map +1 -0
- package/dist/agentic/decorators/query-rewriter.decorator.js +143 -0
- package/dist/agentic/decorators/query-rewriter.decorator.js.map +1 -0
- package/dist/agentic/decorators/self-reflective.decorator.d.ts +20 -0
- package/dist/agentic/decorators/self-reflective.decorator.d.ts.map +1 -0
- package/dist/agentic/decorators/self-reflective.decorator.js +189 -0
- package/dist/agentic/decorators/self-reflective.decorator.js.map +1 -0
- package/dist/agentic/decorators/source-verification.decorator.d.ts +15 -0
- package/dist/agentic/decorators/source-verification.decorator.d.ts.map +1 -0
- package/dist/agentic/decorators/source-verification.decorator.js +121 -0
- package/dist/agentic/decorators/source-verification.decorator.js.map +1 -0
- package/dist/agentic/index.d.ts +9 -0
- package/dist/agentic/index.d.ts.map +1 -0
- package/dist/agentic/index.js +25 -0
- package/dist/agentic/index.js.map +1 -0
- package/dist/agentic/types.d.ts +210 -0
- package/dist/agentic/types.d.ts.map +1 -0
- package/dist/agentic/types.js +7 -0
- package/dist/agentic/types.js.map +1 -0
- package/dist/decorators/embeddable.decorator.d.ts +31 -0
- package/dist/decorators/embeddable.decorator.d.ts.map +1 -0
- package/dist/decorators/embeddable.decorator.js +44 -0
- package/dist/decorators/embeddable.decorator.js.map +1 -0
- package/dist/decorators/rag.decorator.d.ts +58 -0
- package/dist/decorators/rag.decorator.d.ts.map +1 -0
- package/dist/decorators/rag.decorator.js +78 -0
- package/dist/decorators/rag.decorator.js.map +1 -0
- package/dist/decorators/semantic-search.decorator.d.ts +69 -0
- package/dist/decorators/semantic-search.decorator.d.ts.map +1 -0
- package/dist/decorators/semantic-search.decorator.js +116 -0
- package/dist/decorators/semantic-search.decorator.js.map +1 -0
- package/dist/embeddings/cohere-embeddings.d.ts +33 -0
- package/dist/embeddings/cohere-embeddings.d.ts.map +1 -0
- package/dist/embeddings/cohere-embeddings.js +91 -0
- package/dist/embeddings/cohere-embeddings.js.map +1 -0
- package/dist/embeddings/openai-embeddings.d.ts +21 -0
- package/dist/embeddings/openai-embeddings.d.ts.map +1 -0
- package/dist/embeddings/openai-embeddings.js +53 -0
- package/dist/embeddings/openai-embeddings.js.map +1 -0
- package/dist/graph/community-detector.d.ts +45 -0
- package/dist/graph/community-detector.d.ts.map +1 -0
- package/dist/graph/community-detector.js +153 -0
- package/dist/graph/community-detector.js.map +1 -0
- package/dist/graph/community-summarizer.d.ts +41 -0
- package/dist/graph/community-summarizer.d.ts.map +1 -0
- package/dist/graph/community-summarizer.js +119 -0
- package/dist/graph/community-summarizer.js.map +1 -0
- package/dist/graph/entity-extractor.d.ts +47 -0
- package/dist/graph/entity-extractor.d.ts.map +1 -0
- package/dist/graph/entity-extractor.js +224 -0
- package/dist/graph/entity-extractor.js.map +1 -0
- package/dist/graph/graph-rag-pipeline.d.ts +83 -0
- package/dist/graph/graph-rag-pipeline.d.ts.map +1 -0
- package/dist/graph/graph-rag-pipeline.js +390 -0
- package/dist/graph/graph-rag-pipeline.js.map +1 -0
- package/dist/graph/graph.types.d.ts +186 -0
- package/dist/graph/graph.types.d.ts.map +1 -0
- package/dist/graph/graph.types.js +20 -0
- package/dist/graph/graph.types.js.map +1 -0
- package/dist/graph/index.d.ts +15 -0
- package/dist/graph/index.d.ts.map +1 -0
- package/dist/graph/index.js +31 -0
- package/dist/graph/index.js.map +1 -0
- package/dist/graph/knowledge-graph.d.ts +57 -0
- package/dist/graph/knowledge-graph.d.ts.map +1 -0
- package/dist/graph/knowledge-graph.js +198 -0
- package/dist/graph/knowledge-graph.js.map +1 -0
- package/dist/index.d.ts +29 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +58 -0
- package/dist/index.js.map +1 -0
- package/dist/loaders/base.loader.d.ts +108 -0
- package/dist/loaders/base.loader.d.ts.map +1 -0
- package/dist/loaders/base.loader.js +123 -0
- package/dist/loaders/base.loader.js.map +1 -0
- package/dist/loaders/csv-file.loader.d.ts +61 -0
- package/dist/loaders/csv-file.loader.d.ts.map +1 -0
- package/dist/loaders/csv-file.loader.js +162 -0
- package/dist/loaders/csv-file.loader.js.map +1 -0
- package/dist/loaders/directory.loader.d.ts +67 -0
- package/dist/loaders/directory.loader.d.ts.map +1 -0
- package/dist/loaders/directory.loader.js +163 -0
- package/dist/loaders/directory.loader.js.map +1 -0
- package/dist/loaders/docx.loader.d.ts +52 -0
- package/dist/loaders/docx.loader.d.ts.map +1 -0
- package/dist/loaders/docx.loader.js +110 -0
- package/dist/loaders/docx.loader.js.map +1 -0
- package/dist/loaders/github.loader.d.ts +114 -0
- package/dist/loaders/github.loader.d.ts.map +1 -0
- package/dist/loaders/github.loader.js +217 -0
- package/dist/loaders/github.loader.js.map +1 -0
- package/dist/loaders/html-file.loader.d.ts +55 -0
- package/dist/loaders/html-file.loader.d.ts.map +1 -0
- package/dist/loaders/html-file.loader.js +170 -0
- package/dist/loaders/html-file.loader.js.map +1 -0
- package/dist/loaders/index.d.ts +52 -0
- package/dist/loaders/index.d.ts.map +1 -0
- package/dist/loaders/index.js +61 -0
- package/dist/loaders/index.js.map +1 -0
- package/dist/loaders/json-file.loader.d.ts +51 -0
- package/dist/loaders/json-file.loader.d.ts.map +1 -0
- package/dist/loaders/json-file.loader.js +100 -0
- package/dist/loaders/json-file.loader.js.map +1 -0
- package/dist/loaders/markdown-file.loader.d.ts +61 -0
- package/dist/loaders/markdown-file.loader.d.ts.map +1 -0
- package/dist/loaders/markdown-file.loader.js +148 -0
- package/dist/loaders/markdown-file.loader.js.map +1 -0
- package/dist/loaders/pdf.loader.d.ts +64 -0
- package/dist/loaders/pdf.loader.d.ts.map +1 -0
- package/dist/loaders/pdf.loader.js +163 -0
- package/dist/loaders/pdf.loader.js.map +1 -0
- package/dist/loaders/text-file.loader.d.ts +39 -0
- package/dist/loaders/text-file.loader.d.ts.map +1 -0
- package/dist/loaders/text-file.loader.js +69 -0
- package/dist/loaders/text-file.loader.js.map +1 -0
- package/dist/loaders/web.loader.d.ts +87 -0
- package/dist/loaders/web.loader.d.ts.map +1 -0
- package/dist/loaders/web.loader.js +194 -0
- package/dist/loaders/web.loader.js.map +1 -0
- package/dist/loaders/youtube-transcript.loader.d.ts +92 -0
- package/dist/loaders/youtube-transcript.loader.d.ts.map +1 -0
- package/dist/loaders/youtube-transcript.loader.js +254 -0
- package/dist/loaders/youtube-transcript.loader.js.map +1 -0
- package/dist/memory/index.d.ts +11 -0
- package/dist/memory/index.d.ts.map +1 -0
- package/dist/memory/index.js +31 -0
- package/dist/memory/index.js.map +1 -0
- package/dist/memory/memory-manager.d.ts +96 -0
- package/dist/memory/memory-manager.d.ts.map +1 -0
- package/dist/memory/memory-manager.js +369 -0
- package/dist/memory/memory-manager.js.map +1 -0
- package/dist/memory/memory-store.interface.d.ts +73 -0
- package/dist/memory/memory-store.interface.d.ts.map +1 -0
- package/dist/memory/memory-store.interface.js +6 -0
- package/dist/memory/memory-store.interface.js.map +1 -0
- package/dist/memory/stores/buffer-memory.d.ts +47 -0
- package/dist/memory/stores/buffer-memory.d.ts.map +1 -0
- package/dist/memory/stores/buffer-memory.js +280 -0
- package/dist/memory/stores/buffer-memory.js.map +1 -0
- package/dist/memory/stores/hybrid-memory.d.ts +49 -0
- package/dist/memory/stores/hybrid-memory.d.ts.map +1 -0
- package/dist/memory/stores/hybrid-memory.js +194 -0
- package/dist/memory/stores/hybrid-memory.js.map +1 -0
- package/dist/memory/stores/vector-memory.d.ts +48 -0
- package/dist/memory/stores/vector-memory.d.ts.map +1 -0
- package/dist/memory/stores/vector-memory.js +312 -0
- package/dist/memory/stores/vector-memory.js.map +1 -0
- package/dist/memory/types.d.ts +119 -0
- package/dist/memory/types.d.ts.map +1 -0
- package/dist/memory/types.js +18 -0
- package/dist/memory/types.js.map +1 -0
- package/dist/prompts/agentic/adaptive-retrieval.prompt.d.ts +8 -0
- package/dist/prompts/agentic/adaptive-retrieval.prompt.d.ts.map +1 -0
- package/dist/prompts/agentic/adaptive-retrieval.prompt.js +27 -0
- package/dist/prompts/agentic/adaptive-retrieval.prompt.js.map +1 -0
- package/dist/prompts/agentic/corrective-rag.prompt.d.ts +9 -0
- package/dist/prompts/agentic/corrective-rag.prompt.d.ts.map +1 -0
- package/dist/prompts/agentic/corrective-rag.prompt.js +23 -0
- package/dist/prompts/agentic/corrective-rag.prompt.js.map +1 -0
- package/dist/prompts/agentic/hyde.prompt.d.ts +9 -0
- package/dist/prompts/agentic/hyde.prompt.d.ts.map +1 -0
- package/dist/prompts/agentic/hyde.prompt.js +18 -0
- package/dist/prompts/agentic/hyde.prompt.js.map +1 -0
- package/dist/prompts/agentic/multi-hop.prompt.d.ts +15 -0
- package/dist/prompts/agentic/multi-hop.prompt.d.ts.map +1 -0
- package/dist/prompts/agentic/multi-hop.prompt.js +38 -0
- package/dist/prompts/agentic/multi-hop.prompt.js.map +1 -0
- package/dist/prompts/agentic/query-planner.prompt.d.ts +8 -0
- package/dist/prompts/agentic/query-planner.prompt.d.ts.map +1 -0
- package/dist/prompts/agentic/query-planner.prompt.js +30 -0
- package/dist/prompts/agentic/query-planner.prompt.js.map +1 -0
- package/dist/prompts/agentic/query-rewriter.prompt.d.ts +10 -0
- package/dist/prompts/agentic/query-rewriter.prompt.d.ts.map +1 -0
- package/dist/prompts/agentic/query-rewriter.prompt.js +17 -0
- package/dist/prompts/agentic/query-rewriter.prompt.js.map +1 -0
- package/dist/prompts/agentic/self-reflective-improve.prompt.d.ts +10 -0
- package/dist/prompts/agentic/self-reflective-improve.prompt.d.ts.map +1 -0
- package/dist/prompts/agentic/self-reflective-improve.prompt.js +24 -0
- package/dist/prompts/agentic/self-reflective-improve.prompt.js.map +1 -0
- package/dist/prompts/agentic/self-reflective.prompt.d.ts +9 -0
- package/dist/prompts/agentic/self-reflective.prompt.d.ts.map +1 -0
- package/dist/prompts/agentic/self-reflective.prompt.js +32 -0
- package/dist/prompts/agentic/self-reflective.prompt.js.map +1 -0
- package/dist/prompts/community-summary.prompt.d.ts +9 -0
- package/dist/prompts/community-summary.prompt.d.ts.map +1 -0
- package/dist/prompts/community-summary.prompt.js +30 -0
- package/dist/prompts/community-summary.prompt.js.map +1 -0
- package/dist/prompts/entity-extraction.prompt.d.ts +10 -0
- package/dist/prompts/entity-extraction.prompt.d.ts.map +1 -0
- package/dist/prompts/entity-extraction.prompt.js +39 -0
- package/dist/prompts/entity-extraction.prompt.js.map +1 -0
- package/dist/prompts/graph-search.prompt.d.ts +10 -0
- package/dist/prompts/graph-search.prompt.d.ts.map +1 -0
- package/dist/prompts/graph-search.prompt.js +23 -0
- package/dist/prompts/graph-search.prompt.js.map +1 -0
- package/dist/prompts/index.d.ts +13 -0
- package/dist/prompts/index.d.ts.map +1 -0
- package/dist/prompts/index.js +29 -0
- package/dist/prompts/index.js.map +1 -0
- package/dist/prompts/rag-answer.prompt.d.ts +9 -0
- package/dist/prompts/rag-answer.prompt.d.ts.map +1 -0
- package/dist/prompts/rag-answer.prompt.js +20 -0
- package/dist/prompts/rag-answer.prompt.js.map +1 -0
- package/dist/rag-pipeline-with-memory.d.ts +68 -0
- package/dist/rag-pipeline-with-memory.d.ts.map +1 -0
- package/dist/rag-pipeline-with-memory.js +186 -0
- package/dist/rag-pipeline-with-memory.js.map +1 -0
- package/dist/rag-pipeline.d.ts +59 -0
- package/dist/rag-pipeline.d.ts.map +1 -0
- package/dist/rag-pipeline.js +181 -0
- package/dist/rag-pipeline.js.map +1 -0
- package/dist/rag.module.d.ts +26 -0
- package/dist/rag.module.d.ts.map +1 -0
- package/dist/rag.module.js +40 -0
- package/dist/rag.module.js.map +1 -0
- package/dist/rag.service.d.ts +96 -0
- package/dist/rag.service.d.ts.map +1 -0
- package/dist/rag.service.js +173 -0
- package/dist/rag.service.js.map +1 -0
- package/dist/retrieval/bm25.d.ts +57 -0
- package/dist/retrieval/bm25.d.ts.map +1 -0
- package/dist/retrieval/bm25.js +106 -0
- package/dist/retrieval/bm25.js.map +1 -0
- package/dist/retrieval/hybrid-search.d.ts +48 -0
- package/dist/retrieval/hybrid-search.d.ts.map +1 -0
- package/dist/retrieval/hybrid-search.js +123 -0
- package/dist/retrieval/hybrid-search.js.map +1 -0
- package/dist/retrieval/multi-query.d.ts +38 -0
- package/dist/retrieval/multi-query.d.ts.map +1 -0
- package/dist/retrieval/multi-query.js +135 -0
- package/dist/retrieval/multi-query.js.map +1 -0
- package/dist/text-splitters/recursive-text-splitter.d.ts +21 -0
- package/dist/text-splitters/recursive-text-splitter.d.ts.map +1 -0
- package/dist/text-splitters/recursive-text-splitter.js +95 -0
- package/dist/text-splitters/recursive-text-splitter.js.map +1 -0
- package/dist/types/index.d.ts +144 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +16 -0
- package/dist/types/index.js.map +1 -0
- package/dist/utils/similarity.d.ts +16 -0
- package/dist/utils/similarity.d.ts.map +1 -0
- package/dist/utils/similarity.js +58 -0
- package/dist/utils/similarity.js.map +1 -0
- package/dist/vector-stores/chroma.store.d.ts +42 -0
- package/dist/vector-stores/chroma.store.d.ts.map +1 -0
- package/dist/vector-stores/chroma.store.js +242 -0
- package/dist/vector-stores/chroma.store.js.map +1 -0
- package/dist/vector-stores/memory-vector-store.d.ts +20 -0
- package/dist/vector-stores/memory-vector-store.d.ts.map +1 -0
- package/dist/vector-stores/memory-vector-store.js +94 -0
- package/dist/vector-stores/memory-vector-store.js.map +1 -0
- package/dist/vector-stores/pinecone.store.d.ts +34 -0
- package/dist/vector-stores/pinecone.store.d.ts.map +1 -0
- package/dist/vector-stores/pinecone.store.js +146 -0
- package/dist/vector-stores/pinecone.store.js.map +1 -0
- package/dist/vector-stores/qdrant.store.d.ts +33 -0
- package/dist/vector-stores/qdrant.store.d.ts.map +1 -0
- package/dist/vector-stores/qdrant.store.js +174 -0
- package/dist/vector-stores/qdrant.store.js.map +1 -0
- package/dist/vector-stores/weaviate.store.d.ts +37 -0
- package/dist/vector-stores/weaviate.store.d.ts.map +1 -0
- package/dist/vector-stores/weaviate.store.js +226 -0
- package/dist/vector-stores/weaviate.store.js.map +1 -0
- package/package.json +146 -0
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const knowledge_graph_1 = require("../../graph/knowledge-graph");
|
|
4
|
+
function makeEntity(overrides = {}) {
|
|
5
|
+
return {
|
|
6
|
+
id: 'e1',
|
|
7
|
+
name: 'TypeScript',
|
|
8
|
+
type: 'TECHNOLOGY',
|
|
9
|
+
description: 'A typed superset of JavaScript.',
|
|
10
|
+
sourceDocIds: ['doc1'],
|
|
11
|
+
...overrides,
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
function makeRelationship(overrides = {}) {
|
|
15
|
+
return {
|
|
16
|
+
id: 'r1',
|
|
17
|
+
sourceId: 'e1',
|
|
18
|
+
targetId: 'e2',
|
|
19
|
+
type: 'USES',
|
|
20
|
+
description: 'TypeScript uses JavaScript.',
|
|
21
|
+
weight: 7,
|
|
22
|
+
sourceDocIds: ['doc1'],
|
|
23
|
+
...overrides,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
describe('GraphStore — entities', () => {
|
|
27
|
+
let store;
|
|
28
|
+
beforeEach(() => {
|
|
29
|
+
store = new knowledge_graph_1.GraphStore();
|
|
30
|
+
});
|
|
31
|
+
it('adds an entity and retrieves it', () => {
|
|
32
|
+
const e = makeEntity();
|
|
33
|
+
store.addEntity(e);
|
|
34
|
+
expect(store.entities.get('e1')).toEqual(e);
|
|
35
|
+
});
|
|
36
|
+
it('initialises adjacency entry on addEntity', () => {
|
|
37
|
+
store.addEntity(makeEntity());
|
|
38
|
+
expect(store.adjacency.has('e1')).toBe(true);
|
|
39
|
+
});
|
|
40
|
+
it('merges duplicate entity by id', () => {
|
|
41
|
+
store.addEntity(makeEntity({ description: 'original' }));
|
|
42
|
+
store.addEntity(makeEntity({ description: 'additional info', sourceDocIds: ['doc2'] }));
|
|
43
|
+
const merged = store.entities.get('e1');
|
|
44
|
+
expect(merged.description).toContain('original');
|
|
45
|
+
expect(merged.description).toContain('additional info');
|
|
46
|
+
expect(merged.sourceDocIds).toContain('doc1');
|
|
47
|
+
expect(merged.sourceDocIds).toContain('doc2');
|
|
48
|
+
});
|
|
49
|
+
it('mergeEntity throws when entity does not exist', () => {
|
|
50
|
+
expect(() => store.mergeEntity('missing', {})).toThrow('not found');
|
|
51
|
+
});
|
|
52
|
+
it('findEntityByName is case-insensitive', () => {
|
|
53
|
+
store.addEntity(makeEntity({ name: 'TypeScript' }));
|
|
54
|
+
expect(store.findEntityByName('typescript')).toBeDefined();
|
|
55
|
+
expect(store.findEntityByName('TYPESCRIPT')).toBeDefined();
|
|
56
|
+
});
|
|
57
|
+
it('findEntityByName returns undefined for non-existing entity', () => {
|
|
58
|
+
expect(store.findEntityByName('NonExistent')).toBeUndefined();
|
|
59
|
+
});
|
|
60
|
+
it('findEntitiesByName returns partial matches sorted by name length', () => {
|
|
61
|
+
store.addEntity(makeEntity({ id: 'e1', name: 'TypeScript' }));
|
|
62
|
+
store.addEntity(makeEntity({ id: 'e2', name: 'TypeScript Compiler' }));
|
|
63
|
+
const results = store.findEntitiesByName('typescript');
|
|
64
|
+
expect(results).toHaveLength(2);
|
|
65
|
+
expect(results[0].name.length).toBeLessThanOrEqual(results[1].name.length);
|
|
66
|
+
});
|
|
67
|
+
it('findEntitiesByName respects limit', () => {
|
|
68
|
+
for (let i = 0; i < 15; i++) {
|
|
69
|
+
store.addEntity(makeEntity({ id: `e${i}`, name: `TypeScript ${i}` }));
|
|
70
|
+
}
|
|
71
|
+
const results = store.findEntitiesByName('typescript', 5);
|
|
72
|
+
expect(results).toHaveLength(5);
|
|
73
|
+
});
|
|
74
|
+
it('findEntitiesByName returns empty when nothing matches', () => {
|
|
75
|
+
store.addEntity(makeEntity());
|
|
76
|
+
expect(store.findEntitiesByName('Rust')).toHaveLength(0);
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
describe('GraphStore — relationships', () => {
|
|
80
|
+
let store;
|
|
81
|
+
beforeEach(() => {
|
|
82
|
+
store = new knowledge_graph_1.GraphStore();
|
|
83
|
+
store.addEntity(makeEntity({ id: 'e1', name: 'A' }));
|
|
84
|
+
store.addEntity(makeEntity({ id: 'e2', name: 'B' }));
|
|
85
|
+
});
|
|
86
|
+
it('adds a relationship and updates adjacency', () => {
|
|
87
|
+
store.addRelationship(makeRelationship());
|
|
88
|
+
expect(store.relationships.has('r1')).toBe(true);
|
|
89
|
+
expect(store.adjacency.get('e1').has('e2')).toBe(true);
|
|
90
|
+
expect(store.adjacency.get('e2').has('e1')).toBe(true);
|
|
91
|
+
});
|
|
92
|
+
it('creates adjacency entries for previously unknown entity IDs', () => {
|
|
93
|
+
store.addRelationship(makeRelationship({ sourceId: 'e99', targetId: 'e100' }));
|
|
94
|
+
expect(store.adjacency.get('e99')?.has('e100')).toBe(true);
|
|
95
|
+
});
|
|
96
|
+
it('getEntityRelationships returns all rels touching an entity', () => {
|
|
97
|
+
store.addRelationship(makeRelationship({ id: 'r1', sourceId: 'e1', targetId: 'e2' }));
|
|
98
|
+
store.addRelationship(makeRelationship({ id: 'r2', sourceId: 'e2', targetId: 'e1' }));
|
|
99
|
+
const rels = store.getEntityRelationships('e1');
|
|
100
|
+
expect(rels).toHaveLength(2);
|
|
101
|
+
});
|
|
102
|
+
it('getEntityRelationships returns empty for entity with no relationships', () => {
|
|
103
|
+
expect(store.getEntityRelationships('e1')).toHaveLength(0);
|
|
104
|
+
});
|
|
105
|
+
});
|
|
106
|
+
describe('GraphStore — BFS traversal', () => {
|
|
107
|
+
let store;
|
|
108
|
+
beforeEach(() => {
|
|
109
|
+
store = new knowledge_graph_1.GraphStore();
|
|
110
|
+
for (const id of ['e1', 'e2', 'e3', 'e4']) {
|
|
111
|
+
store.addEntity(makeEntity({ id, name: id }));
|
|
112
|
+
}
|
|
113
|
+
store.addRelationship(makeRelationship({ id: 'r1', sourceId: 'e1', targetId: 'e2' }));
|
|
114
|
+
store.addRelationship(makeRelationship({ id: 'r2', sourceId: 'e2', targetId: 'e3' }));
|
|
115
|
+
});
|
|
116
|
+
it('returns only seed entities at depth 0', () => {
|
|
117
|
+
const result = store.bfsNeighbours(['e1'], 0);
|
|
118
|
+
// seed entity plus relationships within visited set
|
|
119
|
+
expect(result.entities.map((e) => e.id)).toContain('e1');
|
|
120
|
+
});
|
|
121
|
+
it('traverses one hop correctly', () => {
|
|
122
|
+
const result = store.bfsNeighbours(['e1'], 1);
|
|
123
|
+
const ids = result.entities.map((e) => e.id);
|
|
124
|
+
expect(ids).toContain('e1');
|
|
125
|
+
expect(ids).toContain('e2');
|
|
126
|
+
});
|
|
127
|
+
it('traverses two hops correctly', () => {
|
|
128
|
+
const result = store.bfsNeighbours(['e1'], 2);
|
|
129
|
+
const ids = result.entities.map((e) => e.id);
|
|
130
|
+
expect(ids).toContain('e3');
|
|
131
|
+
});
|
|
132
|
+
it('returns relationships between visited entities', () => {
|
|
133
|
+
const result = store.bfsNeighbours(['e1'], 2);
|
|
134
|
+
const relIds = result.relationships.map((r) => r.id);
|
|
135
|
+
expect(relIds).toContain('r1');
|
|
136
|
+
expect(relIds).toContain('r2');
|
|
137
|
+
});
|
|
138
|
+
it('stops when frontier is empty', () => {
|
|
139
|
+
const result = store.bfsNeighbours(['e4'], 5);
|
|
140
|
+
// e4 is isolated
|
|
141
|
+
expect(result.entities.map((e) => e.id)).toContain('e4');
|
|
142
|
+
expect(result.relationships).toHaveLength(0);
|
|
143
|
+
});
|
|
144
|
+
});
|
|
145
|
+
describe('GraphStore — communities', () => {
|
|
146
|
+
let store;
|
|
147
|
+
beforeEach(() => {
|
|
148
|
+
store = new knowledge_graph_1.GraphStore();
|
|
149
|
+
});
|
|
150
|
+
it('sets and retrieves communities', () => {
|
|
151
|
+
const communities = [{ id: 'c1', entityIds: ['e1', 'e2'], level: 0 }];
|
|
152
|
+
store.setCommunities(communities);
|
|
153
|
+
expect(store.communities).toHaveLength(1);
|
|
154
|
+
});
|
|
155
|
+
it('getCommunityForEntity finds the right community', () => {
|
|
156
|
+
store.setCommunities([{ id: 'c1', entityIds: ['e1', 'e2'], level: 0 }]);
|
|
157
|
+
expect(store.getCommunityForEntity('e1')?.id).toBe('c1');
|
|
158
|
+
expect(store.getCommunityForEntity('e99')).toBeUndefined();
|
|
159
|
+
});
|
|
160
|
+
it('addCommunityReport stores a report', () => {
|
|
161
|
+
store.addCommunityReport({
|
|
162
|
+
communityId: 'c1',
|
|
163
|
+
title: 'Test',
|
|
164
|
+
summary: 'Summary',
|
|
165
|
+
findings: [],
|
|
166
|
+
rating: 5,
|
|
167
|
+
entityIds: ['e1'],
|
|
168
|
+
});
|
|
169
|
+
expect(store.communityReports.has('c1')).toBe(true);
|
|
170
|
+
});
|
|
171
|
+
});
|
|
172
|
+
describe('GraphStore — getStats', () => {
|
|
173
|
+
it('returns zeroed stats for an empty graph', () => {
|
|
174
|
+
const store = new knowledge_graph_1.GraphStore();
|
|
175
|
+
const stats = store.getStats();
|
|
176
|
+
expect(stats.totalEntities).toBe(0);
|
|
177
|
+
expect(stats.totalRelationships).toBe(0);
|
|
178
|
+
expect(stats.avgRelationshipsPerEntity).toBe(0);
|
|
179
|
+
});
|
|
180
|
+
it('computes entity and relationship type breakdowns', () => {
|
|
181
|
+
const store = new knowledge_graph_1.GraphStore();
|
|
182
|
+
store.addEntity(makeEntity({ id: 'e1', type: 'TECHNOLOGY' }));
|
|
183
|
+
store.addEntity(makeEntity({ id: 'e2', type: 'CONCEPT' }));
|
|
184
|
+
store.addRelationship(makeRelationship({ id: 'r1', type: 'USES' }));
|
|
185
|
+
const stats = store.getStats();
|
|
186
|
+
expect(stats.entityTypeBreakdown['TECHNOLOGY']).toBe(1);
|
|
187
|
+
expect(stats.entityTypeBreakdown['CONCEPT']).toBe(1);
|
|
188
|
+
expect(stats.relationshipTypeBreakdown['USES']).toBe(1);
|
|
189
|
+
expect(stats.topEntities).toBeDefined();
|
|
190
|
+
});
|
|
191
|
+
});
|
|
192
|
+
describe('GraphStore — toJSON & clear', () => {
|
|
193
|
+
it('toJSON serialises the graph', () => {
|
|
194
|
+
const store = new knowledge_graph_1.GraphStore();
|
|
195
|
+
store.addEntity(makeEntity());
|
|
196
|
+
const json = store.toJSON();
|
|
197
|
+
expect(json.entities).toHaveLength(1);
|
|
198
|
+
expect(json.stats.totalEntities).toBe(1);
|
|
199
|
+
});
|
|
200
|
+
it('clear empties the store', () => {
|
|
201
|
+
const store = new knowledge_graph_1.GraphStore();
|
|
202
|
+
store.addEntity(makeEntity());
|
|
203
|
+
store.clear();
|
|
204
|
+
expect(store.entities.size).toBe(0);
|
|
205
|
+
expect(store.relationships.size).toBe(0);
|
|
206
|
+
});
|
|
207
|
+
});
|
|
208
|
+
//# sourceMappingURL=knowledge-graph.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"knowledge-graph.test.js","sourceRoot":"","sources":["../../../src/__tests__/graph/knowledge-graph.test.ts"],"names":[],"mappings":";;AAAA,iEAAyD;AAGzD,SAAS,UAAU,CAAC,YAAkC,EAAE;IACtD,OAAO;QACL,EAAE,EAAE,IAAI;QACR,IAAI,EAAE,YAAY;QAClB,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,iCAAiC;QAC9C,YAAY,EAAE,CAAC,MAAM,CAAC;QACtB,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,YAAwC,EAAE;IAClE,OAAO;QACL,EAAE,EAAE,IAAI;QACR,QAAQ,EAAE,IAAI;QACd,QAAQ,EAAE,IAAI;QACd,IAAI,EAAE,MAAM;QACZ,WAAW,EAAE,6BAA6B;QAC1C,MAAM,EAAE,CAAC;QACT,YAAY,EAAE,CAAC,MAAM,CAAC;QACtB,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;IACrC,IAAI,KAAiB,CAAC;IACtB,UAAU,CAAC,GAAG,EAAE;QACd,KAAK,GAAG,IAAI,4BAAU,EAAE,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,CAAC,GAAG,UAAU,EAAE,CAAC;QACvB,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,KAAK,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACvC,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;QACzD,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,WAAW,EAAE,iBAAiB,EAAE,YAAY,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACxF,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACtE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC;QACpD,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;QAC3D,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;QACpE,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kEAAkE,EAAE,GAAG,EAAE;QAC1E,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC;QAC9D,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAAC;QACvE,MAAM,OAAO,GAAG,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;QACvD,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACxE,CAAC;QACD,MAAM,OAAO,GAAG,KAAK,CAAC,kBAAkB,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAC1D,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,KAAK,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;IAC1C,IAAI,KAAiB,CAAC;IACtB,UAAU,CAAC,GAAG,EAAE;QACd,KAAK,GAAG,IAAI,4BAAU,EAAE,CAAC;QACzB,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;QACrD,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,KAAK,CAAC,eAAe,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAC1C,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,GAAG,EAAE;QACrE,KAAK,CAAC,eAAe,CAAC,gBAAgB,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAC/E,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;QACpE,KAAK,CAAC,eAAe,CAAC,gBAAgB,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACtF,KAAK,CAAC,eAAe,CAAC,gBAAgB,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACtF,MAAM,IAAI,GAAG,KAAK,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uEAAuE,EAAE,GAAG,EAAE;QAC/E,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;IAC1C,IAAI,KAAiB,CAAC;IAEtB,UAAU,CAAC,GAAG,EAAE;QACd,KAAK,GAAG,IAAI,4BAAU,EAAE,CAAC;QACzB,KAAK,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;YAC1C,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;QAChD,CAAC;QACD,KAAK,CAAC,eAAe,CAAC,gBAAgB,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACtF,KAAK,CAAC,eAAe,CAAC,gBAAgB,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACxF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9C,oDAAoD;QACpD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7C,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7C,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACrD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9C,iBAAiB;QACjB,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE;IACxC,IAAI,KAAiB,CAAC;IACtB,UAAU,CAAC,GAAG,EAAE;QACd,KAAK,GAAG,IAAI,4BAAU,EAAE,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACxC,MAAM,WAAW,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACtE,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAClC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,KAAK,CAAC,cAAc,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzD,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,KAAK,CAAC,kBAAkB,CAAC;YACvB,WAAW,EAAE,IAAI;YACjB,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,SAAS;YAClB,QAAQ,EAAE,EAAE;YACZ,MAAM,EAAE,CAAC;YACT,SAAS,EAAE,CAAC,IAAI,CAAC;SAClB,CAAC,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;IACrC,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,KAAK,GAAG,IAAI,4BAAU,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC/B,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,KAAK,GAAG,IAAI,4BAAU,EAAE,CAAC;QAC/B,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC;QAC9D,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;QAC3D,KAAK,CAAC,eAAe,CAAC,gBAAgB,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACpE,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC/B,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,6BAA6B,EAAE,GAAG,EAAE;IAC3C,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,KAAK,GAAG,IAAI,4BAAU,EAAE,CAAC;QAC/B,KAAK,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC;QAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACjC,MAAM,KAAK,GAAG,IAAI,4BAAU,EAAE,CAAC;QAC/B,KAAK,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC;QAC9B,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"base.loader.test.d.ts","sourceRoot":"","sources":["../../../src/__tests__/loaders/base.loader.test.ts"],"names":[],"mappings":"AAAA,OAAO,kBAAkB,CAAC"}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
require("reflect-metadata");
|
|
10
|
+
const base_loader_1 = require("../../loaders/base.loader");
|
|
11
|
+
// Concrete loader for testing BaseDocumentLoader
|
|
12
|
+
class TestLoader extends base_loader_1.BaseDocumentLoader {
|
|
13
|
+
async load() {
|
|
14
|
+
return [this.createDocument('test content', { source: 'test.txt' })];
|
|
15
|
+
}
|
|
16
|
+
exposeCreateDocuments(pages, meta) {
|
|
17
|
+
return this.createDocuments(pages, meta);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
let DecoratedLoader = class DecoratedLoader extends base_loader_1.BaseDocumentLoader {
|
|
21
|
+
async load() {
|
|
22
|
+
return [];
|
|
23
|
+
}
|
|
24
|
+
};
|
|
25
|
+
DecoratedLoader = __decorate([
|
|
26
|
+
(0, base_loader_1.Loader)({
|
|
27
|
+
name: 'DecoratedLoader',
|
|
28
|
+
description: 'A test loader',
|
|
29
|
+
extensions: ['.test'],
|
|
30
|
+
mimeTypes: ['application/test'],
|
|
31
|
+
})
|
|
32
|
+
], DecoratedLoader);
|
|
33
|
+
class UndecoatedLoader extends base_loader_1.BaseDocumentLoader {
|
|
34
|
+
async load() {
|
|
35
|
+
return [];
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
describe('@Loader decorator & getLoaderConfig', () => {
|
|
39
|
+
it('stores config on a decorated class', () => {
|
|
40
|
+
const config = (0, base_loader_1.getLoaderConfig)(DecoratedLoader);
|
|
41
|
+
expect(config).toEqual({
|
|
42
|
+
name: 'DecoratedLoader',
|
|
43
|
+
description: 'A test loader',
|
|
44
|
+
extensions: ['.test'],
|
|
45
|
+
mimeTypes: ['application/test'],
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
it('returns undefined for an undecorated class', () => {
|
|
49
|
+
expect((0, base_loader_1.getLoaderConfig)(UndecoatedLoader)).toBeUndefined();
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
describe('BaseDocumentLoader', () => {
|
|
53
|
+
let loader;
|
|
54
|
+
beforeEach(() => {
|
|
55
|
+
loader = new TestLoader();
|
|
56
|
+
});
|
|
57
|
+
it('createDocument trims content and sets metadata', async () => {
|
|
58
|
+
const docs = await loader.load();
|
|
59
|
+
expect(docs[0].content).toBe('test content');
|
|
60
|
+
expect(docs[0].metadata?.source).toBe('test.txt');
|
|
61
|
+
});
|
|
62
|
+
it('createDocuments returns one doc per non-empty page', () => {
|
|
63
|
+
const docs = loader.exposeCreateDocuments(['page 1', ' ', 'page 3'], { source: 'multi.pdf' });
|
|
64
|
+
expect(docs).toHaveLength(2);
|
|
65
|
+
expect(docs[0].content).toBe('page 1');
|
|
66
|
+
expect(docs[0].metadata?.pageNumber).toBe(1);
|
|
67
|
+
expect(docs[0].metadata?.totalPages).toBe(3);
|
|
68
|
+
expect(docs[1].metadata?.pageNumber).toBe(3);
|
|
69
|
+
});
|
|
70
|
+
it('createDocuments filters out blank pages', () => {
|
|
71
|
+
const docs = loader.exposeCreateDocuments(['', ' ', '\t']);
|
|
72
|
+
expect(docs).toHaveLength(0);
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
describe('DocumentLoaderRegistry', () => {
|
|
76
|
+
beforeEach(() => {
|
|
77
|
+
// Clear registry state between tests by re-registering
|
|
78
|
+
});
|
|
79
|
+
it('registers a loader and retrieves by extension', () => {
|
|
80
|
+
base_loader_1.DocumentLoaderRegistry.register(DecoratedLoader, () => new DecoratedLoader());
|
|
81
|
+
const loader = base_loader_1.DocumentLoaderRegistry.forExtension('.test');
|
|
82
|
+
expect(loader).toBeInstanceOf(DecoratedLoader);
|
|
83
|
+
});
|
|
84
|
+
it('retrieves by MIME type', () => {
|
|
85
|
+
base_loader_1.DocumentLoaderRegistry.register(DecoratedLoader, () => new DecoratedLoader());
|
|
86
|
+
const loader = base_loader_1.DocumentLoaderRegistry.forMimeType('application/test');
|
|
87
|
+
expect(loader).toBeInstanceOf(DecoratedLoader);
|
|
88
|
+
});
|
|
89
|
+
it('returns undefined for unregistered extension', () => {
|
|
90
|
+
expect(base_loader_1.DocumentLoaderRegistry.forExtension('.xyz123')).toBeUndefined();
|
|
91
|
+
});
|
|
92
|
+
it('returns undefined for unregistered MIME type', () => {
|
|
93
|
+
expect(base_loader_1.DocumentLoaderRegistry.forMimeType('application/xyz123')).toBeUndefined();
|
|
94
|
+
});
|
|
95
|
+
it('is case-insensitive for extensions', () => {
|
|
96
|
+
base_loader_1.DocumentLoaderRegistry.register(DecoratedLoader, () => new DecoratedLoader());
|
|
97
|
+
const lower = base_loader_1.DocumentLoaderRegistry.forExtension('.test');
|
|
98
|
+
const upper = base_loader_1.DocumentLoaderRegistry.forExtension('.TEST');
|
|
99
|
+
expect(lower).toBeDefined();
|
|
100
|
+
expect(upper).toBeDefined();
|
|
101
|
+
});
|
|
102
|
+
it('lists registered extensions', () => {
|
|
103
|
+
base_loader_1.DocumentLoaderRegistry.register(DecoratedLoader, () => new DecoratedLoader());
|
|
104
|
+
const exts = base_loader_1.DocumentLoaderRegistry.registeredExtensions();
|
|
105
|
+
expect(exts).toContain('.test');
|
|
106
|
+
});
|
|
107
|
+
it('does nothing when class has no @Loader decorator', () => {
|
|
108
|
+
const before = base_loader_1.DocumentLoaderRegistry.registeredExtensions().length;
|
|
109
|
+
base_loader_1.DocumentLoaderRegistry.register(UndecoatedLoader, () => new UndecoatedLoader());
|
|
110
|
+
const after = base_loader_1.DocumentLoaderRegistry.registeredExtensions().length;
|
|
111
|
+
expect(after).toBe(before);
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
//# sourceMappingURL=base.loader.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"base.loader.test.js","sourceRoot":"","sources":["../../../src/__tests__/loaders/base.loader.test.ts"],"names":[],"mappings":";;;;;;;;AAAA,4BAA0B;AAC1B,2DAKmC;AAGnC,iDAAiD;AACjD,MAAM,UAAW,SAAQ,gCAAkB;IACzC,KAAK,CAAC,IAAI;QACR,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;IACvE,CAAC;IAED,qBAAqB,CAAC,KAAe,EAAE,IAA8B;QACnE,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC3C,CAAC;CACF;AAQD,IAAM,eAAe,GAArB,MAAM,eAAgB,SAAQ,gCAAkB;IAC9C,KAAK,CAAC,IAAI;QACR,OAAO,EAAE,CAAC;IACZ,CAAC;CACF,CAAA;AAJK,eAAe;IANpB,IAAA,oBAAM,EAAC;QACN,IAAI,EAAE,iBAAiB;QACvB,WAAW,EAAE,eAAe;QAC5B,UAAU,EAAE,CAAC,OAAO,CAAC;QACrB,SAAS,EAAE,CAAC,kBAAkB,CAAC;KAChC,CAAC;GACI,eAAe,CAIpB;AAED,MAAM,gBAAiB,SAAQ,gCAAkB;IAC/C,KAAK,CAAC,IAAI;QACR,OAAO,EAAE,CAAC;IACZ,CAAC;CACF;AAED,QAAQ,CAAC,qCAAqC,EAAE,GAAG,EAAE;IACnD,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,MAAM,GAAG,IAAA,6BAAe,EAAC,eAAe,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,IAAI,EAAE,iBAAiB;YACvB,WAAW,EAAE,eAAe;YAC5B,UAAU,EAAE,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE,CAAC,kBAAkB,CAAC;SAChC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,CAAC,IAAA,6BAAe,EAAC,gBAAgB,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;IAC5D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;IAClC,IAAI,MAAkB,CAAC;IAEvB,UAAU,CAAC,GAAG,EAAE;QACd,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC7C,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,MAAM,IAAI,GAAG,MAAM,CAAC,qBAAqB,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;QAC/F,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,IAAI,GAAG,MAAM,CAAC,qBAAqB,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;QAC7D,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,UAAU,CAAC,GAAG,EAAE;QACd,uDAAuD;IACzD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,oCAAsB,CAAC,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,IAAI,eAAe,EAAE,CAAC,CAAC;QAC9E,MAAM,MAAM,GAAG,oCAAsB,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;QAChC,oCAAsB,CAAC,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,IAAI,eAAe,EAAE,CAAC,CAAC;QAC9E,MAAM,MAAM,GAAG,oCAAsB,CAAC,WAAW,CAAC,kBAAkB,CAAC,CAAC;QACtE,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,CAAC,oCAAsB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;IACzE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,CAAC,oCAAsB,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;IACnF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,oCAAsB,CAAC,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,IAAI,eAAe,EAAE,CAAC,CAAC;QAC9E,MAAM,KAAK,GAAG,oCAAsB,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAC3D,MAAM,KAAK,GAAG,oCAAsB,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAC3D,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5B,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,oCAAsB,CAAC,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,IAAI,eAAe,EAAE,CAAC,CAAC;QAC9E,MAAM,IAAI,GAAG,oCAAsB,CAAC,oBAAoB,EAAE,CAAC;QAC3D,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,MAAM,GAAG,oCAAsB,CAAC,oBAAoB,EAAE,CAAC,MAAM,CAAC;QACpE,oCAAsB,CAAC,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC,CAAC;QAChF,MAAM,KAAK,GAAG,oCAAsB,CAAC,oBAAoB,EAAE,CAAC,MAAM,CAAC;QACnE,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv-file.loader.test.d.ts","sourceRoot":"","sources":["../../../src/__tests__/loaders/csv-file.loader.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
jest.mock('fs/promises', () => ({
|
|
4
|
+
readFile: jest.fn(),
|
|
5
|
+
}));
|
|
6
|
+
const promises_1 = require("fs/promises");
|
|
7
|
+
const csv_file_loader_1 = require("../../loaders/csv-file.loader");
|
|
8
|
+
const mockReadFile = promises_1.readFile;
|
|
9
|
+
const CSV_BASIC = `name,age,city\nAlice,30,NYC\nBob,25,LA`;
|
|
10
|
+
const CSV_QUOTED = `name,description\nAlice,"loves cats, dogs"\nBob,"says ""hello""!"`;
|
|
11
|
+
const CSV_NO_HEADER = `Alice,30\nBob,25`;
|
|
12
|
+
const CSV_EMPTY = ``;
|
|
13
|
+
describe('CSVFileLoader', () => {
|
|
14
|
+
afterEach(() => jest.clearAllMocks());
|
|
15
|
+
it('loads CSV with header, one doc per row', async () => {
|
|
16
|
+
mockReadFile.mockResolvedValueOnce(CSV_BASIC);
|
|
17
|
+
const loader = new csv_file_loader_1.CSVFileLoader({ path: '/tmp/data.csv' });
|
|
18
|
+
const docs = await loader.load();
|
|
19
|
+
expect(docs).toHaveLength(2);
|
|
20
|
+
expect(docs[0].content).toContain('Alice');
|
|
21
|
+
expect(docs[0].metadata?.loaderType).toBe('csv');
|
|
22
|
+
expect(docs[0].metadata?.source).toBe('data.csv');
|
|
23
|
+
expect(docs[0].metadata?.row).toBe(0);
|
|
24
|
+
});
|
|
25
|
+
it('uses contentColumns to build content', async () => {
|
|
26
|
+
mockReadFile.mockResolvedValueOnce(CSV_BASIC);
|
|
27
|
+
const loader = new csv_file_loader_1.CSVFileLoader({
|
|
28
|
+
path: '/tmp/data.csv',
|
|
29
|
+
contentColumns: ['name'],
|
|
30
|
+
metadataColumns: ['age', 'city'],
|
|
31
|
+
});
|
|
32
|
+
const docs = await loader.load();
|
|
33
|
+
expect(docs[0].content).toBe('Alice');
|
|
34
|
+
expect(docs[0].metadata?.age).toBe('30');
|
|
35
|
+
expect(docs[0].metadata?.city).toBe('NYC');
|
|
36
|
+
});
|
|
37
|
+
it('uses multiple contentColumns with key:value format', async () => {
|
|
38
|
+
mockReadFile.mockResolvedValueOnce(CSV_BASIC);
|
|
39
|
+
const loader = new csv_file_loader_1.CSVFileLoader({
|
|
40
|
+
path: '/tmp/data.csv',
|
|
41
|
+
contentColumns: ['name', 'city'],
|
|
42
|
+
});
|
|
43
|
+
const docs = await loader.load();
|
|
44
|
+
expect(docs[0].content).toContain('name: Alice');
|
|
45
|
+
expect(docs[0].content).toContain('city: NYC');
|
|
46
|
+
});
|
|
47
|
+
it('handles quoted fields with embedded commas', async () => {
|
|
48
|
+
mockReadFile.mockResolvedValueOnce(CSV_QUOTED);
|
|
49
|
+
const loader = new csv_file_loader_1.CSVFileLoader({ path: '/tmp/data.csv', contentColumns: ['description'] });
|
|
50
|
+
const docs = await loader.load();
|
|
51
|
+
expect(docs[0].content).toBe('loves cats, dogs');
|
|
52
|
+
});
|
|
53
|
+
it('handles escaped quotes inside quoted fields', async () => {
|
|
54
|
+
mockReadFile.mockResolvedValueOnce(CSV_QUOTED);
|
|
55
|
+
const loader = new csv_file_loader_1.CSVFileLoader({ path: '/tmp/data.csv', contentColumns: ['description'] });
|
|
56
|
+
const docs = await loader.load();
|
|
57
|
+
expect(docs[1].content).toContain('"hello"');
|
|
58
|
+
});
|
|
59
|
+
it('handles CSV without header row', async () => {
|
|
60
|
+
mockReadFile.mockResolvedValueOnce(CSV_NO_HEADER);
|
|
61
|
+
const loader = new csv_file_loader_1.CSVFileLoader({ path: '/tmp/data.csv', hasHeader: false });
|
|
62
|
+
const docs = await loader.load();
|
|
63
|
+
expect(docs).toHaveLength(2);
|
|
64
|
+
expect(docs[0].content).toContain('col0');
|
|
65
|
+
});
|
|
66
|
+
it('returns empty array for empty CSV', async () => {
|
|
67
|
+
mockReadFile.mockResolvedValueOnce(CSV_EMPTY);
|
|
68
|
+
const loader = new csv_file_loader_1.CSVFileLoader({ path: '/tmp/empty.csv' });
|
|
69
|
+
const docs = await loader.load();
|
|
70
|
+
expect(docs).toHaveLength(0);
|
|
71
|
+
});
|
|
72
|
+
it('merges extra metadata', async () => {
|
|
73
|
+
mockReadFile.mockResolvedValueOnce(CSV_BASIC);
|
|
74
|
+
const loader = new csv_file_loader_1.CSVFileLoader({
|
|
75
|
+
path: '/tmp/data.csv',
|
|
76
|
+
metadata: { project: 'test' },
|
|
77
|
+
});
|
|
78
|
+
const docs = await loader.load();
|
|
79
|
+
expect(docs[0].metadata?.project).toBe('test');
|
|
80
|
+
});
|
|
81
|
+
it('uses custom delimiter', async () => {
|
|
82
|
+
mockReadFile.mockResolvedValueOnce('a|b|c\n1|2|3');
|
|
83
|
+
const loader = new csv_file_loader_1.CSVFileLoader({ path: '/tmp/data.csv', delimiter: '|' });
|
|
84
|
+
const docs = await loader.load();
|
|
85
|
+
expect(docs).toHaveLength(1);
|
|
86
|
+
});
|
|
87
|
+
it('uses custom content separator', async () => {
|
|
88
|
+
mockReadFile.mockResolvedValueOnce(CSV_BASIC);
|
|
89
|
+
const loader = new csv_file_loader_1.CSVFileLoader({
|
|
90
|
+
path: '/tmp/data.csv',
|
|
91
|
+
contentColumns: ['name', 'city'],
|
|
92
|
+
contentSeparator: ' | ',
|
|
93
|
+
});
|
|
94
|
+
const docs = await loader.load();
|
|
95
|
+
expect(docs[0].content).toContain(' | ');
|
|
96
|
+
});
|
|
97
|
+
});
|
|
98
|
+
//# sourceMappingURL=csv-file.loader.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv-file.loader.test.js","sourceRoot":"","sources":["../../../src/__tests__/loaders/csv-file.loader.test.ts"],"names":[],"mappings":";;AAAA,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC;IAC9B,QAAQ,EAAE,IAAI,CAAC,EAAE,EAAE;CACpB,CAAC,CAAC,CAAC;AAEJ,0CAAuC;AACvC,mEAA8D;AAE9D,MAAM,YAAY,GAAG,mBAAgD,CAAC;AAEtE,MAAM,SAAS,GAAG,wCAAwC,CAAC;AAC3D,MAAM,UAAU,GAAG,mEAAmE,CAAC;AACvF,MAAM,aAAa,GAAG,kBAAkB,CAAC;AACzC,MAAM,SAAS,GAAG,EAAE,CAAC;AAErB,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,SAAS,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;IAEtC,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,YAAY,CAAC,qBAAqB,CAAC,SAAkB,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,IAAI,+BAAa,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;QAC5D,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAClD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACpD,YAAY,CAAC,qBAAqB,CAAC,SAAkB,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,IAAI,+BAAa,CAAC;YAC/B,IAAI,EAAE,eAAe;YACrB,cAAc,EAAE,CAAC,MAAM,CAAC;YACxB,eAAe,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;SACjC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,YAAY,CAAC,qBAAqB,CAAC,SAAkB,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,IAAI,+BAAa,CAAC;YAC/B,IAAI,EAAE,eAAe;YACrB,cAAc,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;SACjC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QACjD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC1D,YAAY,CAAC,qBAAqB,CAAC,UAAmB,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,+BAAa,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QAC7F,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;QAC3D,YAAY,CAAC,qBAAqB,CAAC,UAAmB,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,+BAAa,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QAC7F,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAC9C,YAAY,CAAC,qBAAqB,CAAC,aAAsB,CAAC,CAAC;QAC3D,MAAM,MAAM,GAAG,IAAI,+BAAa,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9E,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QACjD,YAAY,CAAC,qBAAqB,CAAC,SAAkB,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,IAAI,+BAAa,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uBAAuB,EAAE,KAAK,IAAI,EAAE;QACrC,YAAY,CAAC,qBAAqB,CAAC,SAAkB,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,IAAI,+BAAa,CAAC;YAC/B,IAAI,EAAE,eAAe;YACrB,QAAQ,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE;SAC9B,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uBAAuB,EAAE,KAAK,IAAI,EAAE;QACrC,YAAY,CAAC,qBAAqB,CAAC,cAAuB,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,IAAI,+BAAa,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;QAC5E,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;QAC7C,YAAY,CAAC,qBAAqB,CAAC,SAAkB,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,IAAI,+BAAa,CAAC;YAC/B,IAAI,EAAE,eAAe;YACrB,cAAc,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;YAChC,gBAAgB,EAAE,KAAK;SACxB,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"directory.loader.test.d.ts","sourceRoot":"","sources":["../../../src/__tests__/loaders/directory.loader.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
jest.mock('fs/promises', () => ({
|
|
4
|
+
readdir: jest.fn(),
|
|
5
|
+
stat: jest.fn(),
|
|
6
|
+
readFile: jest.fn(),
|
|
7
|
+
}));
|
|
8
|
+
const promises_1 = require("fs/promises");
|
|
9
|
+
const directory_loader_1 = require("../../loaders/directory.loader");
|
|
10
|
+
const mockReaddir = promises_1.readdir;
|
|
11
|
+
const mockStat = promises_1.stat;
|
|
12
|
+
const mockReadFile = promises_1.readFile;
|
|
13
|
+
function makeStat(isDir) {
|
|
14
|
+
return { isDirectory: () => isDir };
|
|
15
|
+
}
|
|
16
|
+
describe('DirectoryLoader', () => {
|
|
17
|
+
afterEach(() => jest.clearAllMocks());
|
|
18
|
+
it('loads .txt files from a flat directory', async () => {
|
|
19
|
+
mockReaddir.mockResolvedValueOnce(['notes.txt']);
|
|
20
|
+
mockStat.mockResolvedValueOnce(makeStat(false));
|
|
21
|
+
mockReadFile.mockResolvedValueOnce('file content');
|
|
22
|
+
const loader = new directory_loader_1.DirectoryLoader({ path: '/tmp/docs' });
|
|
23
|
+
const docs = await loader.load();
|
|
24
|
+
expect(docs.length).toBeGreaterThan(0);
|
|
25
|
+
expect(docs[0].content).toBe('file content');
|
|
26
|
+
expect(docs[0].metadata?.directoryPath).toBe('/tmp/docs');
|
|
27
|
+
});
|
|
28
|
+
it('recurses into subdirectories when recursive is true', async () => {
|
|
29
|
+
mockReaddir
|
|
30
|
+
.mockResolvedValueOnce(['subdir'])
|
|
31
|
+
.mockResolvedValueOnce(['doc.txt']);
|
|
32
|
+
mockStat
|
|
33
|
+
.mockResolvedValueOnce(makeStat(true))
|
|
34
|
+
.mockResolvedValueOnce(makeStat(false));
|
|
35
|
+
mockReadFile.mockResolvedValueOnce('nested content');
|
|
36
|
+
const loader = new directory_loader_1.DirectoryLoader({ path: '/tmp/docs', recursive: true });
|
|
37
|
+
const docs = await loader.load();
|
|
38
|
+
expect(docs.length).toBeGreaterThan(0);
|
|
39
|
+
});
|
|
40
|
+
it('does not recurse when recursive is false', async () => {
|
|
41
|
+
mockReaddir.mockResolvedValueOnce(['subdir']);
|
|
42
|
+
mockStat.mockResolvedValueOnce(makeStat(true));
|
|
43
|
+
const loader = new directory_loader_1.DirectoryLoader({ path: '/tmp/docs', recursive: false });
|
|
44
|
+
const docs = await loader.load();
|
|
45
|
+
expect(docs).toHaveLength(0);
|
|
46
|
+
});
|
|
47
|
+
it('skips hidden files/directories', async () => {
|
|
48
|
+
mockReaddir.mockResolvedValueOnce(['.hidden', 'visible.txt']);
|
|
49
|
+
mockStat.mockResolvedValueOnce(makeStat(false));
|
|
50
|
+
mockReadFile.mockResolvedValueOnce('visible');
|
|
51
|
+
const loader = new directory_loader_1.DirectoryLoader({ path: '/tmp/docs' });
|
|
52
|
+
const docs = await loader.load();
|
|
53
|
+
expect(docs).toHaveLength(1);
|
|
54
|
+
expect(docs[0].content).toBe('visible');
|
|
55
|
+
});
|
|
56
|
+
it('respects includeExtensions filter', async () => {
|
|
57
|
+
mockReaddir.mockResolvedValueOnce(['doc.txt', 'image.png']);
|
|
58
|
+
mockStat
|
|
59
|
+
.mockResolvedValueOnce(makeStat(false))
|
|
60
|
+
.mockResolvedValueOnce(makeStat(false));
|
|
61
|
+
mockReadFile.mockResolvedValueOnce('text content');
|
|
62
|
+
const loader = new directory_loader_1.DirectoryLoader({ path: '/tmp/docs', includeExtensions: ['.txt'] });
|
|
63
|
+
const docs = await loader.load();
|
|
64
|
+
expect(docs).toHaveLength(1);
|
|
65
|
+
});
|
|
66
|
+
it('respects excludeExtensions filter', async () => {
|
|
67
|
+
mockReaddir.mockResolvedValueOnce(['doc.txt', 'data.json']);
|
|
68
|
+
mockStat
|
|
69
|
+
.mockResolvedValueOnce(makeStat(false))
|
|
70
|
+
.mockResolvedValueOnce(makeStat(false));
|
|
71
|
+
mockReadFile.mockResolvedValueOnce('text');
|
|
72
|
+
const loader = new directory_loader_1.DirectoryLoader({ path: '/tmp/docs', excludeExtensions: ['.json'] });
|
|
73
|
+
const docs = await loader.load();
|
|
74
|
+
expect(docs).toHaveLength(1);
|
|
75
|
+
});
|
|
76
|
+
it('stops at maxFiles limit', async () => {
|
|
77
|
+
mockReaddir.mockResolvedValueOnce(['a.txt', 'b.txt', 'c.txt']);
|
|
78
|
+
mockStat
|
|
79
|
+
.mockResolvedValueOnce(makeStat(false))
|
|
80
|
+
.mockResolvedValueOnce(makeStat(false))
|
|
81
|
+
.mockResolvedValueOnce(makeStat(false));
|
|
82
|
+
mockReadFile.mockResolvedValueOnce('a').mockResolvedValueOnce('b');
|
|
83
|
+
const loader = new directory_loader_1.DirectoryLoader({ path: '/tmp/docs', maxFiles: 2 });
|
|
84
|
+
const docs = await loader.load();
|
|
85
|
+
expect(docs.length).toBeLessThanOrEqual(2);
|
|
86
|
+
});
|
|
87
|
+
it('uses user-supplied loader factory', async () => {
|
|
88
|
+
mockReaddir.mockResolvedValueOnce(['doc.txt']);
|
|
89
|
+
mockStat.mockResolvedValueOnce(makeStat(false));
|
|
90
|
+
const fakeDocs = [{ content: 'custom', metadata: {} }];
|
|
91
|
+
const factory = jest.fn().mockReturnValue({ load: jest.fn().mockResolvedValue(fakeDocs) });
|
|
92
|
+
const loader = new directory_loader_1.DirectoryLoader({ path: '/tmp/docs', loaders: { '.txt': factory } });
|
|
93
|
+
const docs = await loader.load();
|
|
94
|
+
expect(docs).toHaveLength(1);
|
|
95
|
+
expect(docs[0].content).toBe('custom');
|
|
96
|
+
expect(factory).toHaveBeenCalledWith(expect.stringContaining('doc.txt'));
|
|
97
|
+
});
|
|
98
|
+
it('auto-detects .md files', async () => {
|
|
99
|
+
mockReaddir.mockResolvedValueOnce(['readme.md']);
|
|
100
|
+
mockStat.mockResolvedValueOnce(makeStat(false));
|
|
101
|
+
mockReadFile.mockResolvedValueOnce('# Hello');
|
|
102
|
+
const loader = new directory_loader_1.DirectoryLoader({ path: '/tmp/docs' });
|
|
103
|
+
const docs = await loader.load();
|
|
104
|
+
expect(docs.length).toBeGreaterThan(0);
|
|
105
|
+
});
|
|
106
|
+
it('auto-detects .json files', async () => {
|
|
107
|
+
mockReaddir.mockResolvedValueOnce(['data.json']);
|
|
108
|
+
mockStat.mockResolvedValueOnce(makeStat(false));
|
|
109
|
+
mockReadFile.mockResolvedValueOnce(JSON.stringify([{ text: 'hello' }]));
|
|
110
|
+
const loader = new directory_loader_1.DirectoryLoader({ path: '/tmp/docs' });
|
|
111
|
+
const docs = await loader.load();
|
|
112
|
+
expect(docs.length).toBeGreaterThan(0);
|
|
113
|
+
});
|
|
114
|
+
it('auto-detects .csv files', async () => {
|
|
115
|
+
mockReaddir.mockResolvedValueOnce(['data.csv']);
|
|
116
|
+
mockStat.mockResolvedValueOnce(makeStat(false));
|
|
117
|
+
mockReadFile.mockResolvedValueOnce('col\nval');
|
|
118
|
+
const loader = new directory_loader_1.DirectoryLoader({ path: '/tmp/docs' });
|
|
119
|
+
const docs = await loader.load();
|
|
120
|
+
expect(docs.length).toBeGreaterThan(0);
|
|
121
|
+
});
|
|
122
|
+
it('auto-detects .html files', async () => {
|
|
123
|
+
mockReaddir.mockResolvedValueOnce(['page.html']);
|
|
124
|
+
mockStat.mockResolvedValueOnce(makeStat(false));
|
|
125
|
+
mockReadFile.mockResolvedValueOnce('<p>hello</p>');
|
|
126
|
+
const loader = new directory_loader_1.DirectoryLoader({ path: '/tmp/docs' });
|
|
127
|
+
const docs = await loader.load();
|
|
128
|
+
expect(docs.length).toBeGreaterThan(0);
|
|
129
|
+
});
|
|
130
|
+
it('skips files with unknown extensions gracefully', async () => {
|
|
131
|
+
mockReaddir.mockResolvedValueOnce(['binary.exe']);
|
|
132
|
+
mockStat.mockResolvedValueOnce(makeStat(false));
|
|
133
|
+
const loader = new directory_loader_1.DirectoryLoader({ path: '/tmp/docs' });
|
|
134
|
+
const docs = await loader.load();
|
|
135
|
+
expect(docs).toHaveLength(0);
|
|
136
|
+
});
|
|
137
|
+
it('skips files that throw errors during loading', async () => {
|
|
138
|
+
mockReaddir.mockResolvedValueOnce(['bad.txt']);
|
|
139
|
+
mockStat.mockResolvedValueOnce(makeStat(false));
|
|
140
|
+
mockReadFile.mockRejectedValueOnce(new Error('ENOENT'));
|
|
141
|
+
const loader = new directory_loader_1.DirectoryLoader({ path: '/tmp/docs' });
|
|
142
|
+
const docs = await loader.load();
|
|
143
|
+
expect(docs).toHaveLength(0);
|
|
144
|
+
});
|
|
145
|
+
it('merges extra metadata into documents', async () => {
|
|
146
|
+
mockReaddir.mockResolvedValueOnce(['notes.txt']);
|
|
147
|
+
mockStat.mockResolvedValueOnce(makeStat(false));
|
|
148
|
+
mockReadFile.mockResolvedValueOnce('content');
|
|
149
|
+
const loader = new directory_loader_1.DirectoryLoader({ path: '/tmp/docs', metadata: { org: 'hazel' } });
|
|
150
|
+
const docs = await loader.load();
|
|
151
|
+
expect(docs[0].metadata?.org).toBe('hazel');
|
|
152
|
+
});
|
|
153
|
+
});
|
|
154
|
+
//# sourceMappingURL=directory.loader.test.js.map
|