@cogitator-ai/rag 0.5.5 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -92
- package/dist/chunkers/chunk-id.d.ts +3 -0
- package/dist/chunkers/chunk-id.d.ts.map +1 -0
- package/dist/chunkers/chunk-id.js +5 -0
- package/dist/chunkers/chunk-id.js.map +1 -0
- package/dist/chunkers/fixed-chunker.js +2 -2
- package/dist/chunkers/fixed-chunker.js.map +1 -1
- package/dist/chunkers/recursive-chunker.js +2 -2
- package/dist/chunkers/recursive-chunker.js.map +1 -1
- package/dist/chunkers/semantic-chunker.js +4 -4
- package/dist/chunkers/semantic-chunker.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/loaders/csv-loader.d.ts.map +1 -1
- package/dist/loaders/csv-loader.js +2 -2
- package/dist/loaders/csv-loader.js.map +1 -1
- package/dist/loaders/document-id.d.ts +7 -0
- package/dist/loaders/document-id.d.ts.map +1 -0
- package/dist/loaders/document-id.js +11 -0
- package/dist/loaders/document-id.js.map +1 -0
- package/dist/loaders/html-loader.d.ts.map +1 -1
- package/dist/loaders/html-loader.js +2 -2
- package/dist/loaders/html-loader.js.map +1 -1
- package/dist/loaders/json-loader.d.ts.map +1 -1
- package/dist/loaders/json-loader.js +5 -5
- package/dist/loaders/json-loader.js.map +1 -1
- package/dist/loaders/markdown-loader.d.ts.map +1 -1
- package/dist/loaders/markdown-loader.js +2 -2
- package/dist/loaders/markdown-loader.js.map +1 -1
- package/dist/loaders/pdf-loader.d.ts.map +1 -1
- package/dist/loaders/pdf-loader.js +3 -3
- package/dist/loaders/pdf-loader.js.map +1 -1
- package/dist/loaders/text-loader.d.ts.map +1 -1
- package/dist/loaders/text-loader.js +2 -2
- package/dist/loaders/text-loader.js.map +1 -1
- package/dist/loaders/web-loader.js +2 -2
- package/dist/loaders/web-loader.js.map +1 -1
- package/dist/rag-pipeline.d.ts +20 -0
- package/dist/rag-pipeline.d.ts.map +1 -1
- package/dist/rag-pipeline.js +90 -12
- package/dist/rag-pipeline.js.map +1 -1
- package/dist/retrievers/chunk-indexer.d.ts +3 -1
- package/dist/retrievers/chunk-indexer.d.ts.map +1 -1
- package/dist/retrievers/chunk-indexer.js.map +1 -1
- package/dist/retrievers/hybrid-retriever.d.ts +1 -0
- package/dist/retrievers/hybrid-retriever.d.ts.map +1 -1
- package/dist/retrievers/hybrid-retriever.js +10 -1
- package/dist/retrievers/hybrid-retriever.js.map +1 -1
- package/dist/retrievers/mmr-retriever.d.ts.map +1 -1
- package/dist/retrievers/mmr-retriever.js +1 -0
- package/dist/retrievers/mmr-retriever.js.map +1 -1
- package/dist/retrievers/similarity-retriever.d.ts.map +1 -1
- package/dist/retrievers/similarity-retriever.js +1 -0
- package/dist/retrievers/similarity-retriever.js.map +1 -1
- package/dist/schema.d.ts +34 -0
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +9 -0
- package/dist/schema.js.map +1 -1
- package/package.json +6 -6
package/README.md
CHANGED
|
@@ -91,98 +91,7 @@ const intranet = new WebLoader({
|
|
|
91
91
|
|
|
92
92
|
#### robots.txt
|
|
93
93
|
|
|
94
|
-
Pass a `robots` checker and every URL is checked against its site's robots.txt before it is fetched, redirect targets included. A disallowed URL fails the load with `RobotsDisallowedError` and is never requested. `RobotsPolicy` from `@cogitator-ai/core` reads and caches robots.txt per site by RFC 9309 (longest match wins, `*` and
|
|
95
|
-
|
|
96
|
-
Retrieval-Augmented Generation pipeline for Cogitator AI agents. Load documents, chunk them, embed, retrieve, and rerank — all with a single builder API.
|
|
97
|
-
|
|
98
|
-
## Installation
|
|
99
|
-
|
|
100
|
-
```bash
|
|
101
|
-
pnpm add @cogitator-ai/rag
|
|
102
|
-
|
|
103
|
-
# Optional dependencies for specific loaders
|
|
104
|
-
pnpm add cheerio # HTML and web page loading
|
|
105
|
-
pnpm add papaparse # CSV loading
|
|
106
|
-
pnpm add pdf-parse # PDF loading
|
|
107
|
-
```
|
|
108
|
-
|
|
109
|
-
`@cogitator-ai/memory` (optional peer) provides the embedding services, vector stores (`InMemoryEmbeddingAdapter`, `PostgresAdapter`, `QdrantAdapter`) and `HybridSearch` used below; any `EmbeddingService` / `EmbeddingAdapter` implementation works. Website docs: [RAG](https://cogitator.app/docs/rag), [Loaders](https://cogitator.app/docs/rag/loaders), [Chunking](https://cogitator.app/docs/rag/chunking), [Retrieval](https://cogitator.app/docs/rag/retrieval), [Reranking](https://cogitator.app/docs/rag/reranking).
|
|
110
|
-
|
|
111
|
-
## Features
|
|
112
|
-
|
|
113
|
-
- **7 Document Loaders** — Text, Markdown, JSON, CSV, HTML, PDF, Web pages
|
|
114
|
-
- **3 Chunking Strategies** — Fixed-size, recursive, semantic (embedding-based)
|
|
115
|
-
- **4 Retrieval Strategies** — Similarity, MMR, hybrid (BM25 + vector), multi-query
|
|
116
|
-
- **2 Rerankers** — LLM-based scoring, Cohere Rerank API
|
|
117
|
-
- **Pipeline Builder** — Fluent API to wire everything together
|
|
118
|
-
- **Agent Tools** — Drop-in `rag_search` and `rag_ingest` tools for Cogitator agents
|
|
119
|
-
- **Zod Validation** — Type-safe configuration with runtime checks
|
|
120
|
-
|
|
121
|
-
---
|
|
122
|
-
|
|
123
|
-
## Quick Start
|
|
124
|
-
|
|
125
|
-
```typescript
|
|
126
|
-
import { RAGPipelineBuilder, TextLoader } from '@cogitator-ai/rag';
|
|
127
|
-
import { InMemoryEmbeddingAdapter, OpenAIEmbeddingService } from '@cogitator-ai/memory';
|
|
128
|
-
|
|
129
|
-
const pipeline = new RAGPipelineBuilder()
|
|
130
|
-
.withLoader(new TextLoader())
|
|
131
|
-
.withEmbeddingService(
|
|
132
|
-
new OpenAIEmbeddingService({
|
|
133
|
-
apiKey: process.env.OPENAI_API_KEY!,
|
|
134
|
-
})
|
|
135
|
-
)
|
|
136
|
-
.withEmbeddingAdapter(new InMemoryEmbeddingAdapter())
|
|
137
|
-
.withConfig({
|
|
138
|
-
chunking: { strategy: 'recursive', chunkSize: 500, chunkOverlap: 50 },
|
|
139
|
-
retrieval: { strategy: 'similarity', topK: 5, threshold: 0.3 },
|
|
140
|
-
})
|
|
141
|
-
.build();
|
|
142
|
-
|
|
143
|
-
// ingest documents from a file or directory
|
|
144
|
-
const { documents, chunks } = await pipeline.ingest('./docs');
|
|
145
|
-
|
|
146
|
-
// query the knowledge base (options override the configured retrieval for this call)
|
|
147
|
-
const results = await pipeline.query('How does authentication work?', { topK: 3 });
|
|
148
|
-
|
|
149
|
-
for (const r of results) {
|
|
150
|
-
console.log(`[${r.score.toFixed(3)}] ${r.source}: ${r.content.slice(0, 100)}...`);
|
|
151
|
-
}
|
|
152
|
-
```
|
|
153
|
-
|
|
154
|
-
Every stored chunk carries the document's loader metadata (e.g. Markdown frontmatter, PDF page numbers, CSV metadata columns, page title/URL) plus `documentId`, `source`, `sourceType`, `order`, `startOffset` and `endOffset`. Retrieval results expose the document path/URL as `result.source` and the rest in `result.metadata`.
|
|
155
|
-
|
|
156
|
-
---
|
|
157
|
-
|
|
158
|
-
## Document Loaders
|
|
159
|
-
|
|
160
|
-
| Loader | Formats | Optional Dep | Notes |
|
|
161
|
-
| ---------------- | --------------------- | ------------ | ------------------------------------------------------ |
|
|
162
|
-
| `TextLoader` | `.txt` | — | Files and directories |
|
|
163
|
-
| `MarkdownLoader` | `.md`, `.mdx` | — | Optional frontmatter -> metadata |
|
|
164
|
-
| `JSONLoader` | `.json` | — | Configurable content field |
|
|
165
|
-
| `CSVLoader` | `.csv` | `papaparse` | Column selection, `row` number in metadata |
|
|
166
|
-
| `HTMLLoader` | `.html`, `.htm` | `cheerio` | CSS selector, scripts/styles removed, block-aware text |
|
|
167
|
-
| `PDFLoader` | `.pdf` | `pdf-parse` | Whole document or one document per page |
|
|
168
|
-
| `WebLoader` | `http://`, `https://` | `cheerio` | HTML, plain text and JSON pages; SSRF-protected |
|
|
169
|
-
|
|
170
|
-
Directory sources (`TextLoader`, `MarkdownLoader`) are loaded in sorted file-name order.
|
|
171
|
-
|
|
172
|
-
### WebLoader security
|
|
173
|
-
|
|
174
|
-
`WebLoader` blocks loopback, private, link-local, CGNAT, multicast and cloud-metadata addresses — including IPv4-mapped IPv6 forms and hostnames that **resolve** to such addresses. The check runs at connect time for every redirect hop, so DNS rebinding cannot bypass it. To ingest an intranet site, opt in explicitly:
|
|
175
|
-
|
|
176
|
-
```typescript
|
|
177
|
-
const intranet = new WebLoader({
|
|
178
|
-
allowPrivateNetwork: true,
|
|
179
|
-
headers: { Authorization: `Bearer ${process.env.WIKI_TOKEN}` },
|
|
180
|
-
timeoutMs: 15_000,
|
|
181
|
-
maxResponseBytes: 10 * 1024 * 1024,
|
|
182
|
-
});
|
|
183
|
-
```
|
|
184
|
-
|
|
185
|
-
patterns, a 4xx robots.txt allows everything, a 5xx or a network failure allows nothing until the cache expires):
|
|
94
|
+
Pass a `robots` checker and every URL is checked against its site's robots.txt before it is fetched, redirect targets included. A disallowed URL fails the load with `RobotsDisallowedError` and is never requested. `RobotsPolicy` from `@cogitator-ai/core` reads and caches robots.txt per site by RFC 9309 (longest match wins, `*` and `$` patterns, a 4xx robots.txt allows everything, a 5xx or a network failure allows nothing until the cache expires):
|
|
186
95
|
|
|
187
96
|
```typescript
|
|
188
97
|
import { RobotsPolicy } from '@cogitator-ai/core';
|
|
@@ -476,6 +385,28 @@ const agent = new Agent({
|
|
|
476
385
|
|
|
477
386
|
---
|
|
478
387
|
|
|
388
|
+
## Re-ingest, Removal and Scoping
|
|
389
|
+
|
|
390
|
+
Ingesting a source again replaces what the pipeline stored for it: once the new chunks are embedded, the old chunks of that `source` are deleted, so no duplicates or stale text are left. Document ids come from the source (plus the row, item or page within it) and chunk ids from the document id and position, so they stay the same across re-ingests. `removeSource(source)` deletes a source, using the `source` results carry (the resolved path for files).
|
|
391
|
+
|
|
392
|
+
```typescript
|
|
393
|
+
await pipeline.ingest('./docs/handbook.md'); // again after an edit: replaced, not duplicated
|
|
394
|
+
await pipeline.removeSource('/srv/app/docs/handbook.md');
|
|
395
|
+
```
|
|
396
|
+
|
|
397
|
+
Every query is scoped to `sourceType: 'document'`, so a store shared with agent memory (for example a Postgres memory store) never returns messages or facts, private ones included, as RAG results. `namespace` in the config keeps several pipelines apart in one store, and `retrieval.filter` (or the `filter` query option) narrows a search further:
|
|
398
|
+
|
|
399
|
+
```typescript
|
|
400
|
+
const pipeline = new RAGPipelineBuilder()
|
|
401
|
+
// ...
|
|
402
|
+
.withConfig({ chunking, retrieval: { strategy: 'similarity', topK: 5 }, namespace: 'handbook' })
|
|
403
|
+
.build();
|
|
404
|
+
|
|
405
|
+
await pipeline.query('vacation policy', { filter: { metadata: { lang: 'en' } } });
|
|
406
|
+
```
|
|
407
|
+
|
|
408
|
+
Replacing and removing need an embedding store with `deleteByFilter`, which every built-in store has. With a custom store without it, chunks of a re-ingested source are appended (with a warning) and `removeSource` throws. A retriever with its own index (`ChunkIndexer`) is told about replaced chunks through `removeChunks()`, as `HybridRetriever` is.
|
|
409
|
+
|
|
479
410
|
## Pipeline Stats
|
|
480
411
|
|
|
481
412
|
```typescript
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunk-id.d.ts","sourceRoot":"","sources":["../../src/chunkers/chunk-id.ts"],"names":[],"mappings":"AAAA,sGAAsG;AACtG,wBAAgB,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEjE"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunk-id.js","sourceRoot":"","sources":["../../src/chunkers/chunk-id.ts"],"names":[],"mappings":"AAAA,sGAAsG;AACtG,MAAM,UAAU,OAAO,CAAC,UAAkB,EAAE,KAAa;IACvD,OAAO,GAAG,UAAU,IAAI,KAAK,EAAE,CAAC;AAClC,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { chunkId } from './chunk-id.js';
|
|
2
2
|
export class FixedSizeChunker {
|
|
3
3
|
chunkSize;
|
|
4
4
|
chunkOverlap;
|
|
@@ -24,7 +24,7 @@ export class FixedSizeChunker {
|
|
|
24
24
|
for (let start = 0; start < text.length; start += step) {
|
|
25
25
|
const end = Math.min(start + this.chunkSize, text.length);
|
|
26
26
|
chunks.push({
|
|
27
|
-
id:
|
|
27
|
+
id: chunkId(documentId, order),
|
|
28
28
|
documentId,
|
|
29
29
|
content: text.slice(start, end),
|
|
30
30
|
startOffset: start,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fixed-chunker.js","sourceRoot":"","sources":["../../src/chunkers/fixed-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"fixed-chunker.js","sourceRoot":"","sources":["../../src/chunkers/fixed-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAQxC,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IAEtC,YAAY,OAAgC;QAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC;QAChD,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,IAAI,EAAE,CAAC;YACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC;gBAC9B,UAAU;gBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;gBAC/B,WAAW,EAAE,KAAK;gBAClB,SAAS,EAAE,GAAG;gBACd,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;YAEH,IAAI,GAAG,KAAK,IAAI,CAAC,MAAM;gBAAE,MAAM;QACjC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { chunkId } from './chunk-id.js';
|
|
2
2
|
const DEFAULT_SEPARATORS = ['\n\n', '\n', '. ', ' ', ''];
|
|
3
3
|
export class RecursiveChunker {
|
|
4
4
|
chunkSize;
|
|
@@ -35,7 +35,7 @@ export class RecursiveChunker {
|
|
|
35
35
|
}
|
|
36
36
|
const chunkEnd = pieces[j - 1].end;
|
|
37
37
|
chunks.push({
|
|
38
|
-
id:
|
|
38
|
+
id: chunkId(documentId, chunks.length),
|
|
39
39
|
documentId,
|
|
40
40
|
content: text.slice(chunkStart, chunkEnd),
|
|
41
41
|
startOffset: chunkStart,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recursive-chunker.js","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"recursive-chunker.js","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,MAAM,kBAAkB,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;AAQzD,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IACrB,UAAU,CAAW;IAEtC,YAAY,OAAgC;QAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;QACvE,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,WAAW,CAAC,MAAc,EAAE,IAAY,EAAE,UAAkB;QAClE,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YACzB,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC;YAEpC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,GAAG,UAAU,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC1E,CAAC,EAAE,CAAC;YACN,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,GAAG,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC;gBACtC,UAAU;gBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC;gBACzC,WAAW,EAAE,UAAU;gBACvB,SAAS,EAAE,QAAQ;gBACnB,KAAK,EAAE,MAAM,CAAC,MAAM;aACrB,CAAC,CAAC;YAEH,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM;gBAAE,MAAM;YAE9B,IAAI,IAAI,GAAG,CAAC,CAAC;YACb,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,aAAa,GAAG,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC;gBACnD,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,CAAE,CAAC,KAAK,IAAI,aAAa,EAAE,CAAC;oBAChE,IAAI,EAAE,CAAC;gBACT,CAAC;gBACD,OAAO,IAAI,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,GAAG,MAAM,CAAC,IAAI,CAAE,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;oBACzE,IAAI,EAAE,CAAC;gBACT,CAAC;YACH,CAAC;YACD,CAAC,GAAG,IAAI,CAAC;QACX,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,IAAU,EAAE,cAAsB;QAChE,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QACxB,IAAI,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,IAAI,KAAK,GAAG,cAAc,CAAC;QAC3B,OAAO,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAE,CAAC;YAC1C,IAAI,SAAS,KAAK,EAAE;gBAAE,MAAM;YAC5B,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAAE,MAAM;YACtE,KAAK,EAAE,CAAC;QACV,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;YAChD,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/C,MAAM,QAAQ,GAAW,EAAE,CAAC;QAC5B,IAAI,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,SAAS,CAAC;YACR,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAC3C,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;gBACnC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;gBACnD,MAAM;YACR,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxE,MAAM,GAAG,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC;QACjC,CAAC;QAED,MAAM,MAAM,GAAW,EAAE,CAAC;QAC1B,IAAI,OAAO,GAAgB,IAAI,CAAC;QAEhC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACpC,IAAI,CAAC,OAAO;gBAAE,SAAS;YAEvB,IAAI,OAAO,IAAI,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC7D,OAAO,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;gBACrD,SAAS;YACX,CAAC;YAED,IAAI,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAElC,IAAI,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;gBACjD,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;gBACzD,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,OAAO,CAAC;YACpB,CAAC;QACH,CAAC;QAED,IAAI,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClC,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAU;QAC1B,MAAM,MAAM,GAAW,EAAE,CAAC;QAC1B,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACvE,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAOD,SAAS,QAAQ,CAAC,IAAY,EAAE,IAAU;IACxC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC1B,OAAO,KAAK,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC;QAAE,KAAK,EAAE,CAAC;IACvD,OAAO,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAE,CAAC;QAAE,GAAG,EAAE,CAAC;IACvD,OAAO,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { chunkId } from './chunk-id.js';
|
|
2
2
|
function cosineSimilarity(a, b) {
|
|
3
3
|
if (a.length !== b.length) {
|
|
4
4
|
throw new Error(`Embedding dimension mismatch: ${a.length} vs ${b.length}. ` +
|
|
@@ -124,7 +124,7 @@ export class SemanticChunker {
|
|
|
124
124
|
const content = originalText.slice(startOffset, endOffset);
|
|
125
125
|
if (content.length <= this.maxChunkSize) {
|
|
126
126
|
chunks.push({
|
|
127
|
-
id:
|
|
127
|
+
id: chunkId(documentId, chunks.length),
|
|
128
128
|
documentId,
|
|
129
129
|
content,
|
|
130
130
|
startOffset,
|
|
@@ -163,7 +163,7 @@ export class SemanticChunker {
|
|
|
163
163
|
const sliceEnd = Math.min(pos + this.maxChunkSize, sentence.length);
|
|
164
164
|
const slice = sentence.slice(pos, sliceEnd);
|
|
165
165
|
chunks.push({
|
|
166
|
-
id:
|
|
166
|
+
id: chunkId(documentId, order),
|
|
167
167
|
documentId,
|
|
168
168
|
content: slice,
|
|
169
169
|
startOffset: sentenceStart + pos,
|
|
@@ -198,7 +198,7 @@ export class SemanticChunker {
|
|
|
198
198
|
const lastStart = this.findSentenceOffset(lastSentence, originalText, startOffset);
|
|
199
199
|
const endOffset = lastStart + lastSentence.length;
|
|
200
200
|
return {
|
|
201
|
-
id:
|
|
201
|
+
id: chunkId(documentId, order),
|
|
202
202
|
documentId,
|
|
203
203
|
content: originalText.slice(startOffset, endOffset),
|
|
204
204
|
startOffset,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAUxC,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,iCAAiC,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,IAAI;YAC1D,iDAAiD,CACpD,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,iBAAiB,GAAG,uBAAuB,CAAC;AAElD,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI;SACR,KAAK,CAAC,iBAAiB,CAAC;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,OAAO,eAAe;IACT,gBAAgB,CAAmB;IACnC,mBAAmB,CAAS;IAC5B,YAAY,CAAS;IACrB,YAAY,CAAS;IAEtC,YAAY,OAA+B;QACzC,IACE,OAAO,CAAC,mBAAmB,KAAK,SAAS;YACzC,CAAC,OAAO,CAAC,mBAAmB,GAAG,CAAC,IAAI,OAAO,CAAC,mBAAmB,GAAG,CAAC,CAAC,EACpE,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;QAClD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAEzE,IAAI,YAAY,GAAG,YAAY,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACjD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,GAAG,CAAC;QAC9D,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,UAAkB;QAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QACrE,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CACb,iCAAiC,UAAU,CAAC,MAAM,gBAAgB,SAAS,CAAC,MAAM,YAAY,CAC/F,CAAC;QACJ,CAAC;QAED,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,gBAAgB,CAAC,MAAkB;QACzC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAEtC,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAEhE,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,cAAc,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClF,OAAO,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,kBAAkB,CAAC,QAAgB,EAAE,YAAoB,EAAE,UAAkB;QACnF,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACvD,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;IACrC,CAAC;IAEO,WAAW,CACjB,MAAkB,EAClB,YAAoB,EACpB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;YAChF,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;YACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;YAElD,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAE3D,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC;oBACtC,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,KAAK,EAAE,MAAM,CAAC,MAAM;iBACrB,CAAC,CAAC;gBACH,UAAU,GAAG,SAAS,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CACpC,KAAK,EACL,YAAY,EACZ,UAAU,EACV,UAAU,EACV,MAAM,CAAC,MAAM,CACd,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;gBACzD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,eAAe,CACrB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,gBAAgB,GAAa,EAAE,CAAC;QACpC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,KAAK,GAAG,UAAU,CAAC;QACvB,IAAI,MAAM,GAAG,UAAU,CAAC;QAExB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;oBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;oBACzB,gBAAgB,GAAG,EAAE,CAAC;oBACtB,aAAa,GAAG,CAAC,CAAC;gBACpB,CAAC;gBAED,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;gBAC9E,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;oBACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;oBAC5C,MAAM,CAAC,IAAI,CAAC;wBACV,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC;wBAC9B,UAAU;wBACV,OAAO,EAAE,KAAK;wBACd,WAAW,EAAE,aAAa,GAAG,GAAG;wBAChC,SAAS,EAAE,aAAa,GAAG,QAAQ;wBACnC,KAAK,EAAE,KAAK,EAAE;qBACf,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,GAAG,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAE1F,IAAI,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;gBACzB,gBAAgB,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC9B,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChC,aAAa,GAAG,SAAS,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,CAClF,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,iBAAiB,CACvB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,KAAa;QAEb,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;QACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;QAElD,OAAO;YACL,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC;YAC9B,UAAU;YACV,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC;YACnD,WAAW;YACX,SAAS;YACT,KAAK;SACN,CAAC;IACJ,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -7,5 +7,5 @@ export * from './retrievers/index.js';
|
|
|
7
7
|
export * from './rerankers/index.js';
|
|
8
8
|
export * from './schema.js';
|
|
9
9
|
export { createSearchTool, createIngestTool, ragTools, type RAGTool, type IngestToolOptions, } from './tools.js';
|
|
10
|
-
export type { RAGDocument, DocumentChunk, DocumentLoader, Chunker, AsyncChunker, ChunkingConfig, ChunkingStrategy, Retriever, RetrievalConfig, RetrievalStrategy, RetrievalResult, Reranker, RAGPipelineConfig, EmbeddingService, EmbeddingAdapter, } from '@cogitator-ai/types';
|
|
10
|
+
export type { RAGDocument, DocumentChunk, DocumentLoader, Chunker, AsyncChunker, ChunkingConfig, ChunkingStrategy, Retriever, RetrievalConfig, RetrievalStrategy, RetrievalResult, Reranker, RAGPipelineConfig, EmbeddingService, EmbeddingAdapter, EmbeddingDeleteFilter, SearchFilter, } from '@cogitator-ai/types';
|
|
11
11
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,WAAW,EAAE,KAAK,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AAErC,cAAc,aAAa,CAAC;AAE5B,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,iBAAiB,GACvB,MAAM,YAAY,CAAC;AAEpB,YAAY,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,OAAO,EACP,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,SAAS,EACT,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,QAAQ,EACR,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,WAAW,EAAE,KAAK,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AAErC,cAAc,aAAa,CAAC;AAE5B,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,iBAAiB,GACvB,MAAM,YAAY,CAAC;AAEpB,YAAY,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,OAAO,EACP,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,SAAS,EACT,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,QAAQ,EACR,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,qBAAqB,EACrB,YAAY,GACb,MAAM,qBAAqB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"csv-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"csv-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAgBvE,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAW;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAS;gBAExB,OAAO,CAAC,EAAE,gBAAgB;IAMhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAmClD,OAAO,CAAC,eAAe;CAWxB"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises';
|
|
2
2
|
import { resolve } from 'node:path';
|
|
3
|
-
import {
|
|
3
|
+
import { documentId } from './document-id.js';
|
|
4
4
|
async function loadPapaparse() {
|
|
5
5
|
let mod;
|
|
6
6
|
try {
|
|
@@ -42,7 +42,7 @@ export class CSVLoader {
|
|
|
42
42
|
const content = row[contentCol] ?? '';
|
|
43
43
|
const metadata = { ...this.extractMetadata(row), row: index + 1 };
|
|
44
44
|
return {
|
|
45
|
-
id:
|
|
45
|
+
id: documentId(filePath, `row-${index + 1}`),
|
|
46
46
|
content,
|
|
47
47
|
source: filePath,
|
|
48
48
|
sourceType: 'csv',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"csv-loader.js","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"csv-loader.js","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,KAAK,UAAU,aAAa;IAC1B,IAAI,GAA8B,CAAC;IACnC,IAAI,CAAC;QACH,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,WAAW,CAAC,CAA8B,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;IACD,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;IAC5B,OAAO,OAAO,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACxE,CAAC;AAQD,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC;IACjB,aAAa,CAAU;IACvB,eAAe,CAAY;IAC3B,SAAS,CAAU;IAEpC,YAAY,OAA0B;QACpC,IAAI,CAAC,aAAa,GAAG,OAAO,EAAE,aAAa,CAAC;QAC5C,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,CAAC;QAChD,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,IAAI,GAAG,MAAM,aAAa,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE9C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAyB,GAAG,EAAE;YACrD,MAAM,EAAE,IAAI;YACZ,cAAc,EAAE,IAAI;YACpB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QACpD,IAAI,UAAU,KAAK,SAAS;YAAE,OAAO,EAAE,CAAC;QAExC,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,KAAK,CACb,6BAA6B,IAAI,CAAC,aAAa,+BAA+B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACnG,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YACpC,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC;YAElE,OAAO;gBACL,EAAE,EAAE,UAAU,CAAC,QAAQ,EAAE,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;gBAC5C,OAAO;gBACP,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,KAAc;gBAC1B,QAAQ;aACT,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,eAAe,CAAC,GAA2B;QACjD,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;gBACf,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A document id that stays the same each time `source` is loaded, so results cite the same
|
|
3
|
+
* document after a re-ingest. `part` tells apart several documents of one source: a row, an
|
|
4
|
+
* item, a page.
|
|
5
|
+
*/
|
|
6
|
+
export declare function documentId(source: string, part?: string | number): string;
|
|
7
|
+
//# sourceMappingURL=document-id.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"document-id.d.ts","sourceRoot":"","sources":["../../src/loaders/document-id.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAGzE"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
/**
|
|
3
|
+
* A document id that stays the same each time `source` is loaded, so results cite the same
|
|
4
|
+
* document after a re-ingest. `part` tells apart several documents of one source: a row, an
|
|
5
|
+
* item, a page.
|
|
6
|
+
*/
|
|
7
|
+
export function documentId(source, part) {
|
|
8
|
+
const key = part === undefined ? source : `${source}#${part}`;
|
|
9
|
+
return `doc_${createHash('sha256').update(key).digest('hex').slice(0, 24)}`;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=document-id.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"document-id.js","sourceRoot":"","sources":["../../src/loaders/document-id.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC;;;;GAIG;AACH,MAAM,UAAU,UAAU,CAAC,MAAc,EAAE,IAAsB;IAC/D,MAAM,GAAG,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,IAAI,EAAE,CAAC;IAC9D,OAAO,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AAC9E,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"html-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/html-loader.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"html-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/html-loader.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAWvE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAgBD;;;GAGG;AACH,qBAAa,UAAW,YAAW,cAAc;IAC/C,QAAQ,CAAC,cAAc,WAAmB;IAC1C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAEd,OAAO,CAAC,EAAE,iBAAiB;IAIjC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAM5C,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC;CA+BhG"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises';
|
|
2
2
|
import { resolve } from 'node:path';
|
|
3
|
-
import {
|
|
3
|
+
import { documentId } from './document-id.js';
|
|
4
4
|
async function loadCheerio() {
|
|
5
5
|
try {
|
|
6
6
|
return await import('cheerio');
|
|
@@ -53,7 +53,7 @@ export class HTMLLoader {
|
|
|
53
53
|
if (title)
|
|
54
54
|
metadata.title = title;
|
|
55
55
|
return {
|
|
56
|
-
id:
|
|
56
|
+
id: documentId(source),
|
|
57
57
|
content,
|
|
58
58
|
source,
|
|
59
59
|
sourceType,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"html-loader.js","sourceRoot":"","sources":["../../src/loaders/html-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"html-loader.js","sourceRoot":"","sources":["../../src/loaders/html-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,KAAK,UAAU,WAAW;IACxB,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,CAAC;AACH,CAAC;AAMD,MAAM,oBAAoB,GAAG,8DAA8D,CAAC;AAC5F,MAAM,cAAc,GAClB,8FAA8F;IAC9F,6FAA6F,CAAC;AAEhG,SAAS,mBAAmB,CAAC,IAAY;IACvC,OAAO,IAAI;SACR,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;SACvB,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;SACzB,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC;SACxB,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACjC,QAAQ,CAAS;IAE1B,YAAY,OAA2B;QACrC,IAAI,CAAC,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,MAAM,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY,EAAE,MAAc,EAAE,UAA0B;QACtE,MAAM,OAAO,GAAG,MAAM,WAAW,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;QAE5D,CAAC,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC;QACjC,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YACpC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEvB,MAAM,OAAO,GAAG,mBAAmB,CACjC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;aACb,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;aACtC,GAAG,EAAE;aACL,IAAI,CAAC,MAAM,CAAC,CAChB,CAAC;QAEF,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL,EAAE,EAAE,UAAU,CAAC,MAAM,CAAC;YACtB,OAAO;YACP,MAAM;YACN,UAAU;YACV,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SACtD,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"json-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAKvE,MAAM,WAAW,iBAAiB;IAChC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,qBAAa,UAAW,YAAW,cAAc;IAC/C,QAAQ,CAAC,cAAc,WAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAW;gBAE/B,OAAO,CAAC,EAAE,iBAAiB;IAKjC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAgBlD,OAAO,CAAC,cAAc;IAsBtB,OAAO,CAAC,cAAc;IActB,OAAO,CAAC,aAAa;IAKrB,OAAO,CAAC,eAAe;CAWxB"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises';
|
|
2
2
|
import { resolve } from 'node:path';
|
|
3
|
-
import {
|
|
3
|
+
import { documentId } from './document-id.js';
|
|
4
4
|
const CONTENT_FIELDS = ['content', 'text', 'body'];
|
|
5
5
|
export class JSONLoader {
|
|
6
6
|
supportedTypes = ['json'];
|
|
@@ -22,12 +22,12 @@ export class JSONLoader {
|
|
|
22
22
|
throw new Error(`JSONLoader: failed to parse "${filePath}": ${message}`, { cause: err });
|
|
23
23
|
}
|
|
24
24
|
const items = Array.isArray(data) ? data : [data];
|
|
25
|
-
return items.map((item) => this.itemToDocument(item, filePath));
|
|
25
|
+
return items.map((item, index) => this.itemToDocument(item, filePath, index));
|
|
26
26
|
}
|
|
27
|
-
itemToDocument(item, source) {
|
|
27
|
+
itemToDocument(item, source, index) {
|
|
28
28
|
if (typeof item !== 'object' || item === null || Array.isArray(item)) {
|
|
29
29
|
return {
|
|
30
|
-
id:
|
|
30
|
+
id: documentId(source, `item-${index}`),
|
|
31
31
|
content: String(item),
|
|
32
32
|
source,
|
|
33
33
|
sourceType: 'json',
|
|
@@ -37,7 +37,7 @@ export class JSONLoader {
|
|
|
37
37
|
const content = this.extractContent(obj);
|
|
38
38
|
const metadata = this.extractMetadata(obj);
|
|
39
39
|
return {
|
|
40
|
-
id:
|
|
40
|
+
id: documentId(source, `item-${index}`),
|
|
41
41
|
content,
|
|
42
42
|
source,
|
|
43
43
|
sourceType: 'json',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json-loader.js","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"json-loader.js","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAOnD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,YAAY,CAAU;IACtB,cAAc,CAAY;IAE3C,YAAY,OAA2B;QACrC,IAAI,CAAC,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,IAAa,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,MAAM,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3F,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAClD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;IAChF,CAAC;IAEO,cAAc,CAAC,IAAa,EAAE,MAAc,EAAE,KAAa;QACjE,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,OAAO;gBACL,EAAE,EAAE,UAAU,CAAC,MAAM,EAAE,QAAQ,KAAK,EAAE,CAAC;gBACvC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC;gBACrB,MAAM;gBACN,UAAU,EAAE,MAAM;aACnB,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,GAAG,IAA+B,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE3C,OAAO;YACL,EAAE,EAAE,UAAU,CAAC,MAAM,EAAE,QAAQ,KAAK,EAAE,CAAC;YACvC,OAAO;YACP,MAAM;YACN,UAAU,EAAE,MAAM;YAClB,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SAClE,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,GAA4B;QACjD,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,IAAI,GAAG,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACpD,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC;IAEO,aAAa,CAAC,KAAc;QAClC,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAC5C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACxC,CAAC;IAEO,eAAe,CAAC,GAA4B;QAClD,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,SAAS,CAAC;QAE3C,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,QAAQ,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"markdown-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"markdown-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAYvE,MAAM,WAAW,qBAAqB;IACpC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,qBAAa,cAAe,YAAW,cAAc;IACnD,QAAQ,CAAC,cAAc,WAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAU;gBAE/B,OAAO,CAAC,EAAE,qBAAqB;IAIrC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAWpC,QAAQ;IAsBtB,OAAO,CAAC,gBAAgB;YAWV,aAAa;CAiB5B"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { readFile, readdir, stat } from 'node:fs/promises';
|
|
2
2
|
import { join, resolve } from 'node:path';
|
|
3
|
-
import {
|
|
3
|
+
import { documentId } from './document-id.js';
|
|
4
4
|
const MD_EXTENSIONS = new Set(['md', 'mdx']);
|
|
5
5
|
const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/;
|
|
6
6
|
function unquote(value) {
|
|
@@ -33,7 +33,7 @@ export class MarkdownLoader {
|
|
|
33
33
|
}
|
|
34
34
|
}
|
|
35
35
|
return {
|
|
36
|
-
id:
|
|
36
|
+
id: documentId(filePath),
|
|
37
37
|
content,
|
|
38
38
|
source: filePath,
|
|
39
39
|
sourceType: 'markdown',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"markdown-loader.js","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"markdown-loader.js","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAE7C,MAAM,cAAc,GAAG,wCAAwC,CAAC;AAEhE,SAAS,OAAO,CAAC,KAAa;IAC5B,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5C,OAAO,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,KAAK,CAAC;AACrC,CAAC;AAMD,MAAM,OAAO,cAAc;IAChB,cAAc,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACvB,gBAAgB,CAAU;IAE3C,YAAY,OAA+B;QACzC,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,KAAK,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,IAAI,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,QAA6C,CAAC;QAElD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,KAAK,EAAE,CAAC;gBACV,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QAED,OAAO;YACL,EAAE,EAAE,UAAU,CAAC,QAAQ,CAAC;YACxB,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,UAAU;YACtB,GAAG,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC;IAEO,gBAAgB,CAAC,GAAW;QAClC,MAAM,MAAM,GAA4B,EAAE,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pdf-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"pdf-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AA2BvE,MAAM,WAAW,gBAAgB;IAC/B,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;gBAEzB,OAAO,CAAC,EAAE,gBAAgB;IAIhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAapC,KAAK;IAwBnB,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,aAAa;IAYrB,OAAO,CAAC,YAAY;CAkBrB"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises';
|
|
2
2
|
import { resolve } from 'node:path';
|
|
3
|
-
import {
|
|
3
|
+
import { documentId } from './document-id.js';
|
|
4
4
|
async function loadPdfParse() {
|
|
5
5
|
try {
|
|
6
6
|
const mod = await import('pdf-parse');
|
|
@@ -58,7 +58,7 @@ export class PDFLoader {
|
|
|
58
58
|
if (parsed.title)
|
|
59
59
|
metadata.title = parsed.title;
|
|
60
60
|
return {
|
|
61
|
-
id:
|
|
61
|
+
id: documentId(source),
|
|
62
62
|
content: parsed.text,
|
|
63
63
|
source,
|
|
64
64
|
sourceType: 'pdf',
|
|
@@ -77,7 +77,7 @@ export class PDFLoader {
|
|
|
77
77
|
if (title)
|
|
78
78
|
metadata.title = title;
|
|
79
79
|
return {
|
|
80
|
-
id:
|
|
80
|
+
id: documentId(source, `page-${pageNumber}`),
|
|
81
81
|
content,
|
|
82
82
|
source,
|
|
83
83
|
sourceType: 'pdf',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pdf-loader.js","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"pdf-loader.js","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAW9C,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;QACtC,OAAO,GAAG,CAAC,QAAQ,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,IAAa;IACjC,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACtF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACzB,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC3E,CAAC;AAMD,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC;IACjB,UAAU,CAAU;IAErC,YAAY,OAA0B;QACpC,IAAI,CAAC,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,KAAK,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,MAAM,YAAY,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QAE5D,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC9C,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;IACjD,CAAC;IAEO,KAAK,CAAC,KAAK,CACjB,QAA6B,EAC7B,MAAc,EACd,MAAc;QAEd,IAAI,MAAiC,CAAC;QACtC,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACxD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YACxD,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,UAAU,EAAE,MAAM,CAAC,KAAK;gBACxB,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC3D,KAAK,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;aAC/B,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,MAAM,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QACxF,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,EAAE,OAAO,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,MAAiB,EAAE,MAAc;QACtD,MAAM,QAAQ,GAA4B,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC;QACvE,IAAI,MAAM,CAAC,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAEhD,OAAO;YACL,EAAE,EAAE,UAAU,CAAC,MAAM,CAAC;YACtB,OAAO,EAAE,MAAM,CAAC,IAAI;YACpB,MAAM;YACN,UAAU,EAAE,KAAK;YACjB,QAAQ;SACT,CAAC;IACJ,CAAC;IAEO,aAAa,CAAC,MAAiB,EAAE,MAAc;QACrD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAEzE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACtF,CAAC;QAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACxB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,CACvF,CAAC;IACJ,CAAC;IAEO,YAAY,CAClB,OAAe,EACf,MAAc,EACd,UAAkB,EAClB,UAAkB,EAClB,KAAyB;QAEzB,MAAM,QAAQ,GAA4B,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QACrE,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL,EAAE,EAAE,UAAU,CAAC,MAAM,EAAE,QAAQ,UAAU,EAAE,CAAC;YAC5C,OAAO;YACP,MAAM;YACN,UAAU,EAAE,KAAK;YACjB,QAAQ;SACT,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"text-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/text-loader.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"text-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/text-loader.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAKvE,qBAAa,UAAW,YAAW,cAAc;IAC/C,QAAQ,CAAC,cAAc,WAAmB;IAEpC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAWpC,QAAQ;YAUR,aAAa;CAiB5B"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { readFile, readdir, stat } from 'node:fs/promises';
|
|
2
2
|
import { join, resolve } from 'node:path';
|
|
3
|
-
import {
|
|
3
|
+
import { documentId } from './document-id.js';
|
|
4
4
|
const TEXT_EXTENSIONS = new Set(['txt', 'text']);
|
|
5
5
|
export class TextLoader {
|
|
6
6
|
supportedTypes = ['txt', 'text'];
|
|
@@ -15,7 +15,7 @@ export class TextLoader {
|
|
|
15
15
|
async loadFile(filePath) {
|
|
16
16
|
const content = await readFile(filePath, 'utf-8');
|
|
17
17
|
return {
|
|
18
|
-
id:
|
|
18
|
+
id: documentId(filePath),
|
|
19
19
|
content,
|
|
20
20
|
source: filePath,
|
|
21
21
|
sourceType: 'text',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"text-loader.js","sourceRoot":"","sources":["../../src/loaders/text-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"text-loader.js","sourceRoot":"","sources":["../../src/loaders/text-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAEjD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAE1C,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAClD,OAAO;YACL,EAAE,EAAE,UAAU,CAAC,QAAQ,CAAC;YACxB,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,MAAM;SACnB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
|