@cogitator-ai/rag 0.3.5 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -0
- package/dist/loaders/index.d.ts +1 -1
- package/dist/loaders/index.d.ts.map +1 -1
- package/dist/loaders/index.js +1 -1
- package/dist/loaders/index.js.map +1 -1
- package/dist/loaders/web-loader.d.ts +13 -1
- package/dist/loaders/web-loader.d.ts.map +1 -1
- package/dist/loaders/web-loader.js +14 -0
- package/dist/loaders/web-loader.js.map +1 -1
- package/dist/rerankers/index.d.ts +1 -1
- package/dist/rerankers/index.d.ts.map +1 -1
- package/dist/rerankers/index.js +1 -1
- package/dist/rerankers/index.js.map +1 -1
- package/dist/rerankers/llm-reranker.d.ts +27 -0
- package/dist/rerankers/llm-reranker.d.ts.map +1 -1
- package/dist/rerankers/llm-reranker.js +33 -5
- package/dist/rerankers/llm-reranker.js.map +1 -1
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -89,6 +89,112 @@ const intranet = new WebLoader({
|
|
|
89
89
|
});
|
|
90
90
|
```
|
|
91
91
|
|
|
92
|
+
#### robots.txt
|
|
93
|
+
|
|
94
|
+
Pass a `robots` checker and every URL is checked against its site's robots.txt before it is fetched, redirect targets included. A disallowed URL fails the load with `RobotsDisallowedError` and is never requested. `RobotsPolicy` from `@cogitator-ai/core` reads and caches robots.txt per site by RFC 9309 (longest match wins, `*` and `# @cogitator-ai/rag
|
|
95
|
+
|
|
96
|
+
Retrieval-Augmented Generation pipeline for Cogitator AI agents. Load documents, chunk them, embed, retrieve, and rerank — all with a single builder API.
|
|
97
|
+
|
|
98
|
+
## Installation
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
pnpm add @cogitator-ai/rag
|
|
102
|
+
|
|
103
|
+
# Optional dependencies for specific loaders
|
|
104
|
+
pnpm add cheerio # HTML and web page loading
|
|
105
|
+
pnpm add papaparse # CSV loading
|
|
106
|
+
pnpm add pdf-parse # PDF loading
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
`@cogitator-ai/memory` (optional peer) provides the embedding services, vector stores (`InMemoryEmbeddingAdapter`, `PostgresAdapter`, `QdrantAdapter`) and `HybridSearch` used below; any `EmbeddingService` / `EmbeddingAdapter` implementation works. Website docs: [RAG](https://cogitator.app/docs/rag), [Loaders](https://cogitator.app/docs/rag/loaders), [Chunking](https://cogitator.app/docs/rag/chunking), [Retrieval](https://cogitator.app/docs/rag/retrieval), [Reranking](https://cogitator.app/docs/rag/reranking).
|
|
110
|
+
|
|
111
|
+
## Features
|
|
112
|
+
|
|
113
|
+
- **7 Document Loaders** — Text, Markdown, JSON, CSV, HTML, PDF, Web pages
|
|
114
|
+
- **3 Chunking Strategies** — Fixed-size, recursive, semantic (embedding-based)
|
|
115
|
+
- **4 Retrieval Strategies** — Similarity, MMR, hybrid (BM25 + vector), multi-query
|
|
116
|
+
- **2 Rerankers** — LLM-based scoring, Cohere Rerank API
|
|
117
|
+
- **Pipeline Builder** — Fluent API to wire everything together
|
|
118
|
+
- **Agent Tools** — Drop-in `rag_search` and `rag_ingest` tools for Cogitator agents
|
|
119
|
+
- **Zod Validation** — Type-safe configuration with runtime checks
|
|
120
|
+
|
|
121
|
+
---
|
|
122
|
+
|
|
123
|
+
## Quick Start
|
|
124
|
+
|
|
125
|
+
```typescript
|
|
126
|
+
import { RAGPipelineBuilder, TextLoader } from '@cogitator-ai/rag';
|
|
127
|
+
import { InMemoryEmbeddingAdapter, OpenAIEmbeddingService } from '@cogitator-ai/memory';
|
|
128
|
+
|
|
129
|
+
const pipeline = new RAGPipelineBuilder()
|
|
130
|
+
.withLoader(new TextLoader())
|
|
131
|
+
.withEmbeddingService(
|
|
132
|
+
new OpenAIEmbeddingService({
|
|
133
|
+
apiKey: process.env.OPENAI_API_KEY!,
|
|
134
|
+
})
|
|
135
|
+
)
|
|
136
|
+
.withEmbeddingAdapter(new InMemoryEmbeddingAdapter())
|
|
137
|
+
.withConfig({
|
|
138
|
+
chunking: { strategy: 'recursive', chunkSize: 500, chunkOverlap: 50 },
|
|
139
|
+
retrieval: { strategy: 'similarity', topK: 5, threshold: 0.3 },
|
|
140
|
+
})
|
|
141
|
+
.build();
|
|
142
|
+
|
|
143
|
+
// ingest documents from a file or directory
|
|
144
|
+
const { documents, chunks } = await pipeline.ingest('./docs');
|
|
145
|
+
|
|
146
|
+
// query the knowledge base (options override the configured retrieval for this call)
|
|
147
|
+
const results = await pipeline.query('How does authentication work?', { topK: 3 });
|
|
148
|
+
|
|
149
|
+
for (const r of results) {
|
|
150
|
+
console.log(`[${r.score.toFixed(3)}] ${r.source}: ${r.content.slice(0, 100)}...`);
|
|
151
|
+
}
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Every stored chunk carries the document's loader metadata (e.g. Markdown frontmatter, PDF page numbers, CSV metadata columns, page title/URL) plus `documentId`, `source`, `sourceType`, `order`, `startOffset` and `endOffset`. Retrieval results expose the document path/URL as `result.source` and the rest in `result.metadata`.
|
|
155
|
+
|
|
156
|
+
---
|
|
157
|
+
|
|
158
|
+
## Document Loaders
|
|
159
|
+
|
|
160
|
+
| Loader | Formats | Optional Dep | Notes |
|
|
161
|
+
| ---------------- | --------------------- | ------------ | ------------------------------------------------------ |
|
|
162
|
+
| `TextLoader` | `.txt` | — | Files and directories |
|
|
163
|
+
| `MarkdownLoader` | `.md`, `.mdx` | — | Optional frontmatter -> metadata |
|
|
164
|
+
| `JSONLoader` | `.json` | — | Configurable content field |
|
|
165
|
+
| `CSVLoader` | `.csv` | `papaparse` | Column selection, `row` number in metadata |
|
|
166
|
+
| `HTMLLoader` | `.html`, `.htm` | `cheerio` | CSS selector, scripts/styles removed, block-aware text |
|
|
167
|
+
| `PDFLoader` | `.pdf` | `pdf-parse` | Whole document or one document per page |
|
|
168
|
+
| `WebLoader` | `http://`, `https://` | `cheerio` | HTML, plain text and JSON pages; SSRF-protected |
|
|
169
|
+
|
|
170
|
+
Directory sources (`TextLoader`, `MarkdownLoader`) are loaded in sorted file-name order.
|
|
171
|
+
|
|
172
|
+
### WebLoader security
|
|
173
|
+
|
|
174
|
+
`WebLoader` blocks loopback, private, link-local, CGNAT, multicast and cloud-metadata addresses — including IPv4-mapped IPv6 forms and hostnames that **resolve** to such addresses. The check runs at connect time for every redirect hop, so DNS rebinding cannot bypass it. To ingest an intranet site, opt in explicitly:
|
|
175
|
+
|
|
176
|
+
```typescript
|
|
177
|
+
const intranet = new WebLoader({
|
|
178
|
+
allowPrivateNetwork: true,
|
|
179
|
+
headers: { Authorization: `Bearer ${process.env.WIKI_TOKEN}` },
|
|
180
|
+
timeoutMs: 15_000,
|
|
181
|
+
maxResponseBytes: 10 * 1024 * 1024,
|
|
182
|
+
});
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
patterns, a 4xx robots.txt allows everything, a 5xx or a network failure allows nothing until the cache expires):
|
|
186
|
+
|
|
187
|
+
```typescript
|
|
188
|
+
import { RobotsPolicy } from '@cogitator-ai/core';
|
|
189
|
+
import { WebLoader } from '@cogitator-ai/rag';
|
|
190
|
+
|
|
191
|
+
const userAgent = 'NewsBot/1.0 (+https://example.com/bot)';
|
|
192
|
+
const loader = new WebLoader({
|
|
193
|
+
headers: { 'user-agent': userAgent },
|
|
194
|
+
robots: new RobotsPolicy({ userAgent }), // matches "User-agent: NewsBot" groups, else "*"
|
|
195
|
+
});
|
|
196
|
+
```
|
|
197
|
+
|
|
92
198
|
Responses are decompressed (gzip, deflate, br), decoded using the declared charset, and limited to `maxResponseBytes` (50MB by default). Non-text content types are rejected.
|
|
93
199
|
|
|
94
200
|
```typescript
|
|
@@ -307,6 +413,17 @@ const pipeline = new RAGPipelineBuilder()
|
|
|
307
413
|
.build();
|
|
308
414
|
```
|
|
309
415
|
|
|
416
|
+
When the model's answer holds no ranking (a reasoning model that spent its token budget on reasoning returns empty content, for example) or `generateFn` throws, the reranker keeps the retrieval order and logs a warning. Pass `onError` to observe that fallback, or `strict: true` to make `rerank()` throw an `LLMRerankError` (with the raw answer in `response` and the original error in `cause`):
|
|
417
|
+
|
|
418
|
+
```typescript
|
|
419
|
+
const observed = new LLMReranker({
|
|
420
|
+
generateFn: (prompt) => llm.generate(prompt),
|
|
421
|
+
onError: (error, { query, response }) => logger.warn({ query, response }, error.message),
|
|
422
|
+
});
|
|
423
|
+
|
|
424
|
+
const strict = new LLMReranker({ generateFn: (prompt) => llm.generate(prompt), strict: true });
|
|
425
|
+
```
|
|
426
|
+
|
|
310
427
|
### Cohere Reranker
|
|
311
428
|
|
|
312
429
|
Uses the Cohere Rerank API (rerank-v3.5 by default).
|
package/dist/loaders/index.d.ts
CHANGED
|
@@ -9,6 +9,6 @@ export { HTMLLoader } from './html-loader.js';
|
|
|
9
9
|
export type { HTMLLoaderOptions } from './html-loader.js';
|
|
10
10
|
export { PDFLoader } from './pdf-loader.js';
|
|
11
11
|
export type { PDFLoaderOptions } from './pdf-loader.js';
|
|
12
|
-
export { WebLoader } from './web-loader.js';
|
|
12
|
+
export { WebLoader, RobotsDisallowedError } from './web-loader.js';
|
|
13
13
|
export type { WebLoaderOptions } from './web-loader.js';
|
|
14
14
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,YAAY,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,YAAY,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,YAAY,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,YAAY,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,YAAY,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,YAAY,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACnE,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC"}
|
package/dist/loaders/index.js
CHANGED
|
@@ -4,5 +4,5 @@ export { JSONLoader } from './json-loader.js';
|
|
|
4
4
|
export { CSVLoader } from './csv-loader.js';
|
|
5
5
|
export { HTMLLoader } from './html-loader.js';
|
|
6
6
|
export { PDFLoader } from './pdf-loader.js';
|
|
7
|
-
export { WebLoader } from './web-loader.js';
|
|
7
|
+
export { WebLoader, RobotsDisallowedError } from './web-loader.js';
|
|
8
8
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,SAAS,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type LookupAddress, type LookupOptions } from 'node:dns';
|
|
2
|
-
import type { DocumentLoader, RAGDocument } from '@cogitator-ai/types';
|
|
2
|
+
import type { DocumentLoader, RAGDocument, RobotsChecker } from '@cogitator-ai/types';
|
|
3
3
|
/**
|
|
4
4
|
* Whether an IP address points to a loopback, private, link-local, CGNAT, multicast or
|
|
5
5
|
* otherwise non-public network (including IPv4-mapped / NAT64 IPv6 forms).
|
|
@@ -22,6 +22,17 @@ export interface WebLoaderOptions {
|
|
|
22
22
|
* Disabled by default to prevent SSRF when URLs come from untrusted input.
|
|
23
23
|
*/
|
|
24
24
|
allowPrivateNetwork?: boolean;
|
|
25
|
+
/**
|
|
26
|
+
* Checks every URL against its site's robots.txt before fetching it, redirect targets
|
|
27
|
+
* included, e.g. `new RobotsPolicy({ userAgent })` from `@cogitator-ai/core`. A URL the site
|
|
28
|
+
* disallows fails the load with `RobotsDisallowedError`
|
|
29
|
+
*/
|
|
30
|
+
robots?: RobotsChecker;
|
|
31
|
+
}
|
|
32
|
+
/** The site's robots.txt does not allow fetching this URL. */
|
|
33
|
+
export declare class RobotsDisallowedError extends Error {
|
|
34
|
+
readonly url: string;
|
|
35
|
+
constructor(url: string);
|
|
25
36
|
}
|
|
26
37
|
/**
|
|
27
38
|
* Loads a web page over http(s). HTML is converted to text (scripts and styles removed);
|
|
@@ -35,6 +46,7 @@ export declare class WebLoader implements DocumentLoader {
|
|
|
35
46
|
private readonly timeoutMs;
|
|
36
47
|
private readonly maxResponseBytes;
|
|
37
48
|
private readonly allowPrivateNetwork;
|
|
49
|
+
private readonly robots?;
|
|
38
50
|
private readonly lookup;
|
|
39
51
|
constructor(options?: WebLoaderOptions);
|
|
40
52
|
load(source: string): Promise<RAGDocument[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"web-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,aAAa,EAAE,MAAM,UAAU,CAAC;AAMvF,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"web-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,KAAK,aAAa,EAAE,KAAK,aAAa,EAAE,MAAM,UAAU,CAAC;AAMvF,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAqDtF;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAUzD;AAED,KAAK,cAAc,GAAG,CACpB,GAAG,EAAE,MAAM,CAAC,cAAc,GAAG,IAAI,EACjC,OAAO,EAAE,MAAM,GAAG,aAAa,EAAE,EACjC,MAAM,CAAC,EAAE,MAAM,KACZ,IAAI,CAAC;AAEV,KAAK,QAAQ,GAAG,CACd,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,cAAc,GAAG,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,IAAI,KAC9E,IAAI,CAAC;AAMV;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,GAAE,QAAyB,IAC7D,UAAU,MAAM,EAAE,SAAS,aAAa,EAAE,UAAU,cAAc,KAAG,IAAI,CAsClF;AA2DD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;;;OAIG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CACxB;AAED,8DAA8D;AAC9D,qBAAa,qBAAsB,SAAQ,KAAK;IAClC,QAAQ,CAAC,GAAG,EAAE,MAAM;gBAAX,GAAG,EAAE,MAAM;CAIjC;AAQD;;;;GAIG;AACH,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAqB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAyB;IAClD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAU;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAgB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAyB;gBAEpC,OAAO,CAAC,EAAE,gBAAgB;IAShC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YA0BpC,SAAS;IAqCvB,OAAO,CAAC,OAAO;IA4Bf,OAAO,CAAC,QAAQ;CA4BjB"}
|
|
@@ -155,6 +155,15 @@ function decompress(response) {
|
|
|
155
155
|
return response;
|
|
156
156
|
}
|
|
157
157
|
}
|
|
158
|
+
/** The site's robots.txt does not allow fetching this URL. */
|
|
159
|
+
export class RobotsDisallowedError extends Error {
|
|
160
|
+
url;
|
|
161
|
+
constructor(url) {
|
|
162
|
+
super(`WebLoader: robots.txt does not allow fetching ${url}`);
|
|
163
|
+
this.url = url;
|
|
164
|
+
this.name = 'RobotsDisallowedError';
|
|
165
|
+
}
|
|
166
|
+
}
|
|
158
167
|
/**
|
|
159
168
|
* Loads a web page over http(s). HTML is converted to text (scripts and styles removed);
|
|
160
169
|
* plain-text and JSON responses are kept as-is. Redirects are followed (max 5) and every
|
|
@@ -167,6 +176,7 @@ export class WebLoader {
|
|
|
167
176
|
timeoutMs;
|
|
168
177
|
maxResponseBytes;
|
|
169
178
|
allowPrivateNetwork;
|
|
179
|
+
robots;
|
|
170
180
|
lookup = createGuardedLookup();
|
|
171
181
|
constructor(options) {
|
|
172
182
|
this.htmlLoader = new HTMLLoader({ selector: options?.selector });
|
|
@@ -174,6 +184,7 @@ export class WebLoader {
|
|
|
174
184
|
this.timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
175
185
|
this.maxResponseBytes = options?.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES;
|
|
176
186
|
this.allowPrivateNetwork = options?.allowPrivateNetwork ?? false;
|
|
187
|
+
this.robots = options?.robots;
|
|
177
188
|
}
|
|
178
189
|
async load(source) {
|
|
179
190
|
const page = await this.fetchPage(source);
|
|
@@ -200,6 +211,9 @@ export class WebLoader {
|
|
|
200
211
|
async fetchPage(source) {
|
|
201
212
|
let current = parseUrl(source, this.allowPrivateNetwork);
|
|
202
213
|
for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) {
|
|
214
|
+
if (this.robots && !(await this.robots.allows(current.href))) {
|
|
215
|
+
throw new RobotsDisallowedError(current.href);
|
|
216
|
+
}
|
|
203
217
|
const response = await this.request(current);
|
|
204
218
|
const status = response.statusCode ?? 0;
|
|
205
219
|
if (status >= 300 && status < 400) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"web-loader.js","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,SAAS,EAA0C,MAAM,UAAU,CAAC;AACvF,OAAO,IAA8B,MAAM,WAAW,CAAC;AACvD,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,sBAAsB,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAEhF,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,MAAM,aAAa,GAAG,CAAC,CAAC;AACxB,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,0BAA0B,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEpD,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,0BAA0B,EAAE,UAAU,CAAC,CAAC,CAAC;AAEzF,MAAM,cAAc,GAAG,IAAI,SAAS,EAAE,CAAC;AACvC,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI;IAC9B,CAAC,SAAS,EAAE,CAAC,CAAC;IACd,CAAC,UAAU,EAAE,CAAC,CAAC;IACf,CAAC,YAAY,EAAE,EAAE,CAAC;IAClB,CAAC,WAAW,EAAE,CAAC,CAAC;IAChB,CAAC,aAAa,EAAE,EAAE,CAAC;IACnB,CAAC,YAAY,EAAE,EAAE,CAAC;IAClB,CAAC,WAAW,EAAE,EAAE,CAAC;IACjB,CAAC,aAAa,EAAE,EAAE,CAAC;IACnB,CAAC,YAAY,EAAE,EAAE,CAAC;IAClB,CAAC,WAAW,EAAE,CAAC,CAAC;IAChB,CAAC,WAAW,EAAE,CAAC,CAAC;CACR,EAAE,CAAC;IACX,cAAc,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AACpD,CAAC;AACD,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI;IAC9B,CAAC,IAAI,EAAE,GAAG,CAAC;IACX,CAAC,KAAK,EAAE,GAAG,CAAC;IACZ,CAAC,QAAQ,EAAE,CAAC,CAAC;IACb,CAAC,QAAQ,EAAE,EAAE,CAAC;IACd,CAAC,QAAQ,EAAE,CAAC,CAAC;CACL,EAAE,CAAC;IACX,cAAc,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;IAChD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACxC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAClC,MAAM,GAAG,GAAG,mCAAmC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3D,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC;YACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC;YAClC,OAAO,CAAC,IAAI,IAAI,CAAC,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,IAAI,MAAM,KAAK,CAAC;QAAE,OAAO,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC1D,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,MAAM,EAAE,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC;QAC5B,IAAI,EAAE;YAAE,OAAO,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAChD,OAAO,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAaD,MAAM,cAAc,GAAa,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE;IACtD,SAAS,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC;AAC/D,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,WAAqB,cAAc;IACrE,OAAO,CAAC,QAAgB,EAAE,OAAsB,EAAE,QAAwB,EAAQ,EAAE;QAClF,QAAQ,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;YACpC,IAAI,GAAG,EAAE,CAAC;gBACR,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBAClB,OAAO;YACT,CAAC;YACD,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CACjC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,CAC9D,CAAC;YACF,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC5E,IAAI,OAAO,EAAE,CAAC;gBACZ,QAAQ,CACN,MAAM,CAAC,MAAM,CACX,IAAI,KAAK,CACP,eAAe,QAAQ,4BAA4B,OAAO,CAAC,OAAO,YAAY,CAC/E,EACD,EAAE,IAAI,EAAE,UAAU,EAAE,CACrB,EACD,EAAE,CACH,CAAC;gBACF,OAAO;YACT,CAAC;YACD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5B,QAAQ,CACN,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,sCAAsC,QAAQ,GAAG,CAAC,EAAE;oBAC1E,IAAI,EAAE,WAAW;iBAClB,CAAC,EACF,EAAE,CACH,CAAC;gBACF,OAAO;YACT,CAAC;YACD,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAE,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC;YAChE,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,SAAiB,EAAE,mBAA4B;IAC/D,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,2BAA2B,SAAS,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CACb,oCAAoC,GAAG,CAAC,QAAQ,qCAAqC,CACtF,CAAC;IACJ,CAAC;IAED,IAAI,mBAAmB;QAAE,OAAO,GAAG,CAAC;IAEpC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ;SAC1B,WAAW,EAAE;SACb,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACtB,IAAI,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,6CAA6C,CAAC,CAAC;IAClG,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,cAAc,CAAC,CAAC;IAC9E,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,WAAmB;IACnD,MAAM,OAAO,GAAG,+BAA+B,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvE,IAAI,OAAO,EAAE,CAAC;QACZ,IAAI,CAAC;YACH,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,UAAU,CAAC,QAAyB;IAC3C,MAAM,QAAQ,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IACnF,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QACvC,KAAK,SAAS;YACZ,OAAO,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QACxC,KAAK,IAAI;YACP,OAAO,QAAQ,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;QACjD;YACE,OAAO,QAAQ,CAAC;IACpB,CAAC;AACH,CAAC;AAoBD;;;;GAIG;AACH,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3B,UAAU,CAAa;IACvB,OAAO,CAA0B;IACjC,SAAS,CAAS;IAClB,gBAAgB,CAAS;IACzB,mBAAmB,CAAU;IAC7B,MAAM,GAAG,mBAAmB,EAAE,CAAC;IAEhD,YAAY,OAA0B;QACpC,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,kBAAkB,CAAC;QAC1D,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,0BAA0B,CAAC;QAChF,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,KAAK,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAElE,IAAI,IAAI,KAAK,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3F,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YACtE,OAAO;gBACL,EAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,EAAE,GAAG,GAAG,CAAC,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,IAAI,WAAW,EAAE,EAAE;aAC3F,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,IAAI,KAAK,kBAAkB,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACtF,OAAO;gBACL;oBACE,EAAE,EAAE,MAAM,EAAE;oBACZ,OAAO,EAAE,IAAI,CAAC,IAAI;oBAClB,MAAM;oBACN,UAAU,EAAE,KAAK;oBACjB,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE;iBAC/C;aACF,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACpF,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,MAAc;QACpC,IAAI,OAAO,GAAG,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAEzD,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,aAAa,EAAE,SAAS,EAAE,EAAE,CAAC;YAChE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;YAExC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;gBAClC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC;gBAC3C,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,KAAK,CACb,uBAAuB,MAAM,iCAAiC,OAAO,CAAC,IAAI,EAAE,CAC7E,CAAC;gBACJ,CAAC;gBACD,OAAO,GAAG,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;gBAC9E,SAAS;YACX,CAAC;YAED,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;gBAClC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,8BAA8B,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,QAAQ,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAC/F,CAAC;YACJ,CAAC;YAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC3D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;YACzD,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,CAAC;QACjF,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,sCAAsC,aAAa,SAAS,MAAM,EAAE,CAAC,CAAC;IACxF,CAAC;IAEO,OAAO,CAAC,GAAQ;QACtB,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3D,OAAO,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACtD,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;gBAC7B,OAAO,EAAE;oBACP,YAAY,EAAE,yBAAyB;oBACvC,MAAM,EAAE,4DAA4D;oBACpE,iBAAiB,EAAE,mBAAmB;oBACtC,GAAG,IAAI,CAAC,OAAO;iBAChB;gBACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;gBAC3C,GAAG,CAAC,CAAC,IAAI,CAAC,mBAAmB,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;aAC1D,CAAC,CAAC;YACH,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YAC9B,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAA8B,EAAE,EAAE;gBACnD,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;oBACzC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACd,CAAC;qBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBACjE,MAAM,CACJ,IAAI,KAAK,CAAC,yBAAyB,GAAG,CAAC,IAAI,oBAAoB,IAAI,CAAC,SAAS,IAAI,CAAC,CACnF,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,GAAG,CAAC,IAAI,YAAY,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBAChF,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,QAAQ,CAAC,QAAyB,EAAE,GAAW;QACrD,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,OAAO,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC7C,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;gBAClC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;gBACtB,IAAI,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBAClC,QAAQ,CAAC,OAAO,EAAE,CAAC;oBACnB,MAAM,CAAC,OAAO,EAAE,CAAC;oBACjB,MAAM,CACJ,IAAI,KAAK,CACP,4BAA4B,GAAG,YAAY,IAAI,CAAC,gBAAgB,cAAc,CAC/E,CACF,CAAC;oBACF,OAAO;gBACT,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACzD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBAClC,MAAM,CAAC,IAAI,KAAK,CAAC,2CAA2C,GAAG,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACtF,CAAC,CAAC,CAAC;YACH,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBACpC,MAAM,CAAC,IAAI,KAAK,CAAC,2CAA2C,GAAG,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACtF,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"web-loader.js","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,SAAS,EAA0C,MAAM,UAAU,CAAC;AACvF,OAAO,IAA8B,MAAM,WAAW,CAAC;AACvD,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,sBAAsB,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAEhF,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,MAAM,aAAa,GAAG,CAAC,CAAC;AACxB,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,0BAA0B,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEpD,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,0BAA0B,EAAE,UAAU,CAAC,CAAC,CAAC;AAEzF,MAAM,cAAc,GAAG,IAAI,SAAS,EAAE,CAAC;AACvC,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI;IAC9B,CAAC,SAAS,EAAE,CAAC,CAAC;IACd,CAAC,UAAU,EAAE,CAAC,CAAC;IACf,CAAC,YAAY,EAAE,EAAE,CAAC;IAClB,CAAC,WAAW,EAAE,CAAC,CAAC;IAChB,CAAC,aAAa,EAAE,EAAE,CAAC;IACnB,CAAC,YAAY,EAAE,EAAE,CAAC;IAClB,CAAC,WAAW,EAAE,EAAE,CAAC;IACjB,CAAC,aAAa,EAAE,EAAE,CAAC;IACnB,CAAC,YAAY,EAAE,EAAE,CAAC;IAClB,CAAC,WAAW,EAAE,CAAC,CAAC;IAChB,CAAC,WAAW,EAAE,CAAC,CAAC;CACR,EAAE,CAAC;IACX,cAAc,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AACpD,CAAC;AACD,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI;IAC9B,CAAC,IAAI,EAAE,GAAG,CAAC;IACX,CAAC,KAAK,EAAE,GAAG,CAAC;IACZ,CAAC,QAAQ,EAAE,CAAC,CAAC;IACb,CAAC,QAAQ,EAAE,EAAE,CAAC;IACd,CAAC,QAAQ,EAAE,CAAC,CAAC;CACL,EAAE,CAAC;IACX,cAAc,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;IAChD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACxC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAClC,MAAM,GAAG,GAAG,mCAAmC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3D,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC;YACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC;YAClC,OAAO,CAAC,IAAI,IAAI,CAAC,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,IAAI,MAAM,KAAK,CAAC;QAAE,OAAO,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC1D,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,MAAM,EAAE,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC;QAC5B,IAAI,EAAE;YAAE,OAAO,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAChD,OAAO,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAaD,MAAM,cAAc,GAAa,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE;IACtD,SAAS,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC;AAC/D,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,WAAqB,cAAc;IACrE,OAAO,CAAC,QAAgB,EAAE,OAAsB,EAAE,QAAwB,EAAQ,EAAE;QAClF,QAAQ,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;YACpC,IAAI,GAAG,EAAE,CAAC;gBACR,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBAClB,OAAO;YACT,CAAC;YACD,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CACjC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,CAC9D,CAAC;YACF,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC5E,IAAI,OAAO,EAAE,CAAC;gBACZ,QAAQ,CACN,MAAM,CAAC,MAAM,CACX,IAAI,KAAK,CACP,eAAe,QAAQ,4BAA4B,OAAO,CAAC,OAAO,YAAY,CAC/E,EACD,EAAE,IAAI,EAAE,UAAU,EAAE,CACrB,EACD,EAAE,CACH,CAAC;gBACF,OAAO;YACT,CAAC;YACD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5B,QAAQ,CACN,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,sCAAsC,QAAQ,GAAG,CAAC,EAAE;oBAC1E,IAAI,EAAE,WAAW;iBAClB,CAAC,EACF,EAAE,CACH,CAAC;gBACF,OAAO;YACT,CAAC;YACD,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAE,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC;YAChE,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,SAAiB,EAAE,mBAA4B;IAC/D,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,2BAA2B,SAAS,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CACb,oCAAoC,GAAG,CAAC,QAAQ,qCAAqC,CACtF,CAAC;IACJ,CAAC;IAED,IAAI,mBAAmB;QAAE,OAAO,GAAG,CAAC;IAEpC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ;SAC1B,WAAW,EAAE;SACb,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACtB,IAAI,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,6CAA6C,CAAC,CAAC;IAClG,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,cAAc,CAAC,CAAC;IAC9E,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,WAAmB;IACnD,MAAM,OAAO,GAAG,+BAA+B,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvE,IAAI,OAAO,EAAE,CAAC;QACZ,IAAI,CAAC;YACH,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,UAAU,CAAC,QAAyB;IAC3C,MAAM,QAAQ,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IACnF,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QACvC,KAAK,SAAS;YACZ,OAAO,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QACxC,KAAK,IAAI;YACP,OAAO,QAAQ,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;QACjD;YACE,OAAO,QAAQ,CAAC;IACpB,CAAC;AACH,CAAC;AAoBD,8DAA8D;AAC9D,MAAM,OAAO,qBAAsB,SAAQ,KAAK;IACzB;IAArB,YAAqB,GAAW;QAC9B,KAAK,CAAC,iDAAiD,GAAG,EAAE,CAAC,CAAC;QAD3C,QAAG,GAAH,GAAG,CAAQ;QAE9B,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAC;IACtC,CAAC;CACF;AAQD;;;;GAIG;AACH,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3B,UAAU,CAAa;IACvB,OAAO,CAA0B;IACjC,SAAS,CAAS;IAClB,gBAAgB,CAAS;IACzB,mBAAmB,CAAU;IAC7B,MAAM,CAAiB;IACvB,MAAM,GAAG,mBAAmB,EAAE,CAAC;IAEhD,YAAY,OAA0B;QACpC,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,kBAAkB,CAAC;QAC1D,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,0BAA0B,CAAC;QAChF,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,KAAK,CAAC;QACjE,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAElE,IAAI,IAAI,KAAK,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3F,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YACtE,OAAO;gBACL,EAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,EAAE,GAAG,GAAG,CAAC,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,IAAI,WAAW,EAAE,EAAE;aAC3F,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,IAAI,KAAK,kBAAkB,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACtF,OAAO;gBACL;oBACE,EAAE,EAAE,MAAM,EAAE;oBACZ,OAAO,EAAE,IAAI,CAAC,IAAI;oBAClB,MAAM;oBACN,UAAU,EAAE,KAAK;oBACjB,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE;iBAC/C;aACF,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACpF,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,MAAc;QACpC,IAAI,OAAO,GAAG,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAEzD,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,aAAa,EAAE,SAAS,EAAE,EAAE,CAAC;YAChE,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBAC7D,MAAM,IAAI,qBAAqB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAChD,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;YAExC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;gBAClC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC;gBAC3C,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,KAAK,CACb,uBAAuB,MAAM,iCAAiC,OAAO,CAAC,IAAI,EAAE,CAC7E,CAAC;gBACJ,CAAC;gBACD,OAAO,GAAG,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;gBAC9E,SAAS;YACX,CAAC;YAED,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;gBAClC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACb,8BAA8B,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,QAAQ,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAC/F,CAAC;YACJ,CAAC;YAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC3D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;YACzD,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,CAAC;QACjF,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,sCAAsC,aAAa,SAAS,MAAM,EAAE,CAAC,CAAC;IACxF,CAAC;IAEO,OAAO,CAAC,GAAQ;QACtB,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3D,OAAO,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACtD,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;gBAC7B,OAAO,EAAE;oBACP,YAAY,EAAE,yBAAyB;oBACvC,MAAM,EAAE,4DAA4D;oBACpE,iBAAiB,EAAE,mBAAmB;oBACtC,GAAG,IAAI,CAAC,OAAO;iBAChB;gBACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;gBAC3C,GAAG,CAAC,CAAC,IAAI,CAAC,mBAAmB,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;aAC1D,CAAC,CAAC;YACH,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YAC9B,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAA8B,EAAE,EAAE;gBACnD,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;oBACzC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACd,CAAC;qBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBACjE,MAAM,CACJ,IAAI,KAAK,CAAC,yBAAyB,GAAG,CAAC,IAAI,oBAAoB,IAAI,CAAC,SAAS,IAAI,CAAC,CACnF,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,GAAG,CAAC,IAAI,YAAY,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBAChF,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,QAAQ,CAAC,QAAyB,EAAE,GAAW;QACrD,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,OAAO,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC7C,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;gBAClC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;gBACtB,IAAI,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBAClC,QAAQ,CAAC,OAAO,EAAE,CAAC;oBACnB,MAAM,CAAC,OAAO,EAAE,CAAC;oBACjB,MAAM,CACJ,IAAI,KAAK,CACP,4BAA4B,GAAG,YAAY,IAAI,CAAC,gBAAgB,cAAc,CAC/E,CACF,CAAC;oBACF,OAAO;gBACT,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACzD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBAClC,MAAM,CAAC,IAAI,KAAK,CAAC,2CAA2C,GAAG,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACtF,CAAC,CAAC,CAAC;YACH,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBACpC,MAAM,CAAC,IAAI,KAAK,CAAC,2CAA2C,GAAG,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACtF,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export { LLMReranker, type LLMRerankerConfig } from './llm-reranker.js';
|
|
1
|
+
export { LLMReranker, LLMRerankError, type LLMRerankerConfig, type LLMRerankFailureContext, } from './llm-reranker.js';
|
|
2
2
|
export { CohereReranker, type CohereRerankerConfig } from './cohere-reranker.js';
|
|
3
3
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/rerankers/index.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/rerankers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,WAAW,EACX,cAAc,EACd,KAAK,iBAAiB,EACtB,KAAK,uBAAuB,GAC7B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,cAAc,EAAE,KAAK,oBAAoB,EAAE,MAAM,sBAAsB,CAAC"}
|
package/dist/rerankers/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/rerankers/index.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/rerankers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,WAAW,EACX,cAAc,GAGf,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,cAAc,EAA6B,MAAM,sBAAsB,CAAC"}
|
|
@@ -1,9 +1,36 @@
|
|
|
1
1
|
import type { Reranker, RetrievalResult } from '@cogitator-ai/types';
|
|
2
|
+
/** What the reranker was doing when it failed. */
|
|
3
|
+
export interface LLMRerankFailureContext {
|
|
4
|
+
query: string;
|
|
5
|
+
/** The model's raw answer, or undefined when `generateFn` itself failed. */
|
|
6
|
+
response?: string;
|
|
7
|
+
}
|
|
2
8
|
export interface LLMRerankerConfig {
|
|
3
9
|
generateFn: (prompt: string) => Promise<string>;
|
|
10
|
+
/**
|
|
11
|
+
* Called when the model's answer cannot be turned into a ranking, or `generateFn` fails. The
|
|
12
|
+
* reranker then returns the results in retrieval order (unless `strict` is set) and does not
|
|
13
|
+
* log to the console.
|
|
14
|
+
*/
|
|
15
|
+
onError?: (error: LLMRerankError, context: LLMRerankFailureContext) => void;
|
|
16
|
+
/** Throw an `LLMRerankError` instead of falling back to the retrieval order. Defaults to false. */
|
|
17
|
+
strict?: boolean;
|
|
18
|
+
}
|
|
19
|
+
/** Reranking failed: the model gave no usable ranking, or `generateFn` threw (see `cause`). */
|
|
20
|
+
export declare class LLMRerankError extends Error {
|
|
21
|
+
/** The model's raw answer, when there was one. */
|
|
22
|
+
readonly response?: string | undefined;
|
|
23
|
+
readonly name = "LLMRerankError";
|
|
24
|
+
constructor(message: string,
|
|
25
|
+
/** The model's raw answer, when there was one. */
|
|
26
|
+
response?: string | undefined, options?: {
|
|
27
|
+
cause?: unknown;
|
|
28
|
+
});
|
|
4
29
|
}
|
|
5
30
|
export declare class LLMReranker implements Reranker {
|
|
6
31
|
private readonly generateFn;
|
|
32
|
+
private readonly onError?;
|
|
33
|
+
private readonly strict;
|
|
7
34
|
constructor(config: LLMRerankerConfig);
|
|
8
35
|
rerank(query: string, results: RetrievalResult[], topN?: number): Promise<RetrievalResult[]>;
|
|
9
36
|
private buildPrompt;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-reranker.d.ts","sourceRoot":"","sources":["../../src/rerankers/llm-reranker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAErE,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"llm-reranker.d.ts","sourceRoot":"","sources":["../../src/rerankers/llm-reranker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAErE,kDAAkD;AAClD,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,4EAA4E;IAC5E,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAChD;;;;OAIG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,uBAAuB,KAAK,IAAI,CAAC;IAC5E,mGAAmG;IACnG,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,+FAA+F;AAC/F,qBAAa,cAAe,SAAQ,KAAK;IAKrC,kDAAkD;IAClD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM;IAL5B,SAAkB,IAAI,oBAAoB;gBAGxC,OAAO,EAAE,MAAM;IACf,kDAAkD;IACzC,QAAQ,CAAC,EAAE,MAAM,YAAA,EAC1B,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAIhC;AAID,qBAAa,WAAY,YAAW,QAAQ;IAC1C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsC;IACjE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAA+B;IACxD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;gBAErB,MAAM,EAAE,iBAAiB;IAM/B,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,eAAe,EAAE,EAC1B,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,eAAe,EAAE,CAAC;IA0C7B,OAAO,CAAC,WAAW;IAenB,OAAO,CAAC,WAAW;CAqCpB"}
|
|
@@ -1,15 +1,30 @@
|
|
|
1
|
+
/** Reranking failed: the model gave no usable ranking, or `generateFn` threw (see `cause`). */
|
|
2
|
+
export class LLMRerankError extends Error {
|
|
3
|
+
response;
|
|
4
|
+
name = 'LLMRerankError';
|
|
5
|
+
constructor(message,
|
|
6
|
+
/** The model's raw answer, when there was one. */
|
|
7
|
+
response, options) {
|
|
8
|
+
super(message, options);
|
|
9
|
+
this.response = response;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
1
12
|
const MAX_DOCUMENTS_IN_PROMPT = 50;
|
|
2
13
|
export class LLMReranker {
|
|
3
14
|
generateFn;
|
|
15
|
+
onError;
|
|
16
|
+
strict;
|
|
4
17
|
constructor(config) {
|
|
5
18
|
this.generateFn = config.generateFn;
|
|
19
|
+
this.onError = config.onError;
|
|
20
|
+
this.strict = config.strict ?? false;
|
|
6
21
|
}
|
|
7
22
|
async rerank(query, results, topN) {
|
|
8
23
|
if (results.length === 0)
|
|
9
24
|
return [];
|
|
25
|
+
let response;
|
|
10
26
|
try {
|
|
11
|
-
|
|
12
|
-
const response = await this.generateFn(prompt);
|
|
27
|
+
response = await this.generateFn(this.buildPrompt(query, results));
|
|
13
28
|
const scores = this.parseScores(response, results.length);
|
|
14
29
|
const scored = results.map((result, i) => {
|
|
15
30
|
const entry = scores.find((s) => s.index === i);
|
|
@@ -23,7 +38,17 @@ export class LLMReranker {
|
|
|
23
38
|
return topN ? reranked.slice(0, topN) : reranked;
|
|
24
39
|
}
|
|
25
40
|
catch (error) {
|
|
26
|
-
|
|
41
|
+
const failure = error instanceof LLMRerankError
|
|
42
|
+
? error
|
|
43
|
+
: new LLMRerankError(`${response === undefined ? 'generateFn failed' : 'Reranking failed'}: ${error instanceof Error ? error.message : String(error)}`, response, { cause: error });
|
|
44
|
+
if (this.strict)
|
|
45
|
+
throw failure;
|
|
46
|
+
if (this.onError) {
|
|
47
|
+
this.onError(failure, { query, response });
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
console.warn('[LLMReranker] Reranking failed, returning original order:', failure);
|
|
51
|
+
}
|
|
27
52
|
const fallback = [...results];
|
|
28
53
|
return topN ? fallback.slice(0, topN) : fallback;
|
|
29
54
|
}
|
|
@@ -42,9 +67,12 @@ export class LLMReranker {
|
|
|
42
67
|
].join('\n');
|
|
43
68
|
}
|
|
44
69
|
parseScores(response, count) {
|
|
70
|
+
if (typeof response !== 'string' || response.trim() === '') {
|
|
71
|
+
throw new LLMRerankError('The model returned an empty response (a reasoning model may have spent its token budget on reasoning)', response);
|
|
72
|
+
}
|
|
45
73
|
const candidates = response.match(/\[[\s\S]*?\]/g);
|
|
46
74
|
if (!candidates)
|
|
47
|
-
throw new
|
|
75
|
+
throw new LLMRerankError('No JSON array found in the response', response);
|
|
48
76
|
for (let i = candidates.length - 1; i >= 0; i--) {
|
|
49
77
|
try {
|
|
50
78
|
const parsed = JSON.parse(candidates[i]);
|
|
@@ -63,7 +91,7 @@ export class LLMReranker {
|
|
|
63
91
|
continue;
|
|
64
92
|
}
|
|
65
93
|
}
|
|
66
|
-
throw new
|
|
94
|
+
throw new LLMRerankError('No JSON array of { index, score } objects found in the response', response);
|
|
67
95
|
}
|
|
68
96
|
}
|
|
69
97
|
//# sourceMappingURL=llm-reranker.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-reranker.js","sourceRoot":"","sources":["../../src/rerankers/llm-reranker.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"llm-reranker.js","sourceRoot":"","sources":["../../src/rerankers/llm-reranker.ts"],"names":[],"mappings":"AAqBA,+FAA+F;AAC/F,MAAM,OAAO,cAAe,SAAQ,KAAK;IAM5B;IALO,IAAI,GAAG,gBAAgB,CAAC;IAE1C,YACE,OAAe;IACf,kDAAkD;IACzC,QAAiB,EAC1B,OAA6B;QAE7B,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAHf,aAAQ,GAAR,QAAQ,CAAS;IAI5B,CAAC;CACF;AAED,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,WAAW;IACL,UAAU,CAAsC;IAChD,OAAO,CAAgC;IACvC,MAAM,CAAU;IAEjC,YAAY,MAAyB;QACnC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QACpC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,MAAM,CACV,KAAa,EACb,OAA0B,EAC1B,IAAa;QAEb,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEpC,IAAI,QAA4B,CAAC;QACjC,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;YACnE,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAE1D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;gBAChD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YACjD,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;gBACrD,GAAG,MAAM;gBACT,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,EAAE,CAAC,CAAC;aAC/C,CAAC,CAAC,CAAC;YAEJ,OAAO,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACnD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GACX,KAAK,YAAY,cAAc;gBAC7B,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,IAAI,cAAc,CAChB,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,kBAAkB,KAClE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CACvD,EAAE,EACF,QAAQ,EACR,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;YACR,IAAI,IAAI,CAAC,MAAM;gBAAE,MAAM,OAAO,CAAC;YAC/B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC7C,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,2DAA2D,EAAE,OAAO,CAAC,CAAC;YACrF,CAAC;YACD,MAAM,QAAQ,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,OAAO,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACnD,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,KAAa,EAAE,OAA0B;QAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEvE,OAAO;YACL,kEAAkE;YAClE,kEAAkE;YAClE,EAAE;YACF,UAAU,KAAK,EAAE;YACjB,EAAE;YACF,YAAY;YACZ,IAAI;SACL,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAEO,WAAW,CAAC,QAAgB,EAAE,KAAa;QACjD,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAC3D,MAAM,IAAI,cAAc,CACtB,uGAAuG,EACvG,QAAQ,CACT,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QACnD,IAAI,CAAC,UAAU;YAAE,MAAM,IAAI,cAAc,CAAC,qCAAqC,EAAE,QAAQ,CAAC,CAAC;QAE3F,KAAK,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;gBAClD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;oBAAE,SAAS;gBAErC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAC1B,CAAC,IAAI,EAA4C,EAAE,CACjD,OAAO,IAAI,KAAK,QAAQ;oBACxB,IAAI,KAAK,IAAI;oBACb,OAAQ,IAAgC,CAAC,KAAK,KAAK,QAAQ;oBAC3D,OAAQ,IAAgC,CAAC,KAAK,KAAK,QAAQ;oBAC1D,IAA0B,CAAC,KAAK,IAAI,CAAC;oBACrC,IAA0B,CAAC,KAAK,GAAG,KAAK,CAC5C,CAAC;gBAEF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;oBAAE,OAAO,MAAM,CAAC;YACvC,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;QACH,CAAC;QAED,MAAM,IAAI,cAAc,CACtB,iEAAiE,EACjE,QAAQ,CACT,CAAC;IACJ,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cogitator-ai/rag",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Retrieval Augmented Generation for Cogitator AI agents",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -18,12 +18,12 @@
|
|
|
18
18
|
"node": ">=22.12.0"
|
|
19
19
|
},
|
|
20
20
|
"dependencies": {
|
|
21
|
-
"@cogitator-ai/types": "0.
|
|
21
|
+
"@cogitator-ai/types": "0.31.0",
|
|
22
22
|
"nanoid": "^6.0.1",
|
|
23
23
|
"zod": "^4.6.5"
|
|
24
24
|
},
|
|
25
25
|
"peerDependencies": {
|
|
26
|
-
"@cogitator-ai/memory": ">=0.11.
|
|
26
|
+
"@cogitator-ai/memory": ">=0.11.2"
|
|
27
27
|
},
|
|
28
28
|
"peerDependenciesMeta": {
|
|
29
29
|
"@cogitator-ai/memory": {
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
"pdf-parse": "^2.4.5"
|
|
37
37
|
},
|
|
38
38
|
"devDependencies": {
|
|
39
|
-
"@cogitator-ai/memory": "0.11.
|
|
39
|
+
"@cogitator-ai/memory": "0.11.2",
|
|
40
40
|
"@types/papaparse": "^5.3.15",
|
|
41
41
|
"typescript": "^6.0.3",
|
|
42
42
|
"vitest": "^5.0.3"
|