@dancrumb/web-crawler 3.0.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/CrawledDocument.d.ts +8 -0
- package/dist/CrawledDocument.d.ts.map +1 -0
- package/dist/CrawledDocument.js +2 -0
- package/dist/CrawledDocument.js.map +1 -0
- package/dist/WebCrawler.d.ts +3 -10
- package/dist/WebCrawler.d.ts.map +1 -1
- package/dist/WebCrawler.js +5 -4
- package/dist/WebCrawler.js.map +1 -1
- package/package.json +4 -3
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CrawledDocument.d.ts","sourceRoot":"","sources":["../src/CrawledDocument.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,eAAe,GAAG;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CrawledDocument.js","sourceRoot":"","sources":["../src/CrawledDocument.ts"],"names":[],"mappings":""}
|
package/dist/WebCrawler.d.ts
CHANGED
|
@@ -1,11 +1,5 @@
|
|
|
1
1
|
import EventEmitter from "node:events";
|
|
2
|
-
type
|
|
3
|
-
url: string;
|
|
4
|
-
canonicalUrl: string;
|
|
5
|
-
content: string | Buffer;
|
|
6
|
-
contentType: string;
|
|
7
|
-
title: string | null;
|
|
8
|
-
};
|
|
2
|
+
import type { CrawledDocument } from "./CrawledDocument.js";
|
|
9
3
|
export declare class WebCrawler extends EventEmitter<{
|
|
10
4
|
progress: [progress: string | null];
|
|
11
5
|
}> {
|
|
@@ -27,7 +21,7 @@ export declare class WebCrawler extends EventEmitter<{
|
|
|
27
21
|
private resolveLink;
|
|
28
22
|
private fetchHead;
|
|
29
23
|
private getContentType;
|
|
30
|
-
crawlLink(initialLink: string,
|
|
24
|
+
crawlLink(initialLink: string, _maxDepth?: number): Promise<string[][]>;
|
|
31
25
|
clear(): void;
|
|
32
26
|
addRoot(root: string): Promise<string>;
|
|
33
27
|
addSupportedType(supportedType: `${string}/${string}`): void;
|
|
@@ -42,7 +36,6 @@ export declare class WebCrawler extends EventEmitter<{
|
|
|
42
36
|
*/
|
|
43
37
|
setMaxDepth(depth: number): number;
|
|
44
38
|
getCrawledUrls(): string[];
|
|
45
|
-
getDocuments():
|
|
39
|
+
getDocuments(): CrawledDocument[];
|
|
46
40
|
}
|
|
47
|
-
export {};
|
|
48
41
|
//# sourceMappingURL=WebCrawler.d.ts.map
|
package/dist/WebCrawler.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WebCrawler.d.ts","sourceRoot":"","sources":["../src/WebCrawler.ts"],"names":[],"mappings":"AAIA,OAAO,YAAY,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"WebCrawler.d.ts","sourceRoot":"","sources":["../src/WebCrawler.ts"],"names":[],"mappings":"AAIA,OAAO,YAAY,MAAM,aAAa,CAAC;AACvC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAM5D,qBAAa,UAAW,SAAQ,YAAY,CAAC;IAC3C,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;CACrC,CAAC;IACA,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,SAAS,CAA6B;IAC9C,OAAO,CAAC,SAAS,CAAsC;IACvD,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,cAAc,CAAoC;IAC1D,OAAO,CAAC,QAAQ,CAAa;;IAO7B,OAAO,CAAC,cAAc;IAWtB,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,WAAW;YAUL,YAAY;YA0BZ,WAAW;IAOzB,OAAO,CAAC,WAAW;YAIL,WAAW;YAoBX,YAAY;YAkBZ,WAAW;YAUX,SAAS;IAiCvB,OAAO,CAAC,cAAc;IAYhB,SAAS,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,GAAE,MAAsB;IA8DtE,KAAK;IASC,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAM5C,gBAAgB,CAAC,aAAa,EAAE,GAAG,MAAM,IAAI,MAAM,EAAE;IAIrD,iBAAiB,CAAC,GAAG,cAAc,EAAE,GAAG,MAAM,IAAI,MAAM,EAAE,EAAE;IAK5D;;;;;;;OAOG;IACH,WAAW,CAAC,KAAK,EAAE,MAAM;IAWzB,cAAc;IAId,YAAY,IAAI,eAAe,EAAE;CAKlC"}
|
package/dist/WebCrawler.js
CHANGED
|
@@ -148,17 +148,18 @@ export class WebCrawler extends EventEmitter {
|
|
|
148
148
|
}
|
|
149
149
|
return contentType;
|
|
150
150
|
}
|
|
151
|
-
async crawlLink(initialLink,
|
|
151
|
+
async crawlLink(initialLink, _maxDepth = this.maxDepth) {
|
|
152
152
|
const root = await this.getCanonical(initialLink);
|
|
153
153
|
this.addRoot(root);
|
|
154
|
-
this.maxDepth =
|
|
154
|
+
this.maxDepth = _maxDepth;
|
|
155
|
+
logDebug(`Crawling ${initialLink} (${root}) up to depth ${this.maxDepth}`);
|
|
155
156
|
let [url, depth] = [root, 0];
|
|
156
157
|
while (url !== undefined) {
|
|
157
158
|
assert(depth !== undefined);
|
|
158
159
|
logDebug("\n*********\n");
|
|
159
160
|
logDebug(`Crawled: ${this.crawled.size}\nUncrawled: ${this.uncrawled.size}`);
|
|
160
161
|
logDebug(`URL: ${url}\nDepth:${depth}\n`);
|
|
161
|
-
if (depth <= maxDepth) {
|
|
162
|
+
if (depth <= this.maxDepth) {
|
|
162
163
|
this.markCrawled(url, depth);
|
|
163
164
|
logDebug(`Crawling ${url}...`);
|
|
164
165
|
const links = await this.getAllLinks(url);
|
|
@@ -169,7 +170,7 @@ export class WebCrawler extends EventEmitter {
|
|
|
169
170
|
continue;
|
|
170
171
|
}
|
|
171
172
|
logDebug(`${resolved} is under one of ${[...this.roots.entries()].join(",")}`);
|
|
172
|
-
if ((this.crawled.get(resolved) ?? maxDepth) < depth) {
|
|
173
|
+
if ((this.crawled.get(resolved) ?? this.maxDepth) < depth) {
|
|
173
174
|
logDebug(`Already crawled ${resolved} at depth ${this.crawled.get(resolved)}`);
|
|
174
175
|
continue;
|
|
175
176
|
}
|
package/dist/WebCrawler.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WebCrawler.js","sourceRoot":"","sources":["../src/WebCrawler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,YAAY,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"WebCrawler.js","sourceRoot":"","sources":["../src/WebCrawler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,YAAY,MAAM,aAAa,CAAC;AAEvC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC;AAEvC,MAAM,OAAO,UAAW,SAAQ,YAE9B;IACQ,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpC,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,SAAS,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC/C,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAC1B,cAAc,GAAG,IAAI,GAAG,EAAyB,CAAC;IAClD,QAAQ,GAAW,CAAC,CAAC;IAE7B;QACE,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAEO,cAAc;QACpB,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAC9B,CAAC;QACD,MAAM,QAAQ,GAAG,CACf,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC/D,GAAG,CACJ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACb,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAEO,YAAY,CAAC,GAAW,EAAE,KAAa;QAC7C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC/B,IAAI,CAAC,cAAc,EAAE,CAAC;IACxB,CAAC;IAEO,WAAW,CAAC,GAAW,EAAE,QAAiB;QAChD,IAAI,KAAK,GAAG,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,cAAc,EAAE,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,WAAW,GACf,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,0BAA0B,CAAC;QAC7D,MAAM,OAAO,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;YACtB,GAAG;YACH,YAAY,EAAE,EAAE;YAChB,OAAO;YACP,KAAK,EAAE,IAAI;YACX,WAAW;SACZ,CAAC,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,YAAY,KAAK,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,EAAE;gBAC/B,GAAG;gBACH,YAAY;gBACZ,OAAO;gBACP,KAAK,EAAE,IAAI;gBACX,WAAW;aACZ,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,GAAW;QACnC,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,OAAO,CAAC;QAC1C,CAAC;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAEO,WAAW,CAAC,GAAW;QAC7B,OAAO,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACvE,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,GAAW;QACnC,IAAI,CAAC;YACH,QAAQ,CAAC,sBAAsB,GAAG,EAAE,CAAC,CAAC;YAEtC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YACpC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE7B,MAAM,WAAW,GAAa,EAAE,CAAC;YAEjC,KAAK,MAAM,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC5B,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YACjD,CAAC;YAED,OAAO,WAAW,CAAC;QACrB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,QAAQ,CAAC,QAAS,CAAW,CAAC,OAAO,6BAA6B,CAAC,CAAC;YACpE,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,WAAW,EAAE,CAAC;oBACxC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAE,CAAC;gBAC/B,CAAC;YACH,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,QAAQ,CACN,QAAS,CAAW,CAAC,OAAO,qCAAqC,CAClE,CAAC;YACF,OAAO,GAAG,CAAC;QACb,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAY,EAAE,IAAY;QAClD,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACzD,CAAC;QACD,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,IAAY;QAEZ,IAAI,OAAY,CAAC;QACjB,IAAI,CAAC;YACH,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,QAAQ,CAAC,GAAG,IAAI,0BAA0B,CAAC,CAAC;YAC5C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACzC,QAAQ,CAAC,cAAc,OAAO,CAAC,QAAQ,8BAA8B,CAAC,CAAC;YACvE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;YAEtC,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,OAAO,QAAQ,CAAC,OAAO,CAAC;YAC1B,CAAC;YACD,QAAQ,CACN,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,oBAAoB,IAAI,EAAE,CACzE,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,QAAQ,CAAC,OAAQ,CAAW,CAAC,OAAO,oBAAoB,IAAI,EAAE,CAAC,CAAC;QAClE,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,cAAc,CAAC,OAAiC;QACtD,MAAM,WAAW,GACf,OAAO,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI,CAAC;QAC7D,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;YACzB,OAAO,WAAW,CAAC;QACrB,CAAC;QACD,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,WAAmB,EAAE,YAAoB,IAAI,CAAC,QAAQ;QACpE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;QAClD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnB,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC;QAC1B,QAAQ,CAAC,YAAY,WAAW,KAAK,IAAI,iBAAiB,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE3E,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,GAA8C,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAExE,OAAO,GAAG,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;YAC5B,QAAQ,CAAC,eAAe,CAAC,CAAC;YAC1B,QAAQ,CACN,YAAY,IAAI,CAAC,OAAO,CAAC,IAAI,gBAAgB,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CACnE,CAAC;YACF,QAAQ,CAAC,QAAQ,GAAG,WAAW,KAAK,IAAI,CAAC,CAAC;YAC1C,IAAI,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC3B,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAC7B,QAAQ,CAAC,YAAY,GAAG,KAAK,CAAC,CAAC;gBAC/B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;oBACnD,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAChC,QAAQ,CAAC,iBAAiB,QAAQ,KAAK,IAAI,GAAG,CAAC,CAAC;wBAChD,SAAS;oBACX,CAAC;oBACD,QAAQ,CACN,GAAG,QAAQ,oBAAoB,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CACrE,CAAC;oBACF,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,KAAK,EAAE,CAAC;wBAC1D,QAAQ,CACN,mBAAmB,QAAQ,aAAa,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CACrE,CAAC;wBACF,SAAS;oBACX,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAClC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;wBAC5C,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;4BAClB,SAAS;wBACX,CAAC;wBACD,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;wBAC9C,IACE,WAAW,KAAK,IAAI;4BACpB,CAAC,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,EAAE,CACxD,WAAW,CAAC,UAAU,CAAC,aAAa,CAAC,CACtC,EACD,CAAC;4BACD,QAAQ,CACN,WAAW,QAAQ,kBAAkB,WAAW,eAAe,CAChE,CAAC;4BACF,SAAS;wBACX,CAAC;wBACD,QAAQ,CAAC,UAAU,QAAQ,uBAAuB,CAAC,CAAC;wBACpD,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;oBACzC,CAAC;gBACH,CAAC;YACH,CAAC;YACD,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;QAED,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAChE,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAY;QACxB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC1B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,gBAAgB,CAAC,aAAoC;QACnD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACzC,CAAC;IAED,iBAAiB,CAAC,GAAG,cAAuC;QAC1D,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;;;OAOG;IACH,WAAW,CAAC,KAAa;QACvB,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QACpB,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QACpB,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,cAAc;QACZ,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,cAAc,EAAE;aACzB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;aACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;IACpC,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dancrumb/web-crawler",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.2.0",
|
|
4
4
|
"description": "",
|
|
5
5
|
"files": [
|
|
6
6
|
"dist"
|
|
7
7
|
],
|
|
8
8
|
"exports": {
|
|
9
|
-
".": "./dist/WebCrawler.js"
|
|
9
|
+
".": "./dist/WebCrawler.js",
|
|
10
|
+
"./CrawledDocument": "./dist/CrawledDocument.js"
|
|
10
11
|
},
|
|
11
12
|
"scripts": {
|
|
12
13
|
"build": "tsc",
|
|
@@ -38,4 +39,4 @@
|
|
|
38
39
|
"path": "./node_modules/git-cz"
|
|
39
40
|
}
|
|
40
41
|
}
|
|
41
|
-
}
|
|
42
|
+
}
|