@dancrumb/web-crawler 1.4.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/WebCrawler.d.ts +4 -0
- package/dist/WebCrawler.d.ts.map +1 -1
- package/dist/WebCrawler.js +16 -1
- package/dist/WebCrawler.js.map +1 -1
- package/package.json +4 -3
package/dist/WebCrawler.d.ts
CHANGED
|
@@ -13,7 +13,9 @@ export declare class WebCrawler extends EventEmitter<{
|
|
|
13
13
|
private uncrawled;
|
|
14
14
|
private documents;
|
|
15
15
|
private roots;
|
|
16
|
+
private supportedTypes;
|
|
16
17
|
private maxDepth;
|
|
18
|
+
constructor();
|
|
17
19
|
private reportProgress;
|
|
18
20
|
private addUncrawled;
|
|
19
21
|
private markCrawled;
|
|
@@ -28,6 +30,8 @@ export declare class WebCrawler extends EventEmitter<{
|
|
|
28
30
|
crawlLink(initialLink: string, maxDepth?: number): Promise<string[][]>;
|
|
29
31
|
clear(): void;
|
|
30
32
|
addRoot(root: string): Promise<string>;
|
|
33
|
+
addSupportedType(supportedType: `${string}/${string}`): void;
|
|
34
|
+
setSupportedTypes(...supportedTypes: `${string}/${string}`[]): void;
|
|
31
35
|
/**
|
|
32
36
|
* Sets the maximum crawl depth.
|
|
33
37
|
*
|
package/dist/WebCrawler.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WebCrawler.d.ts","sourceRoot":"","sources":["../src/WebCrawler.ts"],"names":[],"mappings":"AAIA,OAAO,YAAY,MAAM,aAAa,CAAC;AAMvC,KAAK,QAAQ,GAAG;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB,CAAC;AAEF,qBAAa,UAAW,SAAQ,YAAY,CAAC;IAC3C,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;CACrC,CAAC;IACA,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,SAAS,CAA6B;IAC9C,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,QAAQ,CAAa
|
|
1
|
+
{"version":3,"file":"WebCrawler.d.ts","sourceRoot":"","sources":["../src/WebCrawler.ts"],"names":[],"mappings":"AAIA,OAAO,YAAY,MAAM,aAAa,CAAC;AAMvC,KAAK,QAAQ,GAAG;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB,CAAC;AAEF,qBAAa,UAAW,SAAQ,YAAY,CAAC;IAC3C,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;CACrC,CAAC;IACA,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,SAAS,CAA6B;IAC9C,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,cAAc,CAAoC;IAC1D,OAAO,CAAC,QAAQ,CAAa;;IAO7B,OAAO,CAAC,cAAc;IAWtB,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,WAAW;YAUL,YAAY;YA0BZ,WAAW;IAOzB,OAAO,CAAC,WAAW;YAIL,WAAW;YAoBX,YAAY;YAkBZ,WAAW;YAUX,SAAS;IAiCvB,OAAO,CAAC,cAAc;IAYhB,SAAS,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAsB;IA6DrE,KAAK;IASC,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAM5C,gBAAgB,CAAC,aAAa,EAAE,GAAG,MAAM,IAAI,MAAM,EAAE;IAIrD,iBAAiB,CAAC,GAAG,cAAc,EAAE,GAAG,MAAM,IAAI,MAAM,EAAE,EAAE;IAK5D;;;;;;;OAOG;IACH,WAAW,CAAC,KAAK,EAAE,MAAM;IAWzB,cAAc;IAId,YAAY,IAAI,QAAQ,EAAE;CAK3B"}
|
package/dist/WebCrawler.js
CHANGED
|
@@ -10,7 +10,12 @@ export class WebCrawler extends EventEmitter {
|
|
|
10
10
|
uncrawled = new Map();
|
|
11
11
|
documents = new Map();
|
|
12
12
|
roots = new Set();
|
|
13
|
+
supportedTypes = new Set();
|
|
13
14
|
maxDepth = 0;
|
|
15
|
+
constructor() {
|
|
16
|
+
super();
|
|
17
|
+
this.clear();
|
|
18
|
+
}
|
|
14
19
|
reportProgress() {
|
|
15
20
|
if (this.uncrawled.size === 0) {
|
|
16
21
|
this.emit("progress", null);
|
|
@@ -174,7 +179,8 @@ export class WebCrawler extends EventEmitter {
|
|
|
174
179
|
continue;
|
|
175
180
|
}
|
|
176
181
|
const contentType = this.getContentType(head);
|
|
177
|
-
if (contentType === null ||
|
|
182
|
+
if (contentType === null ||
|
|
183
|
+
![...this.supportedTypes.values()].some((supportedType) => contentType.startsWith(supportedType))) {
|
|
178
184
|
logDebug(`Page at ${resolved} has MIME type ${contentType}, so skipping`);
|
|
179
185
|
continue;
|
|
180
186
|
}
|
|
@@ -192,12 +198,21 @@ export class WebCrawler extends EventEmitter {
|
|
|
192
198
|
this.uncrawled.clear();
|
|
193
199
|
this.documents.clear();
|
|
194
200
|
this.roots.clear();
|
|
201
|
+
this.supportedTypes.clear();
|
|
202
|
+
this.supportedTypes.add("text/html");
|
|
195
203
|
}
|
|
196
204
|
async addRoot(root) {
|
|
197
205
|
const canonical = await this.getCanonical(root);
|
|
198
206
|
this.roots.add(canonical);
|
|
199
207
|
return canonical;
|
|
200
208
|
}
|
|
209
|
+
addSupportedType(supportedType) {
|
|
210
|
+
this.supportedTypes.add(supportedType);
|
|
211
|
+
}
|
|
212
|
+
setSupportedTypes(...supportedTypes) {
|
|
213
|
+
this.supportedTypes.clear();
|
|
214
|
+
supportedTypes.forEach((s) => this.supportedTypes.add(s));
|
|
215
|
+
}
|
|
201
216
|
/**
|
|
202
217
|
* Sets the maximum crawl depth.
|
|
203
218
|
*
|
package/dist/WebCrawler.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WebCrawler.js","sourceRoot":"","sources":["../src/WebCrawler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,YAAY,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC;AAUvC,MAAM,OAAO,UAAW,SAAQ,YAE9B;IACQ,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpC,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAC;IACxC,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAC1B,QAAQ,GAAW,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"WebCrawler.js","sourceRoot":"","sources":["../src/WebCrawler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,YAAY,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC;AAUvC,MAAM,OAAO,UAAW,SAAQ,YAE9B;IACQ,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpC,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAC;IACxC,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAC1B,cAAc,GAAG,IAAI,GAAG,EAAyB,CAAC;IAClD,QAAQ,GAAW,CAAC,CAAC;IAE7B;QACE,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAEO,cAAc;QACpB,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAC9B,CAAC;QACD,MAAM,QAAQ,GAAG,CACf,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC/D,GAAG,CACJ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACb,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAEO,YAAY,CAAC,GAAW,EAAE,KAAa;QAC7C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC/B,IAAI,CAAC,cAAc,EAAE,CAAC;IACxB,CAAC;IAEO,WAAW,CAAC,GAAW,EAAE,QAAiB;QAChD,IAAI,KAAK,GAAG,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,cAAc,EAAE,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,WAAW,GACf,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,0BAA0B,CAAC;QAC7D,MAAM,OAAO,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;YACtB,GAAG;YACH,YAAY,EAAE,EAAE;YAChB,OAAO;YACP,KAAK,EAAE,IAAI;YACX,WAAW;SACZ,CAAC,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,YAAY,KAAK,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,EAAE;gBAC/B,GAAG;gBACH,YAAY;gBACZ,OAAO;gBACP,KAAK,EAAE,IAAI;gBACX,WAAW;aACZ,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,GAAW;QACnC,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,OAAO,CAAC;QAC1C,CAAC;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAEO,WAAW,CAAC,GAAW;QAC7B,OAAO,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACvE,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,GAAW;QACnC,IAAI,CAAC;YACH,QAAQ,CAAC,sBAAsB,GAAG,EAAE,CAAC,CAAC;YAEtC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YACpC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE7B,MAAM,WAAW,GAAa,EAAE,CAAC;YAEjC,KAAK,MAAM,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC5B,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YACjD,CAAC;YAED,OAAO,WAAW,CAAC;QACrB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,QAAQ,CAAC,QAAS,CAAW,CAAC,OAAO,6BAA6B,CAAC,CAAC;YACpE,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,WAAW,EAAE,CAAC;oBACxC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAE,CAAC;gBAC/B,CAAC;YACH,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,QAAQ,CACN,QAAS,CAAW,CAAC,OAAO,qCAAqC,CAClE,CAAC;YACF,OAAO,GAAG,CAAC;QACb,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAY,EAAE,IAAY;QAClD,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACzD,CAAC;QACD,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,IAAY;QAEZ,IAAI,OAAY,CAAC;QACjB,IAAI,CAAC;YACH,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,QAAQ,CAAC,GAAG,IAAI,0BAA0B,CAAC,CAAC;YAC5C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACzC,QAAQ,CAAC,cAAc,OAAO,CAAC,QAAQ,8BAA8B,CAAC,CAAC;YACvE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;YAEtC,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,OAAO,QAAQ,CAAC,OAAO,CAAC;YAC1B,CAAC;YACD,QAAQ,CACN,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,oBAAoB,IAAI,EAAE,CACzE,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,QAAQ,CAAC,OAAQ,CAAW,CAAC,OAAO,oBAAoB,IAAI,EAAE,CAAC,CAAC;QAClE,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,cAAc,CAAC,OAAiC;QACtD,MAAM,WAAW,GACf,OAAO,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI,CAAC;QAC7D,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;YACzB,OAAO,WAAW,CAAC;QACrB,CAAC;QACD,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,WAAmB,EAAE,WAAmB,IAAI,CAAC,QAAQ;QACnE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;QAClD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEzB,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,GAA8C,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAExE,OAAO,GAAG,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;YAC5B,QAAQ,CAAC,eAAe,CAAC,CAAC;YAC1B,QAAQ,CACN,YAAY,IAAI,CAAC,OAAO,CAAC,IAAI,gBAAgB,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CACnE,CAAC;YACF,QAAQ,CAAC,QAAQ,GAAG,WAAW,KAAK,IAAI,CAAC,CAAC;YAC1C,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACtB,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAC7B,QAAQ,CAAC,YAAY,GAAG,KAAK,CAAC,CAAC;gBAC/B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;oBACnD,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAChC,QAAQ,CAAC,iBAAiB,QAAQ,KAAK,IAAI,GAAG,CAAC,CAAC;wBAChD,SAAS;oBACX,CAAC;oBACD,QAAQ,CACN,GAAG,QAAQ,oBAAoB,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CACrE,CAAC;oBACF,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,GAAG,KAAK,EAAE,CAAC;wBACrD,QAAQ,CACN,mBAAmB,QAAQ,aAAa,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CACrE,CAAC;wBACF,SAAS;oBACX,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAClC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;wBAC5C,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;4BAClB,SAAS;wBACX,CAAC;wBACD,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;wBAC9C,IACE,WAAW,KAAK,IAAI;4BACpB,CAAC,CAAC,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,EAAE,CACxD,WAAW,CAAC,UAAU,CAAC,aAAa,CAAC,CACtC,EACD,CAAC;4BACD,QAAQ,CACN,WAAW,QAAQ,kBAAkB,WAAW,eAAe,CAChE,CAAC;4BACF,SAAS;wBACX,CAAC;wBACD,QAAQ,CAAC,UAAU,QAAQ,uBAAuB,CAAC,CAAC;wBACpD,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;oBACzC,CAAC;gBACH,CAAC;YACH,CAAC;YACD,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;QAED,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAChE,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAY;QACxB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC1B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,gBAAgB,CAAC,aAAoC;QACnD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACzC,CAAC;IAED,iBAAiB,CAAC,GAAG,cAAuC;QAC1D,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;;;OAOG;IACH,WAAW,CAAC,KAAa;QACvB,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QACpB,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QACpB,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,cAAc;QACZ,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,cAAc,EAAE;aACzB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;aACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;IACpC,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dancrumb/web-crawler",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "3.0.0",
|
|
4
4
|
"description": "",
|
|
5
5
|
"files": [
|
|
6
6
|
"dist"
|
|
@@ -10,7 +10,8 @@
|
|
|
10
10
|
},
|
|
11
11
|
"scripts": {
|
|
12
12
|
"build": "tsc",
|
|
13
|
-
"test": "vitest --run"
|
|
13
|
+
"test": "vitest --run",
|
|
14
|
+
"prepare": "rm -rf dist && npm run build"
|
|
14
15
|
},
|
|
15
16
|
"keywords": [],
|
|
16
17
|
"author": "Dan Rumney<dancrumb@gmail.com>",
|
|
@@ -37,4 +38,4 @@
|
|
|
37
38
|
"path": "./node_modules/git-cz"
|
|
38
39
|
}
|
|
39
40
|
}
|
|
40
|
-
}
|
|
41
|
+
}
|