@crazygiscool/cap 1.2.1 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/crawler/index.js
CHANGED
|
@@ -72,13 +72,14 @@ export class Crawler {
|
|
|
72
72
|
async discover(categoryPath, maxPages = 500) {
|
|
73
73
|
return this.client.listCategoryMembers(categoryPath, maxPages);
|
|
74
74
|
}
|
|
75
|
-
async discoverAll(categoryPath, maxPages = 500, maxDepth =
|
|
75
|
+
async discoverAll(categoryPath, maxPages = 500, maxDepth = 2) {
|
|
76
76
|
return this.client.discoverAllPages(categoryPath, maxPages, maxDepth);
|
|
77
77
|
}
|
|
78
78
|
async scrape(pageTitle) {
|
|
79
79
|
const html = await this.client.fetchPageHtml(pageTitle);
|
|
80
80
|
const parsed = this.parserFn(html);
|
|
81
|
-
|
|
81
|
+
const name = parsed.title || pageTitle;
|
|
82
|
+
if (!name)
|
|
82
83
|
return null;
|
|
83
84
|
const mapped = mapFields(parsed, this.sourceConfig.fieldMapping, "");
|
|
84
85
|
const source = {
|
|
@@ -87,7 +88,7 @@ export class Crawler {
|
|
|
87
88
|
fetchedAt: new Date(),
|
|
88
89
|
};
|
|
89
90
|
return {
|
|
90
|
-
name
|
|
91
|
+
name,
|
|
91
92
|
description: mapped.description,
|
|
92
93
|
continuityId: mapped.continuityId,
|
|
93
94
|
factions: mapped.factions,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/crawler/index.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/C,SAAS,SAAS,CAChB,GAAW,EACX,IAA+B;IAE/B,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC5B,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;QACD,KAAK,SAAS;YACZ,OAAO,GAAG,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;QACrE;YACE,OAAO,GAAG,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAChB,MAAsB,EACtB,YAA+C,EAC/C,YAAoB;IAEpB,MAAM,MAAM,GAA6G;QACvH,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,YAAY;QACZ,QAAQ,EAAE,EAAE;QACZ,YAAY,EAAE,EAAE;KACjB,CAAC;IAEF,0CAA0C;IAC1C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,IACE,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YACtD,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,gBAAgB,CAAC;YAChE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;YACrD,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC;YACzD,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,oBAAoB,CAAC;YAC/D,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;YACzD,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YACrD,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC;YACtD,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,EACzB,CAAC;YACD,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACpD,MAAM;QACR,CAAC;IACH,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAClC,MAAM,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK;iBAC1B,KAAK,CAAC,UAAU,CAAC;iBACjB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;iBACpB,MAAM,CAAC,OAAO,CAAC,CAAC;QACrB,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;YAC5C,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;gBACnD,MAAM,CAAC,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC;YACnC,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;YAC7C,IAAI,CAAC,MAAM,CAAC,YAAY;gBAAE,MAAM,CAAC,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,OAAO,OAAO;IACV,MAAM,CAAkB;IACxB,QAAQ,CAAmC;IAC3C,YAAY,CAAoB;IAExC,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,QAAQ,GAAG,GAAG;QACjD,OAAO,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,WAAW,CACf,YAAoB,EACpB,QAAQ,GAAG,GAAG,EACd,QAAQ,GAAG,CAAC;QAEZ,OAAO,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,YAAY,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAiB;QAC5B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/crawler/index.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/C,SAAS,SAAS,CAChB,GAAW,EACX,IAA+B;IAE/B,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC5B,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;QACD,KAAK,SAAS;YACZ,OAAO,GAAG,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;QACrE;YACE,OAAO,GAAG,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAChB,MAAsB,EACtB,YAA+C,EAC/C,YAAoB;IAEpB,MAAM,MAAM,GAA6G;QACvH,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,YAAY;QACZ,QAAQ,EAAE,EAAE;QACZ,YAAY,EAAE,EAAE;KACjB,CAAC;IAEF,0CAA0C;IAC1C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,IACE,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YACtD,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,gBAAgB,CAAC;YAChE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;YACrD,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC;YACzD,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,oBAAoB,CAAC;YAC/D,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;YACzD,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YACrD,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC;YACtD,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,EACzB,CAAC;YACD,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACpD,MAAM;QACR,CAAC;IACH,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAClC,MAAM,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK;iBAC1B,KAAK,CAAC,UAAU,CAAC;iBACjB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;iBACpB,MAAM,CAAC,OAAO,CAAC,CAAC;QACrB,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;YAC5C,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;gBACnD,MAAM,CAAC,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC;YACnC,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;YAC7C,IAAI,CAAC,MAAM,CAAC,YAAY;gBAAE,MAAM,CAAC,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,OAAO,OAAO;IACV,MAAM,CAAkB;IACxB,QAAQ,CAAmC;IAC3C,YAAY,CAAoB;IAExC,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,QAAQ,GAAG,GAAG;QACjD,OAAO,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,WAAW,CACf,YAAoB,EACpB,QAAQ,GAAG,GAAG,EACd,QAAQ,GAAG,CAAC;QAEZ,OAAO,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,YAAY,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAiB;QAC5B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC;QACvC,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QAErE,MAAM,MAAM,GAAsB;YAChC,GAAG,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,SAAS,kBAAkB,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE;YAC5F,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE;YAC1B,SAAS,EAAE,IAAI,IAAI,EAAE;SACtB,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,MAAM;SACY,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,YAAoB,EACpB,UAA8C,EAC9C,QAAQ,GAAG,GAAG,EACd,QAAQ,GAAG,CAAC;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACxE,MAAM,OAAO,GAAsB,EAAE,CAAC;QAEtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,UAAU,EAAE,CAAC;gBACX,OAAO,EAAE,CAAC,GAAG,CAAC;gBACd,KAAK,EAAE,MAAM,CAAC,MAAM;gBACpB,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;aACrB,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,IAAI,KAAK;oBAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;YAAC,MAAM,CAAC;gBACP,gCAAgC;YAClC,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -9,9 +9,9 @@ export declare class MediaWikiClient {
|
|
|
9
9
|
private delay;
|
|
10
10
|
private fetchJson;
|
|
11
11
|
private fetchAllCategoryMembers;
|
|
12
|
-
listCategoryMembers(categoryPath: string, maxPages?: number): Promise<string[]>;
|
|
13
|
-
listSubCategories(categoryPath: string, maxPages?: number): Promise<string[]>;
|
|
14
|
-
discoverAllPages(categoryPath: string, maxPages?: number, maxDepth?: number, seen?: Set<string>): Promise<string[]>;
|
|
12
|
+
listCategoryMembers(categoryPath: string, maxPages?: number, signal?: AbortSignal): Promise<string[]>;
|
|
13
|
+
listSubCategories(categoryPath: string, maxPages?: number, signal?: AbortSignal): Promise<string[]>;
|
|
14
|
+
discoverAllPages(categoryPath: string, maxPages?: number, maxDepth?: number, signal?: AbortSignal, seen?: Set<string>): Promise<string[]>;
|
|
15
15
|
fetchPageHtml(pageTitle: string): Promise<string>;
|
|
16
16
|
searchPages(query: string, maxResults?: number): Promise<string[]>;
|
|
17
17
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
export class MediaWikiClient {
|
|
2
2
|
config;
|
|
3
3
|
delayMs;
|
|
4
|
-
constructor(config, delayMs =
|
|
4
|
+
constructor(config, delayMs = 50) {
|
|
5
5
|
this.config = config;
|
|
6
6
|
this.delayMs = delayMs;
|
|
7
7
|
}
|
|
@@ -9,16 +9,17 @@ export class MediaWikiClient {
|
|
|
9
9
|
if (ms > 0)
|
|
10
10
|
await new Promise((r) => setTimeout(r, ms));
|
|
11
11
|
}
|
|
12
|
-
async fetchJson(url) {
|
|
12
|
+
async fetchJson(url, signal) {
|
|
13
13
|
const res = await fetch(url, {
|
|
14
14
|
headers: { "User-Agent": "CAP-Crawler/1.0 (crawler; contact@crazygiscool)" },
|
|
15
|
+
signal,
|
|
15
16
|
});
|
|
16
17
|
if (!res.ok) {
|
|
17
18
|
throw new Error(`MediaWiki API error: ${res.status} ${res.statusText}`);
|
|
18
19
|
}
|
|
19
20
|
return res.json();
|
|
20
21
|
}
|
|
21
|
-
async fetchAllCategoryMembers(categoryPath, maxPages, cmtype) {
|
|
22
|
+
async fetchAllCategoryMembers(categoryPath, maxPages, cmtype, signal) {
|
|
22
23
|
const allTitles = [];
|
|
23
24
|
let cmcontinue;
|
|
24
25
|
while (allTitles.length < maxPages) {
|
|
@@ -33,7 +34,7 @@ export class MediaWikiClient {
|
|
|
33
34
|
if (cmcontinue)
|
|
34
35
|
params.set("cmcontinue", cmcontinue);
|
|
35
36
|
const url = `${this.config.apiUrl}?${params}`;
|
|
36
|
-
const data = (await this.fetchJson(url));
|
|
37
|
+
const data = (await this.fetchJson(url, signal));
|
|
37
38
|
if (data.error) {
|
|
38
39
|
throw new Error(`MediaWiki API: ${data.error.code} — ${data.error.info}`);
|
|
39
40
|
}
|
|
@@ -49,26 +50,26 @@ export class MediaWikiClient {
|
|
|
49
50
|
}
|
|
50
51
|
return allTitles.slice(0, maxPages);
|
|
51
52
|
}
|
|
52
|
-
async listCategoryMembers(categoryPath, maxPages = 500) {
|
|
53
|
-
return this.fetchAllCategoryMembers(categoryPath, maxPages, "page");
|
|
53
|
+
async listCategoryMembers(categoryPath, maxPages = 500, signal) {
|
|
54
|
+
return this.fetchAllCategoryMembers(categoryPath, maxPages, "page", signal);
|
|
54
55
|
}
|
|
55
|
-
async listSubCategories(categoryPath, maxPages = 500) {
|
|
56
|
-
return this.fetchAllCategoryMembers(categoryPath, maxPages, "subcat");
|
|
56
|
+
async listSubCategories(categoryPath, maxPages = 500, signal) {
|
|
57
|
+
return this.fetchAllCategoryMembers(categoryPath, maxPages, "subcat", signal);
|
|
57
58
|
}
|
|
58
|
-
async discoverAllPages(categoryPath, maxPages = 500, maxDepth =
|
|
59
|
+
async discoverAllPages(categoryPath, maxPages = 500, maxDepth = 2, signal, seen = new Set()) {
|
|
59
60
|
if (maxDepth < 0 || seen.has(categoryPath))
|
|
60
61
|
return [];
|
|
61
62
|
seen.add(categoryPath);
|
|
62
|
-
const pages = await this.listCategoryMembers(categoryPath, maxPages);
|
|
63
|
+
const pages = await this.listCategoryMembers(categoryPath, maxPages, signal);
|
|
63
64
|
const all = new Set(pages);
|
|
64
65
|
if (maxDepth > 0) {
|
|
65
66
|
await this.delay();
|
|
66
|
-
const subcategories = await this.listSubCategories(categoryPath, maxPages);
|
|
67
|
+
const subcategories = await this.listSubCategories(categoryPath, maxPages, signal);
|
|
67
68
|
for (const subcat of subcategories) {
|
|
68
69
|
if (all.size >= maxPages)
|
|
69
70
|
break;
|
|
70
71
|
await this.delay();
|
|
71
|
-
const subPages = await this.discoverAllPages(subcat, maxPages - all.size, maxDepth - 1, seen);
|
|
72
|
+
const subPages = await this.discoverAllPages(subcat, maxPages - all.size, maxDepth - 1, signal, seen);
|
|
72
73
|
for (const p of subPages) {
|
|
73
74
|
all.add(p);
|
|
74
75
|
if (all.size >= maxPages)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mediawiki.js","sourceRoot":"","sources":["../../src/crawler/mediawiki.ts"],"names":[],"mappings":"AA0BA,MAAM,OAAO,eAAe;IAClB,MAAM,CAAkB;IACxB,OAAO,CAAS;IAExB,YAAY,MAAuB,EAAE,OAAO,GAAG,
|
|
1
|
+
{"version":3,"file":"mediawiki.js","sourceRoot":"","sources":["../../src/crawler/mediawiki.ts"],"names":[],"mappings":"AA0BA,MAAM,OAAO,eAAe;IAClB,MAAM,CAAkB;IACxB,OAAO,CAAS;IAExB,YAAY,MAAuB,EAAE,OAAO,GAAG,EAAE;QAC/C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,OAAO;QACnC,IAAI,EAAE,GAAG,CAAC;YAAE,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,GAAW,EACX,MAAoB;QAEpB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,OAAO,EAAE,EAAE,YAAY,EAAE,iDAAiD,EAAE;YAC5E,MAAM;SACP,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,wBAAwB,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CACvD,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,CAAC,IAAI,EAAsC,CAAC;IACxD,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,YAAoB,EACpB,QAAgB,EAChB,MAAyB,EACzB,MAAoB;QAEpB,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,IAAI,UAA8B,CAAC;QAEnC,OAAO,SAAS,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;gBACjC,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,iBAAiB;gBACvB,OAAO,EAAE,YAAY;gBACrB,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC3D,MAAM;aACP,CAAC,CAAC;YACH,IAAI,UAAU;gBAAE,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;YAErD,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAmB,CAAC;YAEnE,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CACb,kBAAkB,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CACzD,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,EAAE,eAAe,IAAI,EAAE,CAAC;YAClD,SAAS,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAiB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YAE/D,IAAI,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC;gBAC9B,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACtC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,mBAAmB,CACvB,YAAoB,EACpB,QAAQ,GAAG,GAAG,EACd,MAAoB;QAEpB,OAAO,IAAI,CAAC,uBAAuB,CAAC,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,iBAAiB,CACrB,YAAoB,EACpB,QAAQ,GAAG,GAAG,EACd,MAAoB;QAEpB,OAAO,IAAI,CAAC,uBAAuB,CAAC,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,YAAoB,EACpB,QAAQ,GAAG,GAAG,EACd,QAAQ,GAAG,CAAC,EACZ,MAAoB,EACpB,OAAO,IAAI,GAAG,EAAU;QAExB,IAAI,QAAQ,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC;YAAE,OAAO,EAAE,CAAC;QACtD,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAEvB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,YAAY,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC7E,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QAE3B,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;YACnF,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;gBACnC,IAAI,GAAG,CAAC,IAAI,IAAI,QAAQ;oBAAE,MAAM;gBAChC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACnB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAC1C,MAAM,EACN,QAAQ,GAAG,GAAG,CAAC,IAAI,EACnB,QAAQ,GAAG,CAAC,EACZ,MAAM,EACN,IAAI,CACL,CAAC;gBACF,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;oBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;oBACX,IAAI,GAAG,CAAC,IAAI,IAAI,QAAQ;wBAAE,MAAM;gBAClC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,SAAiB;QACnC,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,MAAM;SACb,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAEtC,CAAC;QAEF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,kBAAkB,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CACzD,CAAC;QACJ,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,KAAa,EAAE,UAAU,GAAG,EAAE;QAC9C,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,KAAK;YACf,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;SAC3C,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAGtC,CAAC;QAEF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,kBAAkB,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5E,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAe,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACtE,CAAC;CACF"}
|