@crazygiscool/cap 1.2.0 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/crawler/index.d.ts
CHANGED
|
@@ -5,6 +5,7 @@ export declare class Crawler<T extends MergeableDocument> {
|
|
|
5
5
|
private sourceConfig;
|
|
6
6
|
constructor(config: CrawlSourceConfig);
|
|
7
7
|
discover(categoryPath: string, maxPages?: number): Promise<string[]>;
|
|
8
|
+
discoverAll(categoryPath: string, maxPages?: number, maxDepth?: number): Promise<string[]>;
|
|
8
9
|
scrape(pageTitle: string): Promise<CrawledEntry<T> | null>;
|
|
9
|
-
crawlCategory(categoryPath: string, onProgress?: (progress: CrawlProgress) => void, maxPages?: number): Promise<CrawledEntry<T>[]>;
|
|
10
|
+
crawlCategory(categoryPath: string, onProgress?: (progress: CrawlProgress) => void, maxPages?: number, maxDepth?: number): Promise<CrawledEntry<T>[]>;
|
|
10
11
|
}
|
package/dist/crawler/index.js
CHANGED
|
@@ -72,10 +72,14 @@ export class Crawler {
|
|
|
72
72
|
async discover(categoryPath, maxPages = 500) {
|
|
73
73
|
return this.client.listCategoryMembers(categoryPath, maxPages);
|
|
74
74
|
}
|
|
75
|
+
async discoverAll(categoryPath, maxPages = 500, maxDepth = 2) {
|
|
76
|
+
return this.client.discoverAllPages(categoryPath, maxPages, maxDepth);
|
|
77
|
+
}
|
|
75
78
|
async scrape(pageTitle) {
|
|
76
79
|
const html = await this.client.fetchPageHtml(pageTitle);
|
|
77
80
|
const parsed = this.parserFn(html);
|
|
78
|
-
|
|
81
|
+
const name = parsed.title || pageTitle;
|
|
82
|
+
if (!name)
|
|
79
83
|
return null;
|
|
80
84
|
const mapped = mapFields(parsed, this.sourceConfig.fieldMapping, "");
|
|
81
85
|
const source = {
|
|
@@ -84,7 +88,7 @@ export class Crawler {
|
|
|
84
88
|
fetchedAt: new Date(),
|
|
85
89
|
};
|
|
86
90
|
return {
|
|
87
|
-
name
|
|
91
|
+
name,
|
|
88
92
|
description: mapped.description,
|
|
89
93
|
continuityId: mapped.continuityId,
|
|
90
94
|
factions: mapped.factions,
|
|
@@ -92,8 +96,8 @@ export class Crawler {
|
|
|
92
96
|
source,
|
|
93
97
|
};
|
|
94
98
|
}
|
|
95
|
-
async crawlCategory(categoryPath, onProgress, maxPages = 500) {
|
|
96
|
-
const titles = await this.
|
|
99
|
+
async crawlCategory(categoryPath, onProgress, maxPages = 500, maxDepth = 3) {
|
|
100
|
+
const titles = await this.discoverAll(categoryPath, maxPages, maxDepth);
|
|
97
101
|
const entries = [];
|
|
98
102
|
for (let i = 0; i < titles.length; i++) {
|
|
99
103
|
onProgress?.({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/crawler/index.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/C,SAAS,SAAS,CAChB,GAAW,EACX,IAA+B;IAE/B,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC5B,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;QACD,KAAK,SAAS;YACZ,OAAO,GAAG,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;QACrE;YACE,OAAO,GAAG,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAChB,MAAsB,EACtB,YAA+C,EAC/C,YAAoB;IAEpB,MAAM,MAAM,GAA6G;QACvH,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,YAAY;QACZ,QAAQ,EAAE,EAAE;QACZ,YAAY,EAAE,EAAE;KACjB,CAAC;IAEF,0CAA0C;IAC1C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,IACE,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YACtD,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,gBAAgB,CAAC;YAChE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;YACrD,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC;YACzD,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,oBAAoB,CAAC;YAC/D,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;YACzD,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YACrD,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC;YACtD,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,EACzB,CAAC;YACD,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACpD,MAAM;QACR,CAAC;IACH,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAClC,MAAM,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK;iBAC1B,KAAK,CAAC,UAAU,CAAC;iBACjB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;iBACpB,MAAM,CAAC,OAAO,CAAC,CAAC;QACrB,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;YAC5C,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;gBACnD,MAAM,CAAC,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC;YACnC,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;YAC7C,IAAI,CAAC,MAAM,CAAC,YAAY;gBAAE,MAAM,CAAC,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,OAAO,OAAO;IACV,MAAM,CAAkB;IACxB,QAAQ,CAAmC;IAC3C,YAAY,CAAoB;IAExC,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,QAAQ,GAAG,GAAG;QACjD,OAAO,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAiB;QAC5B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/crawler/index.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/C,SAAS,SAAS,CAChB,GAAW,EACX,IAA+B;IAE/B,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC5B,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;QACD,KAAK,SAAS;YACZ,OAAO,GAAG,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;QACrE;YACE,OAAO,GAAG,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAChB,MAAsB,EACtB,YAA+C,EAC/C,YAAoB;IAEpB,MAAM,MAAM,GAA6G;QACvH,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,YAAY;QACZ,QAAQ,EAAE,EAAE;QACZ,YAAY,EAAE,EAAE;KACjB,CAAC;IAEF,0CAA0C;IAC1C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,IACE,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YACtD,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,gBAAgB,CAAC;YAChE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;YACrD,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC;YACzD,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,oBAAoB,CAAC;YAC/D,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;YACzD,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YACrD,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC;YACtD,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,EACzB,CAAC;YACD,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACpD,MAAM;QACR,CAAC;IACH,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAClC,MAAM,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK;iBAC1B,KAAK,CAAC,UAAU,CAAC;iBACjB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;iBACpB,MAAM,CAAC,OAAO,CAAC,CAAC;QACrB,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;YAC5C,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;gBACnD,MAAM,CAAC,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC;YACnC,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;YAC7C,IAAI,CAAC,MAAM,CAAC,YAAY;gBAAE,MAAM,CAAC,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,OAAO,OAAO;IACV,MAAM,CAAkB;IACxB,QAAQ,CAAmC;IAC3C,YAAY,CAAoB;IAExC,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,QAAQ,GAAG,GAAG;QACjD,OAAO,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,WAAW,CACf,YAAoB,EACpB,QAAQ,GAAG,GAAG,EACd,QAAQ,GAAG,CAAC;QAEZ,OAAO,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,YAAY,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAiB;QAC5B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC;QACvC,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QAErE,MAAM,MAAM,GAAsB;YAChC,GAAG,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,SAAS,kBAAkB,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE;YAC5F,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE;YAC1B,SAAS,EAAE,IAAI,IAAI,EAAE;SACtB,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,MAAM;SACY,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,YAAoB,EACpB,UAA8C,EAC9C,QAAQ,GAAG,GAAG,EACd,QAAQ,GAAG,CAAC;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACxE,MAAM,OAAO,GAAsB,EAAE,CAAC;QAEtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,UAAU,EAAE,CAAC;gBACX,OAAO,EAAE,CAAC,GAAG,CAAC;gBACd,KAAK,EAAE,MAAM,CAAC,MAAM;gBACpB,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;aACrB,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,IAAI,KAAK;oBAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;YAAC,MAAM,CAAC;gBACP,gCAAgC;YAClC,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -4,8 +4,14 @@ interface MediaWikiConfig {
|
|
|
4
4
|
}
|
|
5
5
|
export declare class MediaWikiClient {
|
|
6
6
|
private config;
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
private delayMs;
|
|
8
|
+
constructor(config: MediaWikiConfig, delayMs?: number);
|
|
9
|
+
private delay;
|
|
10
|
+
private fetchJson;
|
|
11
|
+
private fetchAllCategoryMembers;
|
|
12
|
+
listCategoryMembers(categoryPath: string, maxPages?: number, signal?: AbortSignal): Promise<string[]>;
|
|
13
|
+
listSubCategories(categoryPath: string, maxPages?: number, signal?: AbortSignal): Promise<string[]>;
|
|
14
|
+
discoverAllPages(categoryPath: string, maxPages?: number, maxDepth?: number, signal?: AbortSignal, seen?: Set<string>): Promise<string[]>;
|
|
9
15
|
fetchPageHtml(pageTitle: string): Promise<string>;
|
|
10
16
|
searchPages(query: string, maxResults?: number): Promise<string[]>;
|
|
11
17
|
}
|
|
@@ -1,27 +1,83 @@
|
|
|
1
1
|
export class MediaWikiClient {
|
|
2
2
|
config;
|
|
3
|
-
|
|
3
|
+
delayMs;
|
|
4
|
+
constructor(config, delayMs = 50) {
|
|
4
5
|
this.config = config;
|
|
6
|
+
this.delayMs = delayMs;
|
|
5
7
|
}
|
|
6
|
-
async
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
8
|
+
async delay(ms = this.delayMs) {
|
|
9
|
+
if (ms > 0)
|
|
10
|
+
await new Promise((r) => setTimeout(r, ms));
|
|
11
|
+
}
|
|
12
|
+
async fetchJson(url, signal) {
|
|
13
|
+
const res = await fetch(url, {
|
|
14
|
+
headers: { "User-Agent": "CAP-Crawler/1.0 (crawler; contact@crazygiscool)" },
|
|
15
|
+
signal,
|
|
14
16
|
});
|
|
15
|
-
const url = `${this.config.apiUrl}?${params}`;
|
|
16
|
-
const res = await fetch(url);
|
|
17
17
|
if (!res.ok) {
|
|
18
18
|
throw new Error(`MediaWiki API error: ${res.status} ${res.statusText}`);
|
|
19
19
|
}
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
20
|
+
return res.json();
|
|
21
|
+
}
|
|
22
|
+
async fetchAllCategoryMembers(categoryPath, maxPages, cmtype, signal) {
|
|
23
|
+
const allTitles = [];
|
|
24
|
+
let cmcontinue;
|
|
25
|
+
while (allTitles.length < maxPages) {
|
|
26
|
+
const params = new URLSearchParams({
|
|
27
|
+
action: "query",
|
|
28
|
+
format: "json",
|
|
29
|
+
list: "categorymembers",
|
|
30
|
+
cmtitle: categoryPath,
|
|
31
|
+
cmlimit: String(Math.min(maxPages - allTitles.length, 500)),
|
|
32
|
+
cmtype,
|
|
33
|
+
});
|
|
34
|
+
if (cmcontinue)
|
|
35
|
+
params.set("cmcontinue", cmcontinue);
|
|
36
|
+
const url = `${this.config.apiUrl}?${params}`;
|
|
37
|
+
const data = (await this.fetchJson(url, signal));
|
|
38
|
+
if (data.error) {
|
|
39
|
+
throw new Error(`MediaWiki API: ${data.error.code} — ${data.error.info}`);
|
|
40
|
+
}
|
|
41
|
+
const members = data.query?.categorymembers ?? [];
|
|
42
|
+
allTitles.push(...members.map((m) => m.title));
|
|
43
|
+
if (data.continue?.cmcontinue) {
|
|
44
|
+
cmcontinue = data.continue.cmcontinue;
|
|
45
|
+
await this.delay();
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
break;
|
|
49
|
+
}
|
|
23
50
|
}
|
|
24
|
-
return
|
|
51
|
+
return allTitles.slice(0, maxPages);
|
|
52
|
+
}
|
|
53
|
+
async listCategoryMembers(categoryPath, maxPages = 500, signal) {
|
|
54
|
+
return this.fetchAllCategoryMembers(categoryPath, maxPages, "page", signal);
|
|
55
|
+
}
|
|
56
|
+
async listSubCategories(categoryPath, maxPages = 500, signal) {
|
|
57
|
+
return this.fetchAllCategoryMembers(categoryPath, maxPages, "subcat", signal);
|
|
58
|
+
}
|
|
59
|
+
async discoverAllPages(categoryPath, maxPages = 500, maxDepth = 2, signal, seen = new Set()) {
|
|
60
|
+
if (maxDepth < 0 || seen.has(categoryPath))
|
|
61
|
+
return [];
|
|
62
|
+
seen.add(categoryPath);
|
|
63
|
+
const pages = await this.listCategoryMembers(categoryPath, maxPages, signal);
|
|
64
|
+
const all = new Set(pages);
|
|
65
|
+
if (maxDepth > 0) {
|
|
66
|
+
await this.delay();
|
|
67
|
+
const subcategories = await this.listSubCategories(categoryPath, maxPages, signal);
|
|
68
|
+
for (const subcat of subcategories) {
|
|
69
|
+
if (all.size >= maxPages)
|
|
70
|
+
break;
|
|
71
|
+
await this.delay();
|
|
72
|
+
const subPages = await this.discoverAllPages(subcat, maxPages - all.size, maxDepth - 1, signal, seen);
|
|
73
|
+
for (const p of subPages) {
|
|
74
|
+
all.add(p);
|
|
75
|
+
if (all.size >= maxPages)
|
|
76
|
+
break;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return [...all].slice(0, maxPages);
|
|
25
81
|
}
|
|
26
82
|
async fetchPageHtml(pageTitle) {
|
|
27
83
|
const params = new URLSearchParams({
|
|
@@ -31,11 +87,7 @@ export class MediaWikiClient {
|
|
|
31
87
|
prop: "text",
|
|
32
88
|
});
|
|
33
89
|
const url = `${this.config.apiUrl}?${params}`;
|
|
34
|
-
const
|
|
35
|
-
if (!res.ok) {
|
|
36
|
-
throw new Error(`MediaWiki API error: ${res.status} for page "${pageTitle}"`);
|
|
37
|
-
}
|
|
38
|
-
const data = (await res.json());
|
|
90
|
+
const data = (await this.fetchJson(url));
|
|
39
91
|
if (data.error) {
|
|
40
92
|
throw new Error(`MediaWiki API: ${data.error.code} — ${data.error.info}`);
|
|
41
93
|
}
|
|
@@ -50,11 +102,7 @@ export class MediaWikiClient {
|
|
|
50
102
|
srlimit: String(Math.min(maxResults, 500)),
|
|
51
103
|
});
|
|
52
104
|
const url = `${this.config.apiUrl}?${params}`;
|
|
53
|
-
const
|
|
54
|
-
if (!res.ok) {
|
|
55
|
-
throw new Error(`MediaWiki API error: ${res.status} ${res.statusText}`);
|
|
56
|
-
}
|
|
57
|
-
const data = (await res.json());
|
|
105
|
+
const data = (await this.fetchJson(url));
|
|
58
106
|
if (data.error) {
|
|
59
107
|
throw new Error(`MediaWiki API: ${data.error.code} — ${data.error.info}`);
|
|
60
108
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mediawiki.js","sourceRoot":"","sources":["../../src/crawler/mediawiki.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"mediawiki.js","sourceRoot":"","sources":["../../src/crawler/mediawiki.ts"],"names":[],"mappings":"AA0BA,MAAM,OAAO,eAAe;IAClB,MAAM,CAAkB;IACxB,OAAO,CAAS;IAExB,YAAY,MAAuB,EAAE,OAAO,GAAG,EAAE;QAC/C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,OAAO;QACnC,IAAI,EAAE,GAAG,CAAC;YAAE,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,GAAW,EACX,MAAoB;QAEpB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,OAAO,EAAE,EAAE,YAAY,EAAE,iDAAiD,EAAE;YAC5E,MAAM;SACP,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,wBAAwB,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CACvD,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,CAAC,IAAI,EAAsC,CAAC;IACxD,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,YAAoB,EACpB,QAAgB,EAChB,MAAyB,EACzB,MAAoB;QAEpB,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,IAAI,UAA8B,CAAC;QAEnC,OAAO,SAAS,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;gBACjC,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,iBAAiB;gBACvB,OAAO,EAAE,YAAY;gBACrB,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC3D,MAAM;aACP,CAAC,CAAC;YACH,IAAI,UAAU;gBAAE,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;YAErD,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAmB,CAAC;YAEnE,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CACb,kBAAkB,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CACzD,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,EAAE,eAAe,IAAI,EAAE,CAAC;YAClD,SAAS,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAiB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YAE/D,IAAI,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC;gBAC9B,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACtC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,mBAAmB,CACvB,YAAoB,EACpB,QAAQ,GAAG,GAAG,EACd,MAAoB;QAEpB,OAAO,IAAI,CAAC,uBAAuB,CAAC,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,iBAAiB,CACrB,YAAoB,EACpB,QAAQ,GAAG,GAAG,EACd,MAAoB;QAEpB,OAAO,IAAI,CAAC,uBAAuB,CAAC,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,YAAoB,EACpB,QAAQ,GAAG,GAAG,EACd,QAAQ,GAAG,CAAC,EACZ,MAAoB,EACpB,OAAO,IAAI,GAAG,EAAU;QAExB,IAAI,QAAQ,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC;YAAE,OAAO,EAAE,CAAC;QACtD,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAEvB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,YAAY,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC7E,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QAE3B,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;YACnF,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;gBACnC,IAAI,GAAG,CAAC,IAAI,IAAI,QAAQ;oBAAE,MAAM;gBAChC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACnB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAC1C,MAAM,EACN,QAAQ,GAAG,GAAG,CAAC,IAAI,EACnB,QAAQ,GAAG,CAAC,EACZ,MAAM,EACN,IAAI,CACL,CAAC;gBACF,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;oBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;oBACX,IAAI,GAAG,CAAC,IAAI,IAAI,QAAQ;wBAAE,MAAM;gBAClC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,SAAiB;QACnC,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,MAAM;SACb,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAEtC,CAAC;QAEF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,kBAAkB,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CACzD,CAAC;QACJ,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,KAAa,EAAE,UAAU,GAAG,EAAE;QAC9C,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,KAAK;YACf,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;SAC3C,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAGtC,CAAC;QAEF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,kBAAkB,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5E,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAe,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACtE,CAAC;CACF"}
|