@crazygiscool/cap 1.1.10 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/crawler.test.d.ts +1 -0
- package/dist/__tests__/crawler.test.js +175 -0
- package/dist/__tests__/crawler.test.js.map +1 -0
- package/dist/__tests__/merger.test.d.ts +1 -0
- package/dist/__tests__/merger.test.js +136 -0
- package/dist/__tests__/merger.test.js.map +1 -0
- package/dist/crawler/index.d.ts +11 -0
- package/dist/crawler/index.js +119 -0
- package/dist/crawler/index.js.map +1 -0
- package/dist/crawler/mediawiki.d.ts +18 -0
- package/dist/crawler/mediawiki.js +111 -0
- package/dist/crawler/mediawiki.js.map +1 -0
- package/dist/crawler/merger.d.ts +3 -0
- package/dist/crawler/merger.js +128 -0
- package/dist/crawler/merger.js.map +1 -0
- package/dist/crawler/parser.d.ts +8 -0
- package/dist/crawler/parser.js +95 -0
- package/dist/crawler/parser.js.map +1 -0
- package/dist/crawler/parsers/fandom.d.ts +2 -0
- package/dist/crawler/parsers/fandom.js +5 -0
- package/dist/crawler/parsers/fandom.js.map +1 -0
- package/dist/crawler/parsers/index.d.ts +5 -0
- package/dist/crawler/parsers/index.js +17 -0
- package/dist/crawler/parsers/index.js.map +1 -0
- package/dist/crawler/parsers/tfwiki.d.ts +2 -0
- package/dist/crawler/parsers/tfwiki.js +5 -0
- package/dist/crawler/parsers/tfwiki.js.map +1 -0
- package/dist/crawler/parsers/wikipedia.d.ts +2 -0
- package/dist/crawler/parsers/wikipedia.js +5 -0
- package/dist/crawler/parsers/wikipedia.js.map +1 -0
- package/dist/crawler/types.d.ts +61 -0
- package/dist/crawler/types.js +2 -0
- package/dist/crawler/types.js.map +1 -0
- package/dist/index.d.ts +150 -1
- package/dist/index.js +26 -0
- package/dist/index.js.map +1 -1
- package/package.json +8 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { parseTFWiki } from "../crawler/parsers/tfwiki.js";
|
|
3
|
+
import { parseWikipedia } from "../crawler/parsers/wikipedia.js";
|
|
4
|
+
import { parseFandom } from "../crawler/parsers/fandom.js";
|
|
5
|
+
import { extractFirstParagraph, extractCategories, extractPageTitle } from "../crawler/parser.js";
|
|
6
|
+
import * as cheerio from "cheerio";
|
|
7
|
+
const TFWIKI_HTML = `<!DOCTYPE html>
|
|
8
|
+
<html>
|
|
9
|
+
<head><title>Optimus Prime (G1) — TFWiki</title></head>
|
|
10
|
+
<body>
|
|
11
|
+
<div id="mw-content-text">
|
|
12
|
+
<h1 id="firstHeading">Optimus Prime (G1)</h1>
|
|
13
|
+
<table class="infobox">
|
|
14
|
+
<tr><th colspan="2">Optimus Prime</th></tr>
|
|
15
|
+
<tr><th>Affiliation</th><td>Autobot</td></tr>
|
|
16
|
+
<tr><th>Function</th><td>Leader of the Autobots</td></tr>
|
|
17
|
+
<tr><th>Motto</th><td>"Freedom is the right of all sentient beings"</td></tr>
|
|
18
|
+
<tr><th>Alt Mode</th><td>Freightliner FLT Flat-Nose Semi-Truck</td></tr>
|
|
19
|
+
<tr><th>Color Scheme</th><td>Red, Blue, Silver</td></tr>
|
|
20
|
+
<tr><th>Firepower</th><td>10</td></tr>
|
|
21
|
+
<tr><th>Strength</th><td>10</td></tr>
|
|
22
|
+
<tr><th>Intelligence</th><td>10</td></tr>
|
|
23
|
+
<tr><th>Speed</th><td>8</td></tr>
|
|
24
|
+
</table>
|
|
25
|
+
<p>Optimus Prime is the heroic leader of the Autobots. He is the reincarnation of the original Optimus Prime.</p>
|
|
26
|
+
</div>
|
|
27
|
+
<div id="mw-normal-catlinks">
|
|
28
|
+
<ul>
|
|
29
|
+
<li><a>Autobots</a></li>
|
|
30
|
+
<li><a>Generation 1</a></li>
|
|
31
|
+
<li><a>Primes</a></li>
|
|
32
|
+
</ul>
|
|
33
|
+
</div>
|
|
34
|
+
</body>
|
|
35
|
+
</html>`;
|
|
36
|
+
const WIKIPEDIA_HTML = `<!DOCTYPE html>
|
|
37
|
+
<html>
|
|
38
|
+
<head><title>Optimus Prime - Wikipedia</title></head>
|
|
39
|
+
<body>
|
|
40
|
+
<div id="mw-content-text">
|
|
41
|
+
<h1 id="firstHeading">Optimus Prime</h1>
|
|
42
|
+
<table class="infobox">
|
|
43
|
+
<tr><th colspan="2">Optimus Prime</th></tr>
|
|
44
|
+
<tr><th scope="row">Affiliation</th><td>Autobot</td></tr>
|
|
45
|
+
<tr><th scope="row">Function</th><td>Leader</td></tr>
|
|
46
|
+
<tr><th scope="row">Motto</th><td>"Freedom is the right of all sentient beings"</td></tr>
|
|
47
|
+
<tr><th scope="row">Series</th><td>Generation 1</td></tr>
|
|
48
|
+
</table>
|
|
49
|
+
<p>Optimus Prime is a fictional character and the main protagonist of the Transformers franchise.</p>
|
|
50
|
+
</div>
|
|
51
|
+
<div id="mw-normal-catlinks">
|
|
52
|
+
<ul>
|
|
53
|
+
<li><a>Autobots</a></li>
|
|
54
|
+
<li><a>Fictional characters</a></li>
|
|
55
|
+
</ul>
|
|
56
|
+
</div>
|
|
57
|
+
</body>
|
|
58
|
+
</html>`;
|
|
59
|
+
const FANDOM_HTML = `<!DOCTYPE html>
|
|
60
|
+
<html>
|
|
61
|
+
<head><title>Optimus Prime | Transformers Wiki | Fandom</title></head>
|
|
62
|
+
<body>
|
|
63
|
+
<div id="mw-content-text">
|
|
64
|
+
<h1 id="firstHeading">Optimus Prime</h1>
|
|
65
|
+
<aside class="portable-infobox">
|
|
66
|
+
<div class="pi-item pi-data">
|
|
67
|
+
<h3>Affiliation</h3>
|
|
68
|
+
<div>Autobot</div>
|
|
69
|
+
</div>
|
|
70
|
+
<div class="pi-item pi-data">
|
|
71
|
+
<h3>Function</h3>
|
|
72
|
+
<div>Supreme Commander</div>
|
|
73
|
+
</div>
|
|
74
|
+
<div class="pi-item pi-data">
|
|
75
|
+
<h3>Motto</h3>
|
|
76
|
+
<div>"Freedom is the right of all sentient beings"</div>
|
|
77
|
+
</div>
|
|
78
|
+
</aside>
|
|
79
|
+
<p>Optimus Prime is the noble leader of the Autobots in the Transformers franchise.</p>
|
|
80
|
+
</div>
|
|
81
|
+
<div id="mw-normal-catlinks">
|
|
82
|
+
<ul>
|
|
83
|
+
<li><a>Autobots</a></li>
|
|
84
|
+
<li><a>Leaders</a></li>
|
|
85
|
+
</ul>
|
|
86
|
+
</div>
|
|
87
|
+
</body>
|
|
88
|
+
</html>`;
|
|
89
|
+
const NO_INFOBOX_HTML = `<!DOCTYPE html>
|
|
90
|
+
<html>
|
|
91
|
+
<head><title>Some random page</title></head>
|
|
92
|
+
<body>
|
|
93
|
+
<div id="mw-content-text">
|
|
94
|
+
<h1 id="firstHeading">Some Random Page</h1>
|
|
95
|
+
<p>This is a page with no infobox at all.</p>
|
|
96
|
+
</div>
|
|
97
|
+
</body>
|
|
98
|
+
</html>`;
|
|
99
|
+
describe("parseTFWiki", () => {
|
|
100
|
+
it("extracts infobox fields from TFWiki HTML", () => {
|
|
101
|
+
const result = parseTFWiki(TFWIKI_HTML);
|
|
102
|
+
expect(result.title).toBe("Optimus Prime (G1)");
|
|
103
|
+
const labels = result.infoboxFields.map((f) => f.label);
|
|
104
|
+
expect(labels).toContain("Affiliation");
|
|
105
|
+
expect(labels).toContain("Function");
|
|
106
|
+
expect(labels).toContain("Motto");
|
|
107
|
+
expect(labels).toContain("Firepower");
|
|
108
|
+
const aff = result.infoboxFields.find((f) => f.label === "Affiliation");
|
|
109
|
+
expect(aff?.value).toBe("Autobot");
|
|
110
|
+
const fp = result.infoboxFields.find((f) => f.label === "Firepower");
|
|
111
|
+
expect(fp?.value).toBe("10");
|
|
112
|
+
});
|
|
113
|
+
it("extracts description from first paragraph", () => {
|
|
114
|
+
const result = parseTFWiki(TFWIKI_HTML);
|
|
115
|
+
expect(result.description).toContain("heroic leader");
|
|
116
|
+
});
|
|
117
|
+
it("extracts categories", () => {
|
|
118
|
+
const result = parseTFWiki(TFWIKI_HTML);
|
|
119
|
+
expect(result.categories).toContain("Autobots");
|
|
120
|
+
expect(result.categories).toContain("Generation 1");
|
|
121
|
+
});
|
|
122
|
+
it("returns empty data for pages with no infobox", () => {
|
|
123
|
+
const result = parseTFWiki(NO_INFOBOX_HTML);
|
|
124
|
+
expect(result.infoboxFields).toHaveLength(0);
|
|
125
|
+
expect(result.title).toBe("Some Random Page");
|
|
126
|
+
});
|
|
127
|
+
});
|
|
128
|
+
describe("parseWikipedia", () => {
|
|
129
|
+
it("extracts infobox fields from Wikipedia HTML", () => {
|
|
130
|
+
const result = parseWikipedia(WIKIPEDIA_HTML);
|
|
131
|
+
expect(result.title).toBe("Optimus Prime");
|
|
132
|
+
const aff = result.infoboxFields.find((f) => f.label === "Affiliation");
|
|
133
|
+
expect(aff?.value).toBe("Autobot");
|
|
134
|
+
});
|
|
135
|
+
it("extracts description and categories", () => {
|
|
136
|
+
const result = parseWikipedia(WIKIPEDIA_HTML);
|
|
137
|
+
expect(result.description).toContain("main protagonist");
|
|
138
|
+
expect(result.categories).toContain("Autobots");
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
describe("parseFandom", () => {
|
|
142
|
+
it("extracts infobox fields from Fandom portable infobox", () => {
|
|
143
|
+
const result = parseFandom(FANDOM_HTML);
|
|
144
|
+
expect(result.title).toBe("Optimus Prime");
|
|
145
|
+
const aff = result.infoboxFields.find((f) => f.label === "Affiliation");
|
|
146
|
+
expect(aff?.value).toBe("Autobot");
|
|
147
|
+
const fn = result.infoboxFields.find((f) => f.label === "Function");
|
|
148
|
+
expect(fn?.value).toBe("Supreme Commander");
|
|
149
|
+
});
|
|
150
|
+
it("extracts description and categories", () => {
|
|
151
|
+
const result = parseFandom(FANDOM_HTML);
|
|
152
|
+
expect(result.description).toContain("noble leader");
|
|
153
|
+
expect(result.categories).toContain("Autobots");
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
describe("utility functions", () => {
|
|
157
|
+
it("extractPageTitle returns h1 text", () => {
|
|
158
|
+
const $ = cheerio.load('<h1 id="firstHeading">Test Title</h1>');
|
|
159
|
+
expect(extractPageTitle($)).toBe("Test Title");
|
|
160
|
+
});
|
|
161
|
+
it("extractPageTitle falls back to title tag", () => {
|
|
162
|
+
const $ = cheerio.load("<title>Fallback Title</title>");
|
|
163
|
+
expect(extractPageTitle($)).toBe("Fallback Title");
|
|
164
|
+
});
|
|
165
|
+
it("extractFirstParagraph skips short paragraphs", () => {
|
|
166
|
+
const html = '<div><p>Short</p><p>This is a sufficiently long paragraph for extraction.</p></div>';
|
|
167
|
+
const $ = cheerio.load(html);
|
|
168
|
+
expect(extractFirstParagraph($)).toBe("This is a sufficiently long paragraph for extraction.");
|
|
169
|
+
});
|
|
170
|
+
it("extractCategories returns empty array when no catlinks", () => {
|
|
171
|
+
const $ = cheerio.load("<body></body>");
|
|
172
|
+
expect(extractCategories($)).toEqual([]);
|
|
173
|
+
});
|
|
174
|
+
});
|
|
175
|
+
//# sourceMappingURL=crawler.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crawler.test.js","sourceRoot":"","sources":["../../src/__tests__/crawler.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAClG,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,MAAM,WAAW,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;QA4BZ,CAAC;AAET,MAAM,cAAc,GAAG;;;;;;;;;;;;;;;;;;;;;;QAsBf,CAAC;AAET,MAAM,WAAW,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QA6BZ,CAAC;AAET,MAAM,eAAe,GAAG;;;;;;;;;QAShB,CAAC;AAET,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAEhD,MAAM,MAAM,GAAG,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QAEtC,MAAM,GAAG,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,aAAa,CAAC,CAAC;QACxE,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEnC,MAAM,EAAE,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC;QACrE,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qBAAqB,EAAE,GAAG,EAAE;QAC7B,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,MAAM,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,MAAM,MAAM,GAAG,cAAc,CAAC,cAAc,CAAC,CAAC;QAC9C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAE3C,MAAM,GAAG,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,aAAa,CAAC,CAAC;QACxE,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,MAAM,GAAG,cAAc,CAAC,cAAc,CAAC,CAAC;QAC9C,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAE3C,MAAM,GAAG,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,aAAa,CAAC,CAAC;QACxE,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEnC,MAAM,EAAE,GAAG,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC;QACpE,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACrD,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QAChE,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QACxD,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,IAAI,GAAG,qFAAqF,CAAC;QACnG,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACjG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACxC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { mergeEntry, defaultMergeRules } from "../crawler/merger.js";
|
|
3
|
+
const source1 = {
|
|
4
|
+
url: "https://tfwiki.net/wiki/Optimus_Prime_(G1)",
|
|
5
|
+
wiki: "tfwiki",
|
|
6
|
+
fetchedAt: new Date("2025-01-01"),
|
|
7
|
+
};
|
|
8
|
+
const source2 = {
|
|
9
|
+
url: "https://en.wikipedia.org/wiki/Optimus_Prime",
|
|
10
|
+
wiki: "wikipedia",
|
|
11
|
+
fetchedAt: new Date("2025-01-02"),
|
|
12
|
+
};
|
|
13
|
+
function makeCrawled(overrides = {}) {
|
|
14
|
+
return {
|
|
15
|
+
name: "Optimus Prime",
|
|
16
|
+
description: "Heroic leader of the Autobots",
|
|
17
|
+
continuityId: "Generation 1",
|
|
18
|
+
factions: ["Autobot"],
|
|
19
|
+
customFields: {
|
|
20
|
+
primaryFunction: "Leader",
|
|
21
|
+
sparkId: "SPARK-001",
|
|
22
|
+
},
|
|
23
|
+
source: source1,
|
|
24
|
+
...overrides,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
function makeExisting(overrides = {}) {
|
|
28
|
+
return {
|
|
29
|
+
id: "existing-id",
|
|
30
|
+
name: "Optimus Prime",
|
|
31
|
+
description: "Leader of the Autobots",
|
|
32
|
+
continuityId: "Generation 1",
|
|
33
|
+
factions: ["Autobot"],
|
|
34
|
+
sparkId: "MANUAL-001",
|
|
35
|
+
primaryFunction: "Supreme Commander",
|
|
36
|
+
sources: [source2],
|
|
37
|
+
updatedAt: new Date("2024-01-01"),
|
|
38
|
+
...overrides,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
describe("mergeEntry — new entry (no existing)", () => {
|
|
42
|
+
it("returns action=new with all crawled data", () => {
|
|
43
|
+
const result = mergeEntry(null, makeCrawled(), defaultMergeRules());
|
|
44
|
+
expect(result.action).toBe("new");
|
|
45
|
+
expect(result.doc?.name).toBe("Optimus Prime");
|
|
46
|
+
expect(result.doc?.description).toBe("Heroic leader of the Autobots");
|
|
47
|
+
expect(result.doc?.continuityId).toBe("Generation 1");
|
|
48
|
+
expect(result.doc.primaryFunction).toBe("Leader");
|
|
49
|
+
expect(result.doc.sources).toEqual([source1]);
|
|
50
|
+
});
|
|
51
|
+
it("skips if name is empty and skip-if-no-name is true", () => {
|
|
52
|
+
const result = mergeEntry(null, makeCrawled({ name: "" }), defaultMergeRules());
|
|
53
|
+
expect(result.action).toBe("skipped");
|
|
54
|
+
});
|
|
55
|
+
});
|
|
56
|
+
describe("mergeEntry — duplicate (existing)", () => {
|
|
57
|
+
it("skips if source already exists and skip-if-source-exists is true", () => {
|
|
58
|
+
const crawled = makeCrawled({ source: source2 });
|
|
59
|
+
const existing = makeExisting();
|
|
60
|
+
const result = mergeEntry(existing, crawled, defaultMergeRules());
|
|
61
|
+
expect(result.action).toBe("skipped");
|
|
62
|
+
});
|
|
63
|
+
it("merges sources when new source is different", () => {
|
|
64
|
+
const result = mergeEntry(makeExisting(), makeCrawled(), defaultMergeRules());
|
|
65
|
+
expect(result.action).toBe("merged");
|
|
66
|
+
expect(result.doc.sources).toHaveLength(2);
|
|
67
|
+
expect(result.doc.sources[0]).toEqual(source2);
|
|
68
|
+
expect(result.doc.sources[1]).toEqual(source1);
|
|
69
|
+
});
|
|
70
|
+
it("keeps longer description when longer-description is true", () => {
|
|
71
|
+
const crawled = makeCrawled({ description: "Short" });
|
|
72
|
+
const existing = makeExisting({ description: "This is a much longer description for the character" });
|
|
73
|
+
const result = mergeEntry(existing, crawled, defaultMergeRules());
|
|
74
|
+
expect(result.doc.description).toBe("This is a much longer description for the character");
|
|
75
|
+
});
|
|
76
|
+
it("prefers crawled description when longer", () => {
|
|
77
|
+
const crawled = makeCrawled({ description: "This is a much longer crawled description" });
|
|
78
|
+
const existing = makeExisting({ description: "Short" });
|
|
79
|
+
const result = mergeEntry(existing, crawled, defaultMergeRules());
|
|
80
|
+
expect(result.doc.description).toBe("This is a much longer crawled description");
|
|
81
|
+
});
|
|
82
|
+
it("unions factions", () => {
|
|
83
|
+
const crawled = makeCrawled({ factions: ["Autobot", "Prime"] });
|
|
84
|
+
const existing = makeExisting({ factions: ["Autobot", "Leader"] });
|
|
85
|
+
const result = mergeEntry(existing, crawled, defaultMergeRules());
|
|
86
|
+
const factions = result.doc.factions;
|
|
87
|
+
expect(factions).toContain("Autobot");
|
|
88
|
+
expect(factions).toContain("Prime");
|
|
89
|
+
expect(factions).toContain("Leader");
|
|
90
|
+
expect(factions).toHaveLength(3);
|
|
91
|
+
});
|
|
92
|
+
it("fills empty custom fields when fill-empty-fields is true", () => {
|
|
93
|
+
const existing = makeExisting({ sparkId: undefined });
|
|
94
|
+
const crawled = makeCrawled({ customFields: { sparkId: "SPARK-001" } });
|
|
95
|
+
const result = mergeEntry(existing, crawled, defaultMergeRules());
|
|
96
|
+
expect(result.doc.sparkId).toBe("SPARK-001");
|
|
97
|
+
});
|
|
98
|
+
it("preserves existing custom field when fill-empty-fields is true", () => {
|
|
99
|
+
const existing = makeExisting({ sparkId: "EXISTING-SPARK" });
|
|
100
|
+
const crawled = makeCrawled({ customFields: { sparkId: "CRAWLED-SPARK" } });
|
|
101
|
+
const result = mergeEntry(existing, crawled, defaultMergeRules());
|
|
102
|
+
expect(result.doc.sparkId).toBe("EXISTING-SPARK");
|
|
103
|
+
});
|
|
104
|
+
it("overwrites when crawled-wins is true", () => {
|
|
105
|
+
const existing = makeExisting({ sparkId: "EXISTING" });
|
|
106
|
+
const crawled = makeCrawled({ customFields: { sparkId: "CRAWLED" } });
|
|
107
|
+
const rules = { ...defaultMergeRules(), "crawled-wins": true, "fill-empty-fields": false };
|
|
108
|
+
const result = mergeEntry(existing, crawled, rules);
|
|
109
|
+
expect(result.doc.sparkId).toBe("CRAWLED");
|
|
110
|
+
});
|
|
111
|
+
it("preserves existing continuity when continuity-existing is true", () => {
|
|
112
|
+
const existing = makeExisting({ continuityId: "Beast Wars" });
|
|
113
|
+
const crawled = makeCrawled({ continuityId: "Generation 1" });
|
|
114
|
+
const result = mergeEntry(existing, crawled, defaultMergeRules());
|
|
115
|
+
expect(result.doc.continuityId).toBe("Beast Wars");
|
|
116
|
+
});
|
|
117
|
+
it("overwrites continuity when continuity-existing is false", () => {
|
|
118
|
+
const existing = makeExisting({ continuityId: "Beast Wars" });
|
|
119
|
+
const crawled = makeCrawled({ continuityId: "Generation 1" });
|
|
120
|
+
const rules = { ...defaultMergeRules(), "continuity-existing": false };
|
|
121
|
+
const result = mergeEntry(existing, crawled, rules);
|
|
122
|
+
expect(result.doc.continuityId).toBe("Generation 1");
|
|
123
|
+
});
|
|
124
|
+
});
|
|
125
|
+
describe("defaultMergeRules", () => {
|
|
126
|
+
it("returns all defaults", () => {
|
|
127
|
+
const rules = defaultMergeRules();
|
|
128
|
+
expect(rules["sources-merge"]).toBe(true);
|
|
129
|
+
expect(rules["longer-description"]).toBe(true);
|
|
130
|
+
expect(rules["factions-union"]).toBe(true);
|
|
131
|
+
expect(rules["fill-empty-fields"]).toBe(true);
|
|
132
|
+
expect(rules["skip-if-no-name"]).toBe(true);
|
|
133
|
+
expect(rules["skip-if-source-exists"]).toBe(true);
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
//# sourceMappingURL=merger.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"merger.test.js","sourceRoot":"","sources":["../../src/__tests__/merger.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAQrE,MAAM,OAAO,GAAsB;IACjC,GAAG,EAAE,4CAA4C;IACjD,IAAI,EAAE,QAAQ;IACd,SAAS,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC;CAClC,CAAC;AAEF,MAAM,OAAO,GAAsB;IACjC,GAAG,EAAE,6CAA6C;IAClD,IAAI,EAAE,WAAW;IACjB,SAAS,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC;CAClC,CAAC;AAEF,SAAS,WAAW,CAAC,YAA4C,EAAE;IACjE,OAAO;QACL,IAAI,EAAE,eAAe;QACrB,WAAW,EAAE,+BAA+B;QAC5C,YAAY,EAAE,cAAc;QAC5B,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,YAAY,EAAE;YACZ,eAAe,EAAE,QAAQ;YACzB,OAAO,EAAE,WAAW;SACrB;QACD,MAAM,EAAE,OAAO;QACf,GAAG,SAAS;KACY,CAAC;AAC7B,CAAC;AAED,SAAS,YAAY,CAAC,YAA8B,EAAE;IACpD,OAAO;QACL,EAAE,EAAE,aAAa;QACjB,IAAI,EAAE,eAAe;QACrB,WAAW,EAAE,wBAAwB;QACrC,YAAY,EAAE,cAAc;QAC5B,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,OAAO,EAAE,YAAY;QACrB,eAAe,EAAE,mBAAmB;QACpC,OAAO,EAAE,CAAC,OAAO,CAAC;QAClB,SAAS,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC;QACjC,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,sCAAsC,EAAE,GAAG,EAAE;IACpD,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACpE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QACtE,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACtD,MAAM,CAAE,MAAM,CAAC,GAAW,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3D,MAAM,CAAE,MAAM,CAAC,GAAW,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAChF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,mCAAmC,EAAE,GAAG,EAAE;IACjD,EAAE,CAAC,kEAAkE,EAAE,GAAG,EAAE;QAC1E,MAAM,OAAO,GAAG,WAAW,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACjD,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAClE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,MAAM,MAAM,GAAG,UAAU,CAAC,YAAY,EAAE,EAAE,WAAW,EAAE,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAC9E,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,CAAE,MAAM,CAAC,GAAW,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACpD,MAAM,CAAE,MAAM,CAAC,GAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxD,MAAM,CAAE,MAAM,CAAC,GAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,OAAO,GAAG,WAAW,CAAC,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,WAAW,EAAE,qDAAqD,EAAE,CAAC,CAAC;QACtG,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAClE,MAAM,CAAE,MAAM,CAAC,GAAW,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;IACtG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,OAAO,GAAG,WAAW,CAAC,EAAE,WAAW,EAAE,2CAA2C,EAAE,CAAC,CAAC;QAC1F,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAClE,MAAM,CAAE,MAAM,CAAC,GAAW,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;IAC5F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iBAAiB,EAAE,GAAG,EAAE;QACzB,MAAM,OAAO,GAAG,WAAW,CAAC,EAAE,QAAQ,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,QAAQ,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;QACnE,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAI,MAAM,CAAC,GAAW,CAAC,QAAQ,CAAC;QAC9C,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACpC,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,WAAW,CAAC,EAAE,YAAY,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;QACxE,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAClE,MAAM,CAAE,MAAM,CAAC,GAAW,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,WAAW,CAAC,EAAE,YAAY,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;QAC5E,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAClE,MAAM,CAAE,MAAM,CAAC,GAAW,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,WAAW,CAAC,EAAE,YAAY,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;QACtE,MAAM,KAAK,GAAG,EAAE,GAAG,iBAAiB,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,CAAC;QAC3F,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QACpD,MAAM,CAAE,MAAM,CAAC,GAAW,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC,CAAC;QAC9D,MAAM,OAAO,GAAG,WAAW,CAAC,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAClE,MAAM,CAAE,MAAM,CAAC,GAAW,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC,CAAC;QAC9D,MAAM,OAAO,GAAG,WAAW,CAAC,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC,CAAC;QAC9D,MAAM,KAAK,GAAG,EAAE,GAAG,iBAAiB,EAAE,EAAE,qBAAqB,EAAE,KAAK,EAAE,CAAC;QACvE,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QACpD,MAAM,CAAE,MAAM,CAAC,GAAW,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;QAC9B,MAAM,KAAK,GAAG,iBAAiB,EAAE,CAAC;QAClC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { CrawlSourceConfig, CrawledEntry, CrawlProgress, MergeableDocument } from "./types.js";
|
|
2
|
+
export declare class Crawler<T extends MergeableDocument> {
|
|
3
|
+
private client;
|
|
4
|
+
private parserFn;
|
|
5
|
+
private sourceConfig;
|
|
6
|
+
constructor(config: CrawlSourceConfig);
|
|
7
|
+
discover(categoryPath: string, maxPages?: number): Promise<string[]>;
|
|
8
|
+
discoverAll(categoryPath: string, maxPages?: number, maxDepth?: number): Promise<string[]>;
|
|
9
|
+
scrape(pageTitle: string): Promise<CrawledEntry<T> | null>;
|
|
10
|
+
crawlCategory(categoryPath: string, onProgress?: (progress: CrawlProgress) => void, maxPages?: number, maxDepth?: number): Promise<CrawledEntry<T>[]>;
|
|
11
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { MediaWikiClient } from "./mediawiki.js";
|
|
2
|
+
import { getParser } from "./parsers/index.js";
|
|
3
|
+
function castValue(raw, type) {
|
|
4
|
+
switch (type) {
|
|
5
|
+
case "number": {
|
|
6
|
+
const n = parseInt(raw, 10);
|
|
7
|
+
return isNaN(n) ? raw : n;
|
|
8
|
+
}
|
|
9
|
+
case "boolean":
|
|
10
|
+
return raw.toLowerCase() === "yes" || raw.toLowerCase() === "true";
|
|
11
|
+
default:
|
|
12
|
+
return raw;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
function mapFields(parsed, fieldMapping, continuityId) {
|
|
16
|
+
const result = {
|
|
17
|
+
description: parsed.description,
|
|
18
|
+
continuityId,
|
|
19
|
+
factions: [],
|
|
20
|
+
customFields: {},
|
|
21
|
+
};
|
|
22
|
+
// Try to infer continuity from categories
|
|
23
|
+
for (const cat of parsed.categories) {
|
|
24
|
+
const lower = cat.toLowerCase();
|
|
25
|
+
if (lower.includes("generation 1") || lower.includes("g1") ||
|
|
26
|
+
lower.includes("beast wars") || lower.includes("beast machines") ||
|
|
27
|
+
lower.includes("armada") || lower.includes("energon") ||
|
|
28
|
+
lower.includes("cybertron") || lower.includes("animated") ||
|
|
29
|
+
lower.includes("prime") || lower.includes("robots in disguise") ||
|
|
30
|
+
lower.includes("aligned") || lower.includes("cyberverse") ||
|
|
31
|
+
lower.includes("earthspark") || lower.includes("wfc") ||
|
|
32
|
+
lower.includes("siege") || lower.includes("earthrise") ||
|
|
33
|
+
lower.includes("kingdom")) {
|
|
34
|
+
result.continuityId = cat.replace(/^Category:/, "");
|
|
35
|
+
break;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
for (const field of parsed.infoboxFields) {
|
|
39
|
+
const mapping = fieldMapping[field.label];
|
|
40
|
+
if (!mapping)
|
|
41
|
+
continue;
|
|
42
|
+
if (mapping.target === "factions") {
|
|
43
|
+
result.factions = field.value
|
|
44
|
+
.split(/[,;/\n]+/)
|
|
45
|
+
.map((s) => s.trim())
|
|
46
|
+
.filter(Boolean);
|
|
47
|
+
}
|
|
48
|
+
else if (mapping.target === "description") {
|
|
49
|
+
if (field.value.length > result.description.length) {
|
|
50
|
+
result.description = field.value;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
else if (mapping.target === "continuityId") {
|
|
54
|
+
if (!result.continuityId)
|
|
55
|
+
result.continuityId = field.value;
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
result.customFields[mapping.target] = castValue(field.value, mapping.type);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return result;
|
|
62
|
+
}
|
|
63
|
+
export class Crawler {
|
|
64
|
+
client;
|
|
65
|
+
parserFn;
|
|
66
|
+
sourceConfig;
|
|
67
|
+
constructor(config) {
|
|
68
|
+
this.client = new MediaWikiClient(config);
|
|
69
|
+
this.parserFn = getParser(config.parser);
|
|
70
|
+
this.sourceConfig = config;
|
|
71
|
+
}
|
|
72
|
+
async discover(categoryPath, maxPages = 500) {
|
|
73
|
+
return this.client.listCategoryMembers(categoryPath, maxPages);
|
|
74
|
+
}
|
|
75
|
+
async discoverAll(categoryPath, maxPages = 500, maxDepth = 3) {
|
|
76
|
+
return this.client.discoverAllPages(categoryPath, maxPages, maxDepth);
|
|
77
|
+
}
|
|
78
|
+
async scrape(pageTitle) {
|
|
79
|
+
const html = await this.client.fetchPageHtml(pageTitle);
|
|
80
|
+
const parsed = this.parserFn(html);
|
|
81
|
+
if (!parsed.title)
|
|
82
|
+
return null;
|
|
83
|
+
const mapped = mapFields(parsed, this.sourceConfig.fieldMapping, "");
|
|
84
|
+
const source = {
|
|
85
|
+
url: `${this.sourceConfig.baseUrl}/wiki/${encodeURIComponent(pageTitle.replace(/ /g, "_"))}`,
|
|
86
|
+
wiki: this.sourceConfig.id,
|
|
87
|
+
fetchedAt: new Date(),
|
|
88
|
+
};
|
|
89
|
+
return {
|
|
90
|
+
name: parsed.title,
|
|
91
|
+
description: mapped.description,
|
|
92
|
+
continuityId: mapped.continuityId,
|
|
93
|
+
factions: mapped.factions,
|
|
94
|
+
customFields: mapped.customFields,
|
|
95
|
+
source,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
async crawlCategory(categoryPath, onProgress, maxPages = 500, maxDepth = 3) {
|
|
99
|
+
const titles = await this.discoverAll(categoryPath, maxPages, maxDepth);
|
|
100
|
+
const entries = [];
|
|
101
|
+
for (let i = 0; i < titles.length; i++) {
|
|
102
|
+
onProgress?.({
|
|
103
|
+
current: i + 1,
|
|
104
|
+
total: titles.length,
|
|
105
|
+
pageTitle: titles[i],
|
|
106
|
+
});
|
|
107
|
+
try {
|
|
108
|
+
const entry = await this.scrape(titles[i]);
|
|
109
|
+
if (entry)
|
|
110
|
+
entries.push(entry);
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
// skip pages that fail to parse
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return entries;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/crawler/index.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/C,SAAS,SAAS,CAChB,GAAW,EACX,IAA+B;IAE/B,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC5B,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;QACD,KAAK,SAAS;YACZ,OAAO,GAAG,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;QACrE;YACE,OAAO,GAAG,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAChB,MAAsB,EACtB,YAA+C,EAC/C,YAAoB;IAEpB,MAAM,MAAM,GAA6G;QACvH,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,YAAY;QACZ,QAAQ,EAAE,EAAE;QACZ,YAAY,EAAE,EAAE;KACjB,CAAC;IAEF,0CAA0C;IAC1C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,IACE,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YACtD,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,gBAAgB,CAAC;YAChE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;YACrD,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC;YACzD,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,oBAAoB,CAAC;YAC/D,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;YACzD,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YACrD,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC;YACtD,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,EACzB,CAAC;YACD,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACpD,MAAM;QACR,CAAC;IACH,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAClC,MAAM,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK;iBAC1B,KAAK,CAAC,UAAU,CAAC;iBACjB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;iBACpB,MAAM,CAAC,OAAO,CAAC,CAAC;QACrB,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;YAC5C,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;gBACnD,MAAM,CAAC,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC;YACnC,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;YAC7C,IAAI,CAAC,MAAM,CAAC,YAAY;gBAAE,MAAM,CAAC,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,OAAO,OAAO;IACV,MAAM,CAAkB;IACxB,QAAQ,CAAmC;IAC3C,YAAY,CAAoB;IAExC,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,YAAoB,EAAE,QAAQ,GAAG,GAAG;QACjD,OAAO,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,WAAW,CACf,YAAoB,EACpB,QAAQ,GAAG,GAAG,EACd,QAAQ,GAAG,CAAC;QAEZ,OAAO,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,YAAY,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAiB;QAC5B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QAE/B,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QAErE,MAAM,MAAM,GAAsB;YAChC,GAAG,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,SAAS,kBAAkB,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE;YAC5F,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE;YAC1B,SAAS,EAAE,IAAI,IAAI,EAAE;SACtB,CAAC;QAEF,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,KAAK;YAClB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,MAAM;SACY,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,YAAoB,EACpB,UAA8C,EAC9C,QAAQ,GAAG,GAAG,EACd,QAAQ,GAAG,CAAC;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACxE,MAAM,OAAO,GAAsB,EAAE,CAAC;QAEtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,UAAU,EAAE,CAAC;gBACX,OAAO,EAAE,CAAC,GAAG,CAAC;gBACd,KAAK,EAAE,MAAM,CAAC,MAAM;gBACpB,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;aACrB,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,IAAI,KAAK;oBAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;YAAC,MAAM,CAAC;gBACP,gCAAgC;YAClC,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
interface MediaWikiConfig {
|
|
2
|
+
baseUrl: string;
|
|
3
|
+
apiUrl: string;
|
|
4
|
+
}
|
|
5
|
+
export declare class MediaWikiClient {
|
|
6
|
+
private config;
|
|
7
|
+
private delayMs;
|
|
8
|
+
constructor(config: MediaWikiConfig, delayMs?: number);
|
|
9
|
+
private delay;
|
|
10
|
+
private fetchJson;
|
|
11
|
+
private fetchAllCategoryMembers;
|
|
12
|
+
listCategoryMembers(categoryPath: string, maxPages?: number): Promise<string[]>;
|
|
13
|
+
listSubCategories(categoryPath: string, maxPages?: number): Promise<string[]>;
|
|
14
|
+
discoverAllPages(categoryPath: string, maxPages?: number, maxDepth?: number, seen?: Set<string>): Promise<string[]>;
|
|
15
|
+
fetchPageHtml(pageTitle: string): Promise<string>;
|
|
16
|
+
searchPages(query: string, maxResults?: number): Promise<string[]>;
|
|
17
|
+
}
|
|
18
|
+
export {};
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
export class MediaWikiClient {
|
|
2
|
+
config;
|
|
3
|
+
delayMs;
|
|
4
|
+
constructor(config, delayMs = 250) {
|
|
5
|
+
this.config = config;
|
|
6
|
+
this.delayMs = delayMs;
|
|
7
|
+
}
|
|
8
|
+
async delay(ms = this.delayMs) {
|
|
9
|
+
if (ms > 0)
|
|
10
|
+
await new Promise((r) => setTimeout(r, ms));
|
|
11
|
+
}
|
|
12
|
+
async fetchJson(url) {
|
|
13
|
+
const res = await fetch(url, {
|
|
14
|
+
headers: { "User-Agent": "CAP-Crawler/1.0 (crawler; contact@crazygiscool)" },
|
|
15
|
+
});
|
|
16
|
+
if (!res.ok) {
|
|
17
|
+
throw new Error(`MediaWiki API error: ${res.status} ${res.statusText}`);
|
|
18
|
+
}
|
|
19
|
+
return res.json();
|
|
20
|
+
}
|
|
21
|
+
async fetchAllCategoryMembers(categoryPath, maxPages, cmtype) {
|
|
22
|
+
const allTitles = [];
|
|
23
|
+
let cmcontinue;
|
|
24
|
+
while (allTitles.length < maxPages) {
|
|
25
|
+
const params = new URLSearchParams({
|
|
26
|
+
action: "query",
|
|
27
|
+
format: "json",
|
|
28
|
+
list: "categorymembers",
|
|
29
|
+
cmtitle: categoryPath,
|
|
30
|
+
cmlimit: String(Math.min(maxPages - allTitles.length, 500)),
|
|
31
|
+
cmtype,
|
|
32
|
+
});
|
|
33
|
+
if (cmcontinue)
|
|
34
|
+
params.set("cmcontinue", cmcontinue);
|
|
35
|
+
const url = `${this.config.apiUrl}?${params}`;
|
|
36
|
+
const data = (await this.fetchJson(url));
|
|
37
|
+
if (data.error) {
|
|
38
|
+
throw new Error(`MediaWiki API: ${data.error.code} — ${data.error.info}`);
|
|
39
|
+
}
|
|
40
|
+
const members = data.query?.categorymembers ?? [];
|
|
41
|
+
allTitles.push(...members.map((m) => m.title));
|
|
42
|
+
if (data.continue?.cmcontinue) {
|
|
43
|
+
cmcontinue = data.continue.cmcontinue;
|
|
44
|
+
await this.delay();
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
break;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return allTitles.slice(0, maxPages);
|
|
51
|
+
}
|
|
52
|
+
async listCategoryMembers(categoryPath, maxPages = 500) {
|
|
53
|
+
return this.fetchAllCategoryMembers(categoryPath, maxPages, "page");
|
|
54
|
+
}
|
|
55
|
+
async listSubCategories(categoryPath, maxPages = 500) {
|
|
56
|
+
return this.fetchAllCategoryMembers(categoryPath, maxPages, "subcat");
|
|
57
|
+
}
|
|
58
|
+
async discoverAllPages(categoryPath, maxPages = 500, maxDepth = 3, seen = new Set()) {
|
|
59
|
+
if (maxDepth < 0 || seen.has(categoryPath))
|
|
60
|
+
return [];
|
|
61
|
+
seen.add(categoryPath);
|
|
62
|
+
const pages = await this.listCategoryMembers(categoryPath, maxPages);
|
|
63
|
+
const all = new Set(pages);
|
|
64
|
+
if (maxDepth > 0) {
|
|
65
|
+
await this.delay();
|
|
66
|
+
const subcategories = await this.listSubCategories(categoryPath, maxPages);
|
|
67
|
+
for (const subcat of subcategories) {
|
|
68
|
+
if (all.size >= maxPages)
|
|
69
|
+
break;
|
|
70
|
+
await this.delay();
|
|
71
|
+
const subPages = await this.discoverAllPages(subcat, maxPages - all.size, maxDepth - 1, seen);
|
|
72
|
+
for (const p of subPages) {
|
|
73
|
+
all.add(p);
|
|
74
|
+
if (all.size >= maxPages)
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return [...all].slice(0, maxPages);
|
|
80
|
+
}
|
|
81
|
+
async fetchPageHtml(pageTitle) {
|
|
82
|
+
const params = new URLSearchParams({
|
|
83
|
+
action: "parse",
|
|
84
|
+
format: "json",
|
|
85
|
+
page: pageTitle,
|
|
86
|
+
prop: "text",
|
|
87
|
+
});
|
|
88
|
+
const url = `${this.config.apiUrl}?${params}`;
|
|
89
|
+
const data = (await this.fetchJson(url));
|
|
90
|
+
if (data.error) {
|
|
91
|
+
throw new Error(`MediaWiki API: ${data.error.code} — ${data.error.info}`);
|
|
92
|
+
}
|
|
93
|
+
return data.parse?.text?.["*"] ?? "";
|
|
94
|
+
}
|
|
95
|
+
async searchPages(query, maxResults = 50) {
|
|
96
|
+
const params = new URLSearchParams({
|
|
97
|
+
action: "query",
|
|
98
|
+
format: "json",
|
|
99
|
+
list: "search",
|
|
100
|
+
srsearch: query,
|
|
101
|
+
srlimit: String(Math.min(maxResults, 500)),
|
|
102
|
+
});
|
|
103
|
+
const url = `${this.config.apiUrl}?${params}`;
|
|
104
|
+
const data = (await this.fetchJson(url));
|
|
105
|
+
if (data.error) {
|
|
106
|
+
throw new Error(`MediaWiki API: ${data.error.code} — ${data.error.info}`);
|
|
107
|
+
}
|
|
108
|
+
return (data.query?.search ?? []).map((s) => s.title);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
//# sourceMappingURL=mediawiki.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mediawiki.js","sourceRoot":"","sources":["../../src/crawler/mediawiki.ts"],"names":[],"mappings":"AA0BA,MAAM,OAAO,eAAe;IAClB,MAAM,CAAkB;IACxB,OAAO,CAAS;IAExB,YAAY,MAAuB,EAAE,OAAO,GAAG,GAAG;QAChD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,OAAO;QACnC,IAAI,EAAE,GAAG,CAAC;YAAE,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,GAAW;QAEX,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,OAAO,EAAE,EAAE,YAAY,EAAE,iDAAiD,EAAE;SAC7E,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,wBAAwB,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CACvD,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,CAAC,IAAI,EAAsC,CAAC;IACxD,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,YAAoB,EACpB,QAAgB,EAChB,MAAyB;QAEzB,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,IAAI,UAA8B,CAAC;QAEnC,OAAO,SAAS,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;gBACjC,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,iBAAiB;gBACvB,OAAO,EAAE,YAAY;gBACrB,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC3D,MAAM;aACP,CAAC,CAAC;YACH,IAAI,UAAU;gBAAE,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;YAErD,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAmB,CAAC;YAE3D,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CACb,kBAAkB,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CACzD,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,EAAE,eAAe,IAAI,EAAE,CAAC;YAClD,SAAS,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAiB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YAE/D,IAAI,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC;gBAC9B,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACtC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,mBAAmB,CACvB,YAAoB,EACpB,QAAQ,GAAG,GAAG;QAEd,OAAO,IAAI,CAAC,uBAAuB,CAAC,YAAY,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IACtE,CAAC;IAED,KAAK,CAAC,iBAAiB,CACrB,YAAoB,EACpB,QAAQ,GAAG,GAAG;QAEd,OAAO,IAAI,CAAC,uBAAuB,CAAC,YAAY,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,YAAoB,EACpB,QAAQ,GAAG,GAAG,EACd,QAAQ,GAAG,CAAC,EACZ,OAAO,IAAI,GAAG,EAAU;QAExB,IAAI,QAAQ,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC;YAAE,OAAO,EAAE,CAAC;QACtD,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAEvB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QACrE,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QAE3B,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;YAC3E,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;gBACnC,IAAI,GAAG,CAAC,IAAI,IAAI,QAAQ;oBAAE,MAAM;gBAChC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACnB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAC1C,MAAM,EACN,QAAQ,GAAG,GAAG,CAAC,IAAI,EACnB,QAAQ,GAAG,CAAC,EACZ,IAAI,CACL,CAAC;gBACF,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;oBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;oBACX,IAAI,GAAG,CAAC,IAAI,IAAI,QAAQ;wBAAE,MAAM;gBAClC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,SAAiB;QACnC,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,MAAM;SACb,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAEtC,CAAC;QAEF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,kBAAkB,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CACzD,CAAC;QACJ,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,KAAa,EAAE,UAAU,GAAG,EAAE;QAC9C,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,KAAK;YACf,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;SAC3C,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAGtC,CAAC;QAEF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,kBAAkB,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5E,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAe,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACtE,CAAC;CACF"}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import type { CrawledEntry, MergeableDocument, MergeConfig, MergeResult } from "./types.js";
|
|
2
|
+
export declare function defaultMergeRules(): MergeConfig;
|
|
3
|
+
export declare function mergeEntry<T extends MergeableDocument>(existing: T | null, crawled: CrawledEntry<T>, rules: MergeConfig): MergeResult<T>;
|