@abraca/wiki 2.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -0
- package/dist/abracadabra-wiki.cjs +1418 -0
- package/dist/abracadabra-wiki.cjs.map +1 -0
- package/dist/abracadabra-wiki.esm.js +1387 -0
- package/dist/abracadabra-wiki.esm.js.map +1 -0
- package/dist/index.d.ts +27 -0
- package/package.json +44 -0
- package/src/connect.ts +69 -0
- package/src/crypto.ts +70 -0
- package/src/index.ts +508 -0
- package/src/parser.ts +62 -0
- package/src/render.ts +91 -0
- package/src/snapshot.ts +210 -0
- package/src/types.ts +45 -0
- package/src/wikipedia.ts +154 -0
|
@@ -0,0 +1,1387 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import wtf from "wtf_wikipedia";
|
|
3
|
+
import wtfApiPlugin from "wtf-plugin-api";
|
|
4
|
+
import { DocumentManager } from "@abraca/dabra";
|
|
5
|
+
import * as ed from "@noble/ed25519";
|
|
6
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
7
|
+
import { existsSync } from "node:fs";
|
|
8
|
+
import { homedir } from "node:os";
|
|
9
|
+
import { dirname, join } from "node:path";
|
|
10
|
+
|
|
11
|
+
//#region packages/wiki/src/parser.ts
|
|
12
|
+
/**
|
|
13
|
+
* Parse CLI arguments into a structured object.
|
|
14
|
+
* @param argv Raw `process.argv` (includes node path and script path).
|
|
15
|
+
*/
|
|
16
|
+
function parseArgs(argv) {
|
|
17
|
+
const args = argv.slice(2);
|
|
18
|
+
const result = {
|
|
19
|
+
positional: [],
|
|
20
|
+
params: {},
|
|
21
|
+
flags: /* @__PURE__ */ new Set()
|
|
22
|
+
};
|
|
23
|
+
for (let i = 0; i < args.length; i++) {
|
|
24
|
+
const arg = args[i];
|
|
25
|
+
if (arg.startsWith("--")) {
|
|
26
|
+
const stripped = arg.slice(2);
|
|
27
|
+
const eqIdx = stripped.indexOf("=");
|
|
28
|
+
if (eqIdx !== -1) result.params[stripped.slice(0, eqIdx)] = stripped.slice(eqIdx + 1);
|
|
29
|
+
else result.flags.add(stripped);
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
const eqIdx = arg.indexOf("=");
|
|
33
|
+
if (eqIdx > 0) {
|
|
34
|
+
const key = arg.slice(0, eqIdx);
|
|
35
|
+
let value = arg.slice(eqIdx + 1);
|
|
36
|
+
if (value.startsWith("\"") && value.endsWith("\"") || value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1);
|
|
37
|
+
result.params[key] = value;
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
result.positional.push(arg);
|
|
41
|
+
}
|
|
42
|
+
return result;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
//#endregion
|
|
46
|
+
//#region packages/wiki/src/wikipedia.ts
|
|
47
|
+
/**
|
|
48
|
+
* Rate-limited wrapper around wtf_wikipedia + wtf-plugin-api.
|
|
49
|
+
*
|
|
50
|
+
* Responsibilities:
|
|
51
|
+
* - Throttle requests to respect Wikimedia API etiquette
|
|
52
|
+
* - Cache parsed Documents by canonical title
|
|
53
|
+
* - Resolve redirects so callers always see the redirect target
|
|
54
|
+
* - Expose getCategoryPages via wtf-plugin-api
|
|
55
|
+
*/
|
|
56
|
+
let pluginExtended = false;
|
|
57
|
+
function ensurePlugin() {
|
|
58
|
+
if (pluginExtended) return;
|
|
59
|
+
wtf.extend(wtfApiPlugin);
|
|
60
|
+
pluginExtended = true;
|
|
61
|
+
}
|
|
62
|
+
/** A token-bucket-ish throttle: at most `rate` calls per second, FIFO. */
|
|
63
|
+
var RateLimiter = class {
|
|
64
|
+
lastTickMs = 0;
|
|
65
|
+
constructor(intervalMs) {
|
|
66
|
+
this.intervalMs = intervalMs;
|
|
67
|
+
}
|
|
68
|
+
async wait() {
|
|
69
|
+
const now = Date.now();
|
|
70
|
+
const earliest = this.lastTickMs + this.intervalMs;
|
|
71
|
+
if (now < earliest) await new Promise((r) => setTimeout(r, earliest - now));
|
|
72
|
+
this.lastTickMs = Math.max(now, earliest);
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
var WikipediaClient = class {
|
|
76
|
+
cache = /* @__PURE__ */ new Map();
|
|
77
|
+
redirects = /* @__PURE__ */ new Map();
|
|
78
|
+
limiter;
|
|
79
|
+
fetchOpts;
|
|
80
|
+
constructor(config) {
|
|
81
|
+
this.config = config;
|
|
82
|
+
ensurePlugin();
|
|
83
|
+
this.limiter = new RateLimiter(Math.max(50, Math.floor(1e3 / Math.max(.1, config.rate))));
|
|
84
|
+
this.fetchOpts = {
|
|
85
|
+
lang: config.lang,
|
|
86
|
+
"Api-User-Agent": config.userAgent,
|
|
87
|
+
follow_redirects: true
|
|
88
|
+
};
|
|
89
|
+
if (config.domain) this.fetchOpts.domain = config.domain;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Fetch and parse a Wikipedia article.
|
|
93
|
+
* - Returns the cached Document if we've seen this title before.
|
|
94
|
+
* - Follows redirects and caches under both source and target titles.
|
|
95
|
+
* - Returns null when the page does not exist.
|
|
96
|
+
*/
|
|
97
|
+
async fetchArticle(rawTitle) {
|
|
98
|
+
const title = canonicalTitle(rawTitle);
|
|
99
|
+
if (this.cache.has(title)) return this.cache.get(title);
|
|
100
|
+
if (this.redirects.has(title)) {
|
|
101
|
+
const target = this.redirects.get(title);
|
|
102
|
+
return this.cache.get(target) ?? null;
|
|
103
|
+
}
|
|
104
|
+
await this.limiter.wait();
|
|
105
|
+
let doc;
|
|
106
|
+
try {
|
|
107
|
+
doc = await wtf.fetch(title, this.fetchOpts);
|
|
108
|
+
} catch (err) {
|
|
109
|
+
throw new Error(`Wikipedia fetch failed for "${title}": ${err?.message ?? err}`);
|
|
110
|
+
}
|
|
111
|
+
if (!doc) return null;
|
|
112
|
+
if (typeof doc.isRedirect === "function" && doc.isRedirect()) {
|
|
113
|
+
const target = doc.redirectTo?.()?.page;
|
|
114
|
+
if (typeof target === "string") {
|
|
115
|
+
this.redirects.set(title, canonicalTitle(target));
|
|
116
|
+
return await this.fetchArticle(target);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const resolvedTitle = canonicalTitle(doc.title?.() ?? title);
|
|
120
|
+
this.cache.set(resolvedTitle, doc);
|
|
121
|
+
if (resolvedTitle !== title) this.redirects.set(title, resolvedTitle);
|
|
122
|
+
return doc;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Fetch the member pages of a category (and optionally sub-categories).
|
|
126
|
+
* @param category Category title (with or without "Category:" prefix).
|
|
127
|
+
* @param recursive Whether to traverse sub-categories.
|
|
128
|
+
* @param maxDepth Recursion depth when recursive=true.
|
|
129
|
+
*/
|
|
130
|
+
async fetchCategoryPages(category, recursive, maxDepth) {
|
|
131
|
+
await this.limiter.wait();
|
|
132
|
+
const opts = {
|
|
133
|
+
lang: this.config.lang,
|
|
134
|
+
"Api-User-Agent": this.config.userAgent,
|
|
135
|
+
recursive,
|
|
136
|
+
maxDepth
|
|
137
|
+
};
|
|
138
|
+
if (this.config.domain) opts.domain = this.config.domain;
|
|
139
|
+
return (await wtf.getCategoryPages(category, opts) ?? []).map((m) => ({
|
|
140
|
+
title: canonicalTitle(m.title),
|
|
141
|
+
type: m.type === "subcat" ? "subcat" : "page"
|
|
142
|
+
}));
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
/** Normalize a Wikipedia title — trim, collapse spaces, strip leading/trailing colons. */
|
|
146
|
+
function canonicalTitle(s) {
|
|
147
|
+
return (s ?? "").toString().replace(/_/g, " ").replace(/\s+/g, " ").trim();
|
|
148
|
+
}
|
|
149
|
+
/** Detect a category-namespaced title. */
|
|
150
|
+
const CATEGORY_PREFIX = /^(Category|Catégorie|Kategorie|Categoría|Categoria|Categorie|Kategoria):/i;
|
|
151
|
+
function isCategoryTitle(title) {
|
|
152
|
+
return CATEGORY_PREFIX.test(title);
|
|
153
|
+
}
|
|
154
|
+
/** Strip the "Category:" prefix for display. */
|
|
155
|
+
function stripCategoryPrefix(title) {
|
|
156
|
+
return title.replace(CATEGORY_PREFIX, "").trim();
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
//#endregion
|
|
160
|
+
//#region packages/wiki/src/snapshot.ts
|
|
161
|
+
function snapshotArticle(doc, title) {
|
|
162
|
+
return {
|
|
163
|
+
title,
|
|
164
|
+
linkTitles: collectLinkTitles(doc),
|
|
165
|
+
categories: collectCategories(doc),
|
|
166
|
+
sections: snapshotSections(doc.sections?.() ?? []),
|
|
167
|
+
infobox: snapshotInfobox(doc.infobox?.()),
|
|
168
|
+
lead: leadParagraph(doc),
|
|
169
|
+
url: typeof doc.url === "function" ? doc.url() : null
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
function prettyCategoryLabel(catTitle) {
|
|
173
|
+
return stripCategoryPrefix(catTitle);
|
|
174
|
+
}
|
|
175
|
+
function collectLinkTitles(doc) {
|
|
176
|
+
const links = doc.links?.() ?? [];
|
|
177
|
+
const out = /* @__PURE__ */ new Set();
|
|
178
|
+
for (const l of links) {
|
|
179
|
+
if (!l) continue;
|
|
180
|
+
const page = typeof l.page === "function" ? l.page() : null;
|
|
181
|
+
if (typeof page !== "string" || page.length === 0) continue;
|
|
182
|
+
if (isCategoryTitle(page)) continue;
|
|
183
|
+
out.add(canonicalTitle(page));
|
|
184
|
+
}
|
|
185
|
+
return [...out];
|
|
186
|
+
}
|
|
187
|
+
function collectCategories(doc) {
|
|
188
|
+
const out = [];
|
|
189
|
+
for (const c of doc.categories?.() ?? []) {
|
|
190
|
+
const norm = canonicalTitle(c);
|
|
191
|
+
if (norm) out.push(norm);
|
|
192
|
+
}
|
|
193
|
+
return out;
|
|
194
|
+
}
|
|
195
|
+
function snapshotSections(rawSections) {
|
|
196
|
+
const all = rawSections.map((s) => ({
|
|
197
|
+
raw: s,
|
|
198
|
+
title: s.title?.() || "",
|
|
199
|
+
parentRef: typeof s.parent === "function" ? s.parent() : null,
|
|
200
|
+
children: []
|
|
201
|
+
}));
|
|
202
|
+
const byRaw = /* @__PURE__ */ new Map();
|
|
203
|
+
for (const s of all) byRaw.set(s.raw, s);
|
|
204
|
+
const roots = [];
|
|
205
|
+
for (const s of all) if (s.parentRef && byRaw.has(s.parentRef)) byRaw.get(s.parentRef).children.push(materialize(s));
|
|
206
|
+
else roots.push(s);
|
|
207
|
+
return roots.map(materialize);
|
|
208
|
+
}
|
|
209
|
+
function materialize(node) {
|
|
210
|
+
const lists = node.raw.lists?.() ?? [];
|
|
211
|
+
const paragraphs = node.raw.paragraphs?.() ?? [];
|
|
212
|
+
let listLength = 0;
|
|
213
|
+
for (const l of lists) {
|
|
214
|
+
const lines = l.lines?.() ?? [];
|
|
215
|
+
listLength += lines.length;
|
|
216
|
+
}
|
|
217
|
+
const isList = lists.length > 0 && (paragraphs.length === 0 || listLength >= paragraphs.length * 2);
|
|
218
|
+
const bodyParts = [];
|
|
219
|
+
for (const p of paragraphs) {
|
|
220
|
+
const md = paragraphMarkdown(p);
|
|
221
|
+
if (md) bodyParts.push(md);
|
|
222
|
+
}
|
|
223
|
+
for (const l of lists) {
|
|
224
|
+
const lines = l.lines?.() ?? [];
|
|
225
|
+
for (const line of lines) {
|
|
226
|
+
const text = lineText(line);
|
|
227
|
+
if (text) bodyParts.push(`- ${text}`);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return {
|
|
231
|
+
title: node.title,
|
|
232
|
+
body: bodyParts.join("\n\n"),
|
|
233
|
+
isList,
|
|
234
|
+
listLength,
|
|
235
|
+
children: node.children
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
function snapshotInfobox(box) {
|
|
239
|
+
if (!box) return void 0;
|
|
240
|
+
const data = typeof box.json === "function" ? box.json() : null;
|
|
241
|
+
if (!data || typeof data !== "object") return void 0;
|
|
242
|
+
const rows = [];
|
|
243
|
+
for (const [key, val] of Object.entries(data)) {
|
|
244
|
+
const value = stringifyInfoboxValue(val);
|
|
245
|
+
if (!value) continue;
|
|
246
|
+
rows.push({
|
|
247
|
+
key: humanKey(key),
|
|
248
|
+
value
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
return rows.length > 0 ? rows : void 0;
|
|
252
|
+
}
|
|
253
|
+
function stringifyInfoboxValue(val) {
|
|
254
|
+
if (val == null) return "";
|
|
255
|
+
if (typeof val === "string") return val;
|
|
256
|
+
if (typeof val === "number" || typeof val === "boolean") return String(val);
|
|
257
|
+
if (Array.isArray(val)) return val.map(stringifyInfoboxValue).filter(Boolean).join(", ");
|
|
258
|
+
if (typeof val === "object") {
|
|
259
|
+
const o = val;
|
|
260
|
+
if (typeof o.text === "string") return o.text;
|
|
261
|
+
if (typeof o.number === "number") return String(o.number);
|
|
262
|
+
}
|
|
263
|
+
return "";
|
|
264
|
+
}
|
|
265
|
+
function humanKey(k) {
|
|
266
|
+
return k.replace(/_/g, " ").replace(/^./, (m) => m.toUpperCase());
|
|
267
|
+
}
|
|
268
|
+
function leadParagraph(doc) {
|
|
269
|
+
const first = (doc.paragraphs?.() ?? [])[0];
|
|
270
|
+
if (!first) return "";
|
|
271
|
+
return paragraphMarkdown(first);
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Render a paragraph as markdown, replacing internal links with `[[Title]]`.
|
|
275
|
+
* The streaming orchestrator's link rewriter later swaps `[[Title]]` →
|
|
276
|
+
* `[[docId|label]]` once IDs are known.
|
|
277
|
+
*/
|
|
278
|
+
function paragraphMarkdown(paragraph) {
|
|
279
|
+
const sentences = paragraph.sentences?.() ?? [];
|
|
280
|
+
const out = [];
|
|
281
|
+
for (const s of sentences) out.push(sentenceWithWikilinks(s));
|
|
282
|
+
return out.join(" ").trim();
|
|
283
|
+
}
|
|
284
|
+
function sentenceWithWikilinks(sentence) {
|
|
285
|
+
const text = (sentence.text?.() ?? "").toString();
|
|
286
|
+
const links = sentence.links?.() ?? [];
|
|
287
|
+
if (links.length === 0) return text;
|
|
288
|
+
let result = text;
|
|
289
|
+
const replacements = links.map((l) => {
|
|
290
|
+
const page = typeof l.page === "function" ? l.page() : null;
|
|
291
|
+
const display = typeof l.text === "function" ? l.text() : null;
|
|
292
|
+
if (typeof page !== "string" || page.length === 0) return null;
|
|
293
|
+
if (isCategoryTitle(page)) return null;
|
|
294
|
+
const shown = display && display.length > 0 ? display : page;
|
|
295
|
+
return {
|
|
296
|
+
page: canonicalTitle(page),
|
|
297
|
+
shown
|
|
298
|
+
};
|
|
299
|
+
}).filter((x) => x !== null).sort((a, b) => b.shown.length - a.shown.length);
|
|
300
|
+
for (const { page, shown } of replacements) {
|
|
301
|
+
if (!result.includes(shown)) continue;
|
|
302
|
+
const replacement = shown === page ? `[[${page}]]` : `[[${page}|${shown}]]`;
|
|
303
|
+
result = result.replace(shown, replacement);
|
|
304
|
+
}
|
|
305
|
+
return result;
|
|
306
|
+
}
|
|
307
|
+
function lineText(line) {
|
|
308
|
+
if (!line) return "";
|
|
309
|
+
if (typeof line === "string") return line;
|
|
310
|
+
if (typeof line.text === "string") return line.text;
|
|
311
|
+
if (typeof line.text === "function") return line.text();
|
|
312
|
+
return "";
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
//#endregion
|
|
316
|
+
//#region packages/wiki/src/render.ts
|
|
317
|
+
const ICONS = {
|
|
318
|
+
graph: "git-fork",
|
|
319
|
+
article: "book-open",
|
|
320
|
+
category: "tag",
|
|
321
|
+
infobox: "info",
|
|
322
|
+
outline: "list",
|
|
323
|
+
gallery: "images",
|
|
324
|
+
section: "pilcrow",
|
|
325
|
+
categories: "tags"
|
|
326
|
+
};
|
|
327
|
+
/** Decide a page type for a section based on its shape. */
|
|
328
|
+
function pickSectionType(section) {
|
|
329
|
+
if (section.children.length > 0) return {
|
|
330
|
+
type: "outline",
|
|
331
|
+
icon: ICONS.outline
|
|
332
|
+
};
|
|
333
|
+
if (section.isList && section.listLength >= 5) return {
|
|
334
|
+
type: "outline",
|
|
335
|
+
icon: ICONS.outline
|
|
336
|
+
};
|
|
337
|
+
return {
|
|
338
|
+
type: "doc",
|
|
339
|
+
icon: ICONS.section
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
/** Render the lead paragraph as the article-doc body. */
|
|
343
|
+
function renderArticleLead(article) {
|
|
344
|
+
return article.lead ?? "";
|
|
345
|
+
}
|
|
346
|
+
/** Render the article as a single doc, sections + infobox inlined. */
|
|
347
|
+
function renderArticleSingleDoc(article) {
|
|
348
|
+
const parts = [];
|
|
349
|
+
if (article.lead) parts.push(article.lead);
|
|
350
|
+
if (article.infobox && article.infobox.length > 0) parts.push("## Infobox", renderInfoboxBody(article.infobox));
|
|
351
|
+
for (const section of article.sections) parts.push(...renderSectionInline(section, 2));
|
|
352
|
+
return parts.join("\n\n");
|
|
353
|
+
}
|
|
354
|
+
function renderSectionInline(section, level) {
|
|
355
|
+
const out = [];
|
|
356
|
+
const prefix = "#".repeat(Math.min(6, level));
|
|
357
|
+
if (section.title) out.push(`${prefix} ${section.title}`);
|
|
358
|
+
if (section.body.trim()) out.push(section.body);
|
|
359
|
+
for (const child of section.children) out.push(...renderSectionInline(child, level + 1));
|
|
360
|
+
return out;
|
|
361
|
+
}
|
|
362
|
+
function renderInfoboxBody(rows) {
|
|
363
|
+
return rows.map((r) => `- **${r.key}:** ${r.value}`).join("\n");
|
|
364
|
+
}
|
|
365
|
+
function renderCategoryBody(members, subcategories) {
|
|
366
|
+
const parts = [];
|
|
367
|
+
if (members.length > 0) {
|
|
368
|
+
parts.push("## Pages");
|
|
369
|
+
parts.push(members.map((m) => `- [[${m}]]`).join("\n"));
|
|
370
|
+
}
|
|
371
|
+
if (subcategories.length > 0) {
|
|
372
|
+
parts.push("## Sub-categories");
|
|
373
|
+
parts.push(subcategories.map((s) => `- ${s}`).join("\n"));
|
|
374
|
+
}
|
|
375
|
+
return parts.join("\n\n");
|
|
376
|
+
}
|
|
377
|
+
/**
|
|
378
|
+
* Replace `[[Title]]` / `[[Title|Alias]]` in markdown with
|
|
379
|
+
* `[[docId|label]]` using the title→docId map. Unresolved titles fall
|
|
380
|
+
* back to plain text (their alias or original title).
|
|
381
|
+
*/
|
|
382
|
+
function rewriteLinks(markdown, titleToDocId) {
|
|
383
|
+
return markdown.replace(/\[\[([^\]|]+?)(?:\|([^\]]+?))?\]\]/g, (_match, target, alias) => {
|
|
384
|
+
const title = target.trim();
|
|
385
|
+
const docId = titleToDocId.get(title);
|
|
386
|
+
const display = (alias && alias.trim().length > 0 ? alias : title).trim();
|
|
387
|
+
if (!docId) return display;
|
|
388
|
+
return `[[${docId}|${display}]]`;
|
|
389
|
+
});
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
//#endregion
|
|
393
|
+
//#region node_modules/@noble/hashes/utils.js
|
|
394
|
+
/**
|
|
395
|
+
* Checks if something is Uint8Array. Be careful: nodejs Buffer will return true.
|
|
396
|
+
* @param a - value to test
|
|
397
|
+
* @returns `true` when the value is a Uint8Array-compatible view.
|
|
398
|
+
* @example
|
|
399
|
+
* Check whether a value is a Uint8Array-compatible view.
|
|
400
|
+
* ```ts
|
|
401
|
+
* isBytes(new Uint8Array([1, 2, 3]));
|
|
402
|
+
* ```
|
|
403
|
+
*/
|
|
404
|
+
function isBytes(a) {
|
|
405
|
+
return a instanceof Uint8Array || ArrayBuffer.isView(a) && a.constructor.name === "Uint8Array" && "BYTES_PER_ELEMENT" in a && a.BYTES_PER_ELEMENT === 1;
|
|
406
|
+
}
|
|
407
|
+
/**
|
|
408
|
+
* Asserts something is Uint8Array.
|
|
409
|
+
* @param value - value to validate
|
|
410
|
+
* @param length - optional exact length constraint
|
|
411
|
+
* @param title - label included in thrown errors
|
|
412
|
+
* @returns The validated byte array.
|
|
413
|
+
* @throws On wrong argument types. {@link TypeError}
|
|
414
|
+
* @throws On wrong argument ranges or values. {@link RangeError}
|
|
415
|
+
* @example
|
|
416
|
+
* Validate that a value is a byte array.
|
|
417
|
+
* ```ts
|
|
418
|
+
* abytes(new Uint8Array([1, 2, 3]));
|
|
419
|
+
* ```
|
|
420
|
+
*/
|
|
421
|
+
function abytes(value, length, title = "") {
|
|
422
|
+
const bytes = isBytes(value);
|
|
423
|
+
const len = value?.length;
|
|
424
|
+
const needsLen = length !== void 0;
|
|
425
|
+
if (!bytes || needsLen && len !== length) {
|
|
426
|
+
const prefix = title && `"${title}" `;
|
|
427
|
+
const ofLen = needsLen ? ` of length ${length}` : "";
|
|
428
|
+
const got = bytes ? `length=${len}` : `type=${typeof value}`;
|
|
429
|
+
const message = prefix + "expected Uint8Array" + ofLen + ", got " + got;
|
|
430
|
+
if (!bytes) throw new TypeError(message);
|
|
431
|
+
throw new RangeError(message);
|
|
432
|
+
}
|
|
433
|
+
return value;
|
|
434
|
+
}
|
|
435
|
+
/**
|
|
436
|
+
* Asserts a hash instance has not been destroyed or finished.
|
|
437
|
+
* @param instance - hash instance to validate
|
|
438
|
+
* @param checkFinished - whether to reject finalized instances
|
|
439
|
+
* @throws If the hash instance has already been destroyed or finalized. {@link Error}
|
|
440
|
+
* @example
|
|
441
|
+
* Validate that a hash instance is still usable.
|
|
442
|
+
* ```ts
|
|
443
|
+
* import { aexists } from '@noble/hashes/utils.js';
|
|
444
|
+
* import { sha256 } from '@noble/hashes/sha2.js';
|
|
445
|
+
* const hash = sha256.create();
|
|
446
|
+
* aexists(hash);
|
|
447
|
+
* ```
|
|
448
|
+
*/
|
|
449
|
+
function aexists(instance, checkFinished = true) {
|
|
450
|
+
if (instance.destroyed) throw new Error("Hash instance has been destroyed");
|
|
451
|
+
if (checkFinished && instance.finished) throw new Error("Hash#digest() has already been called");
|
|
452
|
+
}
|
|
453
|
+
/**
|
|
454
|
+
* Asserts output is a sufficiently-sized byte array.
|
|
455
|
+
* @param out - destination buffer
|
|
456
|
+
* @param instance - hash instance providing output length
|
|
457
|
+
* Oversized buffers are allowed; downstream code only promises to fill the first `outputLen` bytes.
|
|
458
|
+
* @throws On wrong argument types. {@link TypeError}
|
|
459
|
+
* @throws On wrong argument ranges or values. {@link RangeError}
|
|
460
|
+
* @example
|
|
461
|
+
* Validate a caller-provided digest buffer.
|
|
462
|
+
* ```ts
|
|
463
|
+
* import { aoutput } from '@noble/hashes/utils.js';
|
|
464
|
+
* import { sha256 } from '@noble/hashes/sha2.js';
|
|
465
|
+
* const hash = sha256.create();
|
|
466
|
+
* aoutput(new Uint8Array(hash.outputLen), hash);
|
|
467
|
+
* ```
|
|
468
|
+
*/
|
|
469
|
+
function aoutput(out, instance) {
|
|
470
|
+
abytes(out, void 0, "digestInto() output");
|
|
471
|
+
const min = instance.outputLen;
|
|
472
|
+
if (out.length < min) throw new RangeError("\"digestInto() output\" expected to be of length >=" + min);
|
|
473
|
+
}
|
|
474
|
+
/**
|
|
475
|
+
* Zeroizes typed arrays in place. Warning: JS provides no guarantees.
|
|
476
|
+
* @param arrays - arrays to overwrite with zeros
|
|
477
|
+
* @example
|
|
478
|
+
* Zeroize sensitive buffers in place.
|
|
479
|
+
* ```ts
|
|
480
|
+
* clean(new Uint8Array([1, 2, 3]));
|
|
481
|
+
* ```
|
|
482
|
+
*/
|
|
483
|
+
function clean(...arrays) {
|
|
484
|
+
for (let i = 0; i < arrays.length; i++) arrays[i].fill(0);
|
|
485
|
+
}
|
|
486
|
+
/**
|
|
487
|
+
* Creates a DataView for byte-level manipulation.
|
|
488
|
+
* @param arr - source typed array
|
|
489
|
+
* @returns DataView over the same buffer region.
|
|
490
|
+
* @example
|
|
491
|
+
* Create a DataView over an existing buffer.
|
|
492
|
+
* ```ts
|
|
493
|
+
* createView(new Uint8Array(4));
|
|
494
|
+
* ```
|
|
495
|
+
*/
|
|
496
|
+
function createView(arr) {
|
|
497
|
+
return new DataView(arr.buffer, arr.byteOffset, arr.byteLength);
|
|
498
|
+
}
|
|
499
|
+
/** Whether the current platform is little-endian. */
|
|
500
|
+
const isLE = new Uint8Array(new Uint32Array([287454020]).buffer)[0] === 68;
|
|
501
|
+
const hasHexBuiltin = typeof Uint8Array.from([]).toHex === "function" && typeof Uint8Array.fromHex === "function";
|
|
502
|
+
/**
|
|
503
|
+
* Creates a callable hash function from a stateful class constructor.
|
|
504
|
+
* @param hashCons - hash constructor or factory
|
|
505
|
+
* @param info - optional metadata such as DER OID
|
|
506
|
+
* @returns Frozen callable hash wrapper with `.create()`.
|
|
507
|
+
* Wrapper construction eagerly calls `hashCons(undefined)` once to read
|
|
508
|
+
* `outputLen` / `blockLen`, so constructor side effects happen at module
|
|
509
|
+
* init time.
|
|
510
|
+
* @example
|
|
511
|
+
* Wrap a stateful hash constructor into a callable helper.
|
|
512
|
+
* ```ts
|
|
513
|
+
* import { createHasher } from '@noble/hashes/utils.js';
|
|
514
|
+
* import { sha256 } from '@noble/hashes/sha2.js';
|
|
515
|
+
* const wrapped = createHasher(sha256.create, { oid: sha256.oid });
|
|
516
|
+
* wrapped(new Uint8Array([1]));
|
|
517
|
+
* ```
|
|
518
|
+
*/
|
|
519
|
+
function createHasher(hashCons, info = {}) {
|
|
520
|
+
const hashC = (msg, opts) => hashCons(opts).update(msg).digest();
|
|
521
|
+
const tmp = hashCons(void 0);
|
|
522
|
+
hashC.outputLen = tmp.outputLen;
|
|
523
|
+
hashC.blockLen = tmp.blockLen;
|
|
524
|
+
hashC.canXOF = tmp.canXOF;
|
|
525
|
+
hashC.create = (opts) => hashCons(opts);
|
|
526
|
+
Object.assign(hashC, info);
|
|
527
|
+
return Object.freeze(hashC);
|
|
528
|
+
}
|
|
529
|
+
/**
|
|
530
|
+
* Creates OID metadata for NIST hashes with prefix `06 09 60 86 48 01 65 03 04 02`.
|
|
531
|
+
* @param suffix - final OID byte for the selected hash.
|
|
532
|
+
* The helper accepts any byte even though only the documented NIST hash
|
|
533
|
+
* suffixes are meaningful downstream.
|
|
534
|
+
* @returns Object containing the DER-encoded OID.
|
|
535
|
+
* @example
|
|
536
|
+
* Build OID metadata for a NIST hash.
|
|
537
|
+
* ```ts
|
|
538
|
+
* oidNist(0x01);
|
|
539
|
+
* ```
|
|
540
|
+
*/
|
|
541
|
+
const oidNist = (suffix) => ({ oid: Uint8Array.from([
|
|
542
|
+
6,
|
|
543
|
+
9,
|
|
544
|
+
96,
|
|
545
|
+
134,
|
|
546
|
+
72,
|
|
547
|
+
1,
|
|
548
|
+
101,
|
|
549
|
+
3,
|
|
550
|
+
4,
|
|
551
|
+
2,
|
|
552
|
+
suffix
|
|
553
|
+
]) });
|
|
554
|
+
|
|
555
|
+
//#endregion
|
|
556
|
+
//#region node_modules/@noble/hashes/_md.js
|
|
557
|
+
/**
|
|
558
|
+
* Internal Merkle-Damgard hash utils.
|
|
559
|
+
* @module
|
|
560
|
+
*/
|
|
561
|
+
/**
|
|
562
|
+
* Merkle-Damgard hash construction base class.
|
|
563
|
+
* Could be used to create MD5, RIPEMD, SHA1, SHA2.
|
|
564
|
+
* Accepts only byte-aligned `Uint8Array` input, even when the underlying spec describes bit
|
|
565
|
+
* strings with partial-byte tails.
|
|
566
|
+
* @param blockLen - internal block size in bytes
|
|
567
|
+
* @param outputLen - digest size in bytes
|
|
568
|
+
* @param padOffset - trailing length field size in bytes
|
|
569
|
+
* @param isLE - whether length and state words are encoded in little-endian
|
|
570
|
+
* @example
|
|
571
|
+
* Use a concrete subclass to get the shared Merkle-Damgard update/digest flow.
|
|
572
|
+
* ```ts
|
|
573
|
+
* import { _SHA1 } from '@noble/hashes/legacy.js';
|
|
574
|
+
* const hash = new _SHA1();
|
|
575
|
+
* hash.update(new Uint8Array([97, 98, 99]));
|
|
576
|
+
* hash.digest();
|
|
577
|
+
* ```
|
|
578
|
+
*/
|
|
579
|
+
var HashMD = class {
|
|
580
|
+
blockLen;
|
|
581
|
+
outputLen;
|
|
582
|
+
canXOF = false;
|
|
583
|
+
padOffset;
|
|
584
|
+
isLE;
|
|
585
|
+
buffer;
|
|
586
|
+
view;
|
|
587
|
+
finished = false;
|
|
588
|
+
length = 0;
|
|
589
|
+
pos = 0;
|
|
590
|
+
destroyed = false;
|
|
591
|
+
constructor(blockLen, outputLen, padOffset, isLE) {
|
|
592
|
+
this.blockLen = blockLen;
|
|
593
|
+
this.outputLen = outputLen;
|
|
594
|
+
this.padOffset = padOffset;
|
|
595
|
+
this.isLE = isLE;
|
|
596
|
+
this.buffer = new Uint8Array(blockLen);
|
|
597
|
+
this.view = createView(this.buffer);
|
|
598
|
+
}
|
|
599
|
+
update(data) {
|
|
600
|
+
aexists(this);
|
|
601
|
+
abytes(data);
|
|
602
|
+
const { view, buffer, blockLen } = this;
|
|
603
|
+
const len = data.length;
|
|
604
|
+
for (let pos = 0; pos < len;) {
|
|
605
|
+
const take = Math.min(blockLen - this.pos, len - pos);
|
|
606
|
+
if (take === blockLen) {
|
|
607
|
+
const dataView = createView(data);
|
|
608
|
+
for (; blockLen <= len - pos; pos += blockLen) this.process(dataView, pos);
|
|
609
|
+
continue;
|
|
610
|
+
}
|
|
611
|
+
buffer.set(data.subarray(pos, pos + take), this.pos);
|
|
612
|
+
this.pos += take;
|
|
613
|
+
pos += take;
|
|
614
|
+
if (this.pos === blockLen) {
|
|
615
|
+
this.process(view, 0);
|
|
616
|
+
this.pos = 0;
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
this.length += data.length;
|
|
620
|
+
this.roundClean();
|
|
621
|
+
return this;
|
|
622
|
+
}
|
|
623
|
+
digestInto(out) {
|
|
624
|
+
aexists(this);
|
|
625
|
+
aoutput(out, this);
|
|
626
|
+
this.finished = true;
|
|
627
|
+
const { buffer, view, blockLen, isLE } = this;
|
|
628
|
+
let { pos } = this;
|
|
629
|
+
buffer[pos++] = 128;
|
|
630
|
+
clean(this.buffer.subarray(pos));
|
|
631
|
+
if (this.padOffset > blockLen - pos) {
|
|
632
|
+
this.process(view, 0);
|
|
633
|
+
pos = 0;
|
|
634
|
+
}
|
|
635
|
+
for (let i = pos; i < blockLen; i++) buffer[i] = 0;
|
|
636
|
+
view.setBigUint64(blockLen - 8, BigInt(this.length * 8), isLE);
|
|
637
|
+
this.process(view, 0);
|
|
638
|
+
const oview = createView(out);
|
|
639
|
+
const len = this.outputLen;
|
|
640
|
+
if (len % 4) throw new Error("_sha2: outputLen must be aligned to 32bit");
|
|
641
|
+
const outLen = len / 4;
|
|
642
|
+
const state = this.get();
|
|
643
|
+
if (outLen > state.length) throw new Error("_sha2: outputLen bigger than state");
|
|
644
|
+
for (let i = 0; i < outLen; i++) oview.setUint32(4 * i, state[i], isLE);
|
|
645
|
+
}
|
|
646
|
+
digest() {
|
|
647
|
+
const { buffer, outputLen } = this;
|
|
648
|
+
this.digestInto(buffer);
|
|
649
|
+
const res = buffer.slice(0, outputLen);
|
|
650
|
+
this.destroy();
|
|
651
|
+
return res;
|
|
652
|
+
}
|
|
653
|
+
_cloneInto(to) {
|
|
654
|
+
to ||= new this.constructor();
|
|
655
|
+
to.set(...this.get());
|
|
656
|
+
const { blockLen, buffer, length, finished, destroyed, pos } = this;
|
|
657
|
+
to.destroyed = destroyed;
|
|
658
|
+
to.finished = finished;
|
|
659
|
+
to.length = length;
|
|
660
|
+
to.pos = pos;
|
|
661
|
+
if (length % blockLen) to.buffer.set(buffer);
|
|
662
|
+
return to;
|
|
663
|
+
}
|
|
664
|
+
clone() {
|
|
665
|
+
return this._cloneInto();
|
|
666
|
+
}
|
|
667
|
+
};
|
|
668
|
+
/** Initial SHA512 state from RFC 6234 §6.3: eight RFC 64-bit `H(0)` words stored as sixteen
|
|
669
|
+
* big-endian 32-bit halves. Derived from the fractional parts of the square roots of the first
|
|
670
|
+
* eight prime numbers. Exported as a shared table; callers must treat it as read-only because
|
|
671
|
+
* constructors copy halves from it by index. */
|
|
672
|
+
const SHA512_IV = /* @__PURE__ */ Uint32Array.from([
|
|
673
|
+
1779033703,
|
|
674
|
+
4089235720,
|
|
675
|
+
3144134277,
|
|
676
|
+
2227873595,
|
|
677
|
+
1013904242,
|
|
678
|
+
4271175723,
|
|
679
|
+
2773480762,
|
|
680
|
+
1595750129,
|
|
681
|
+
1359893119,
|
|
682
|
+
2917565137,
|
|
683
|
+
2600822924,
|
|
684
|
+
725511199,
|
|
685
|
+
528734635,
|
|
686
|
+
4215389547,
|
|
687
|
+
1541459225,
|
|
688
|
+
327033209
|
|
689
|
+
]);
|
|
690
|
+
|
|
691
|
+
//#endregion
|
|
692
|
+
//#region node_modules/@noble/hashes/_u64.js
|
|
693
|
+
const U32_MASK64 = /* @__PURE__ */ BigInt(2 ** 32 - 1);
|
|
694
|
+
const _32n = /* @__PURE__ */ BigInt(32);
|
|
695
|
+
function fromBig(n, le = false) {
|
|
696
|
+
if (le) return {
|
|
697
|
+
h: Number(n & U32_MASK64),
|
|
698
|
+
l: Number(n >> _32n & U32_MASK64)
|
|
699
|
+
};
|
|
700
|
+
return {
|
|
701
|
+
h: Number(n >> _32n & U32_MASK64) | 0,
|
|
702
|
+
l: Number(n & U32_MASK64) | 0
|
|
703
|
+
};
|
|
704
|
+
}
|
|
705
|
+
function split(lst, le = false) {
|
|
706
|
+
const len = lst.length;
|
|
707
|
+
let Ah = new Uint32Array(len);
|
|
708
|
+
let Al = new Uint32Array(len);
|
|
709
|
+
for (let i = 0; i < len; i++) {
|
|
710
|
+
const { h, l } = fromBig(lst[i], le);
|
|
711
|
+
[Ah[i], Al[i]] = [h, l];
|
|
712
|
+
}
|
|
713
|
+
return [Ah, Al];
|
|
714
|
+
}
|
|
715
|
+
const shrSH = (h, _l, s) => h >>> s;
|
|
716
|
+
const shrSL = (h, l, s) => h << 32 - s | l >>> s;
|
|
717
|
+
const rotrSH = (h, l, s) => h >>> s | l << 32 - s;
|
|
718
|
+
const rotrSL = (h, l, s) => h << 32 - s | l >>> s;
|
|
719
|
+
const rotrBH = (h, l, s) => h << 64 - s | l >>> s - 32;
|
|
720
|
+
const rotrBL = (h, l, s) => h >>> s - 32 | l << 64 - s;
|
|
721
|
+
function add(Ah, Al, Bh, Bl) {
|
|
722
|
+
const l = (Al >>> 0) + (Bl >>> 0);
|
|
723
|
+
return {
|
|
724
|
+
h: Ah + Bh + (l / 2 ** 32 | 0) | 0,
|
|
725
|
+
l: l | 0
|
|
726
|
+
};
|
|
727
|
+
}
|
|
728
|
+
const add3L = (Al, Bl, Cl) => (Al >>> 0) + (Bl >>> 0) + (Cl >>> 0);
|
|
729
|
+
const add3H = (low, Ah, Bh, Ch) => Ah + Bh + Ch + (low / 2 ** 32 | 0) | 0;
|
|
730
|
+
const add4L = (Al, Bl, Cl, Dl) => (Al >>> 0) + (Bl >>> 0) + (Cl >>> 0) + (Dl >>> 0);
|
|
731
|
+
const add4H = (low, Ah, Bh, Ch, Dh) => Ah + Bh + Ch + Dh + (low / 2 ** 32 | 0) | 0;
|
|
732
|
+
const add5L = (Al, Bl, Cl, Dl, El) => (Al >>> 0) + (Bl >>> 0) + (Cl >>> 0) + (Dl >>> 0) + (El >>> 0);
|
|
733
|
+
const add5H = (low, Ah, Bh, Ch, Dh, Eh) => Ah + Bh + Ch + Dh + Eh + (low / 2 ** 32 | 0) | 0;
|
|
734
|
+
|
|
735
|
+
//#endregion
|
|
736
|
+
//#region node_modules/@noble/hashes/sha2.js
|
|
737
|
+
/**
|
|
738
|
+
* SHA2 hash function. A.k.a. sha256, sha384, sha512, sha512_224, sha512_256.
|
|
739
|
+
* SHA256 is the fastest hash implementable in JS, even faster than Blake3.
|
|
740
|
+
* Check out {@link https://www.rfc-editor.org/rfc/rfc4634 | RFC 4634} and
|
|
741
|
+
* {@link https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.180-4.pdf | FIPS 180-4}.
|
|
742
|
+
* @module
|
|
743
|
+
*/
|
|
744
|
+
const K512 = split([
|
|
745
|
+
"0x428a2f98d728ae22",
|
|
746
|
+
"0x7137449123ef65cd",
|
|
747
|
+
"0xb5c0fbcfec4d3b2f",
|
|
748
|
+
"0xe9b5dba58189dbbc",
|
|
749
|
+
"0x3956c25bf348b538",
|
|
750
|
+
"0x59f111f1b605d019",
|
|
751
|
+
"0x923f82a4af194f9b",
|
|
752
|
+
"0xab1c5ed5da6d8118",
|
|
753
|
+
"0xd807aa98a3030242",
|
|
754
|
+
"0x12835b0145706fbe",
|
|
755
|
+
"0x243185be4ee4b28c",
|
|
756
|
+
"0x550c7dc3d5ffb4e2",
|
|
757
|
+
"0x72be5d74f27b896f",
|
|
758
|
+
"0x80deb1fe3b1696b1",
|
|
759
|
+
"0x9bdc06a725c71235",
|
|
760
|
+
"0xc19bf174cf692694",
|
|
761
|
+
"0xe49b69c19ef14ad2",
|
|
762
|
+
"0xefbe4786384f25e3",
|
|
763
|
+
"0x0fc19dc68b8cd5b5",
|
|
764
|
+
"0x240ca1cc77ac9c65",
|
|
765
|
+
"0x2de92c6f592b0275",
|
|
766
|
+
"0x4a7484aa6ea6e483",
|
|
767
|
+
"0x5cb0a9dcbd41fbd4",
|
|
768
|
+
"0x76f988da831153b5",
|
|
769
|
+
"0x983e5152ee66dfab",
|
|
770
|
+
"0xa831c66d2db43210",
|
|
771
|
+
"0xb00327c898fb213f",
|
|
772
|
+
"0xbf597fc7beef0ee4",
|
|
773
|
+
"0xc6e00bf33da88fc2",
|
|
774
|
+
"0xd5a79147930aa725",
|
|
775
|
+
"0x06ca6351e003826f",
|
|
776
|
+
"0x142929670a0e6e70",
|
|
777
|
+
"0x27b70a8546d22ffc",
|
|
778
|
+
"0x2e1b21385c26c926",
|
|
779
|
+
"0x4d2c6dfc5ac42aed",
|
|
780
|
+
"0x53380d139d95b3df",
|
|
781
|
+
"0x650a73548baf63de",
|
|
782
|
+
"0x766a0abb3c77b2a8",
|
|
783
|
+
"0x81c2c92e47edaee6",
|
|
784
|
+
"0x92722c851482353b",
|
|
785
|
+
"0xa2bfe8a14cf10364",
|
|
786
|
+
"0xa81a664bbc423001",
|
|
787
|
+
"0xc24b8b70d0f89791",
|
|
788
|
+
"0xc76c51a30654be30",
|
|
789
|
+
"0xd192e819d6ef5218",
|
|
790
|
+
"0xd69906245565a910",
|
|
791
|
+
"0xf40e35855771202a",
|
|
792
|
+
"0x106aa07032bbd1b8",
|
|
793
|
+
"0x19a4c116b8d2d0c8",
|
|
794
|
+
"0x1e376c085141ab53",
|
|
795
|
+
"0x2748774cdf8eeb99",
|
|
796
|
+
"0x34b0bcb5e19b48a8",
|
|
797
|
+
"0x391c0cb3c5c95a63",
|
|
798
|
+
"0x4ed8aa4ae3418acb",
|
|
799
|
+
"0x5b9cca4f7763e373",
|
|
800
|
+
"0x682e6ff3d6b2b8a3",
|
|
801
|
+
"0x748f82ee5defb2fc",
|
|
802
|
+
"0x78a5636f43172f60",
|
|
803
|
+
"0x84c87814a1f0ab72",
|
|
804
|
+
"0x8cc702081a6439ec",
|
|
805
|
+
"0x90befffa23631e28",
|
|
806
|
+
"0xa4506cebde82bde9",
|
|
807
|
+
"0xbef9a3f7b2c67915",
|
|
808
|
+
"0xc67178f2e372532b",
|
|
809
|
+
"0xca273eceea26619c",
|
|
810
|
+
"0xd186b8c721c0c207",
|
|
811
|
+
"0xeada7dd6cde0eb1e",
|
|
812
|
+
"0xf57d4f7fee6ed178",
|
|
813
|
+
"0x06f067aa72176fba",
|
|
814
|
+
"0x0a637dc5a2c898a6",
|
|
815
|
+
"0x113f9804bef90dae",
|
|
816
|
+
"0x1b710b35131c471b",
|
|
817
|
+
"0x28db77f523047d84",
|
|
818
|
+
"0x32caab7b40c72493",
|
|
819
|
+
"0x3c9ebe0a15c9bebc",
|
|
820
|
+
"0x431d67c49c100d4c",
|
|
821
|
+
"0x4cc5d4becb3e42b6",
|
|
822
|
+
"0x597f299cfc657e2a",
|
|
823
|
+
"0x5fcb6fab3ad6faec",
|
|
824
|
+
"0x6c44198c4a475817"
|
|
825
|
+
].map((n) => BigInt(n)));
|
|
826
|
+
const SHA512_Kh = K512[0];
|
|
827
|
+
const SHA512_Kl = K512[1];
|
|
828
|
+
const SHA512_W_H = /* @__PURE__ */ new Uint32Array(80);
|
|
829
|
+
const SHA512_W_L = /* @__PURE__ */ new Uint32Array(80);
|
|
830
|
+
/** Internal SHA-384 / SHA-512 compression engine from RFC 6234 §6.4. */
|
|
831
|
+
var SHA2_64B = class extends HashMD {
|
|
832
|
+
constructor(outputLen) {
|
|
833
|
+
super(128, outputLen, 16, false);
|
|
834
|
+
}
|
|
835
|
+
get() {
|
|
836
|
+
const { Ah, Al, Bh, Bl, Ch, Cl, Dh, Dl, Eh, El, Fh, Fl, Gh, Gl, Hh, Hl } = this;
|
|
837
|
+
return [
|
|
838
|
+
Ah,
|
|
839
|
+
Al,
|
|
840
|
+
Bh,
|
|
841
|
+
Bl,
|
|
842
|
+
Ch,
|
|
843
|
+
Cl,
|
|
844
|
+
Dh,
|
|
845
|
+
Dl,
|
|
846
|
+
Eh,
|
|
847
|
+
El,
|
|
848
|
+
Fh,
|
|
849
|
+
Fl,
|
|
850
|
+
Gh,
|
|
851
|
+
Gl,
|
|
852
|
+
Hh,
|
|
853
|
+
Hl
|
|
854
|
+
];
|
|
855
|
+
}
|
|
856
|
+
set(Ah, Al, Bh, Bl, Ch, Cl, Dh, Dl, Eh, El, Fh, Fl, Gh, Gl, Hh, Hl) {
|
|
857
|
+
this.Ah = Ah | 0;
|
|
858
|
+
this.Al = Al | 0;
|
|
859
|
+
this.Bh = Bh | 0;
|
|
860
|
+
this.Bl = Bl | 0;
|
|
861
|
+
this.Ch = Ch | 0;
|
|
862
|
+
this.Cl = Cl | 0;
|
|
863
|
+
this.Dh = Dh | 0;
|
|
864
|
+
this.Dl = Dl | 0;
|
|
865
|
+
this.Eh = Eh | 0;
|
|
866
|
+
this.El = El | 0;
|
|
867
|
+
this.Fh = Fh | 0;
|
|
868
|
+
this.Fl = Fl | 0;
|
|
869
|
+
this.Gh = Gh | 0;
|
|
870
|
+
this.Gl = Gl | 0;
|
|
871
|
+
this.Hh = Hh | 0;
|
|
872
|
+
this.Hl = Hl | 0;
|
|
873
|
+
}
|
|
874
|
+
process(view, offset) {
|
|
875
|
+
for (let i = 0; i < 16; i++, offset += 4) {
|
|
876
|
+
SHA512_W_H[i] = view.getUint32(offset);
|
|
877
|
+
SHA512_W_L[i] = view.getUint32(offset += 4);
|
|
878
|
+
}
|
|
879
|
+
for (let i = 16; i < 80; i++) {
|
|
880
|
+
const W15h = SHA512_W_H[i - 15] | 0;
|
|
881
|
+
const W15l = SHA512_W_L[i - 15] | 0;
|
|
882
|
+
const s0h = rotrSH(W15h, W15l, 1) ^ rotrSH(W15h, W15l, 8) ^ shrSH(W15h, W15l, 7);
|
|
883
|
+
const s0l = rotrSL(W15h, W15l, 1) ^ rotrSL(W15h, W15l, 8) ^ shrSL(W15h, W15l, 7);
|
|
884
|
+
const W2h = SHA512_W_H[i - 2] | 0;
|
|
885
|
+
const W2l = SHA512_W_L[i - 2] | 0;
|
|
886
|
+
const s1h = rotrSH(W2h, W2l, 19) ^ rotrBH(W2h, W2l, 61) ^ shrSH(W2h, W2l, 6);
|
|
887
|
+
const s1l = rotrSL(W2h, W2l, 19) ^ rotrBL(W2h, W2l, 61) ^ shrSL(W2h, W2l, 6);
|
|
888
|
+
const SUMl = add4L(s0l, s1l, SHA512_W_L[i - 7], SHA512_W_L[i - 16]);
|
|
889
|
+
SHA512_W_H[i] = add4H(SUMl, s0h, s1h, SHA512_W_H[i - 7], SHA512_W_H[i - 16]) | 0;
|
|
890
|
+
SHA512_W_L[i] = SUMl | 0;
|
|
891
|
+
}
|
|
892
|
+
let { Ah, Al, Bh, Bl, Ch, Cl, Dh, Dl, Eh, El, Fh, Fl, Gh, Gl, Hh, Hl } = this;
|
|
893
|
+
for (let i = 0; i < 80; i++) {
|
|
894
|
+
const sigma1h = rotrSH(Eh, El, 14) ^ rotrSH(Eh, El, 18) ^ rotrBH(Eh, El, 41);
|
|
895
|
+
const sigma1l = rotrSL(Eh, El, 14) ^ rotrSL(Eh, El, 18) ^ rotrBL(Eh, El, 41);
|
|
896
|
+
const CHIh = Eh & Fh ^ ~Eh & Gh;
|
|
897
|
+
const CHIl = El & Fl ^ ~El & Gl;
|
|
898
|
+
const T1ll = add5L(Hl, sigma1l, CHIl, SHA512_Kl[i], SHA512_W_L[i]);
|
|
899
|
+
const T1h = add5H(T1ll, Hh, sigma1h, CHIh, SHA512_Kh[i], SHA512_W_H[i]);
|
|
900
|
+
const T1l = T1ll | 0;
|
|
901
|
+
const sigma0h = rotrSH(Ah, Al, 28) ^ rotrBH(Ah, Al, 34) ^ rotrBH(Ah, Al, 39);
|
|
902
|
+
const sigma0l = rotrSL(Ah, Al, 28) ^ rotrBL(Ah, Al, 34) ^ rotrBL(Ah, Al, 39);
|
|
903
|
+
const MAJh = Ah & Bh ^ Ah & Ch ^ Bh & Ch;
|
|
904
|
+
const MAJl = Al & Bl ^ Al & Cl ^ Bl & Cl;
|
|
905
|
+
Hh = Gh | 0;
|
|
906
|
+
Hl = Gl | 0;
|
|
907
|
+
Gh = Fh | 0;
|
|
908
|
+
Gl = Fl | 0;
|
|
909
|
+
Fh = Eh | 0;
|
|
910
|
+
Fl = El | 0;
|
|
911
|
+
({h: Eh, l: El} = add(Dh | 0, Dl | 0, T1h | 0, T1l | 0));
|
|
912
|
+
Dh = Ch | 0;
|
|
913
|
+
Dl = Cl | 0;
|
|
914
|
+
Ch = Bh | 0;
|
|
915
|
+
Cl = Bl | 0;
|
|
916
|
+
Bh = Ah | 0;
|
|
917
|
+
Bl = Al | 0;
|
|
918
|
+
const All = add3L(T1l, sigma0l, MAJl);
|
|
919
|
+
Ah = add3H(All, T1h, sigma0h, MAJh);
|
|
920
|
+
Al = All | 0;
|
|
921
|
+
}
|
|
922
|
+
({h: Ah, l: Al} = add(this.Ah | 0, this.Al | 0, Ah | 0, Al | 0));
|
|
923
|
+
({h: Bh, l: Bl} = add(this.Bh | 0, this.Bl | 0, Bh | 0, Bl | 0));
|
|
924
|
+
({h: Ch, l: Cl} = add(this.Ch | 0, this.Cl | 0, Ch | 0, Cl | 0));
|
|
925
|
+
({h: Dh, l: Dl} = add(this.Dh | 0, this.Dl | 0, Dh | 0, Dl | 0));
|
|
926
|
+
({h: Eh, l: El} = add(this.Eh | 0, this.El | 0, Eh | 0, El | 0));
|
|
927
|
+
({h: Fh, l: Fl} = add(this.Fh | 0, this.Fl | 0, Fh | 0, Fl | 0));
|
|
928
|
+
({h: Gh, l: Gl} = add(this.Gh | 0, this.Gl | 0, Gh | 0, Gl | 0));
|
|
929
|
+
({h: Hh, l: Hl} = add(this.Hh | 0, this.Hl | 0, Hh | 0, Hl | 0));
|
|
930
|
+
this.set(Ah, Al, Bh, Bl, Ch, Cl, Dh, Dl, Eh, El, Fh, Fl, Gh, Gl, Hh, Hl);
|
|
931
|
+
}
|
|
932
|
+
roundClean() {
|
|
933
|
+
clean(SHA512_W_H, SHA512_W_L);
|
|
934
|
+
}
|
|
935
|
+
destroy() {
|
|
936
|
+
this.destroyed = true;
|
|
937
|
+
clean(this.buffer);
|
|
938
|
+
this.set(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
|
|
939
|
+
}
|
|
940
|
+
};
|
|
941
|
+
/** Internal SHA-512 hash class grounded in RFC 6234 §6.3 and §6.4. */
|
|
942
|
+
var _SHA512 = class extends SHA2_64B {
|
|
943
|
+
Ah = SHA512_IV[0] | 0;
|
|
944
|
+
Al = SHA512_IV[1] | 0;
|
|
945
|
+
Bh = SHA512_IV[2] | 0;
|
|
946
|
+
Bl = SHA512_IV[3] | 0;
|
|
947
|
+
Ch = SHA512_IV[4] | 0;
|
|
948
|
+
Cl = SHA512_IV[5] | 0;
|
|
949
|
+
Dh = SHA512_IV[6] | 0;
|
|
950
|
+
Dl = SHA512_IV[7] | 0;
|
|
951
|
+
Eh = SHA512_IV[8] | 0;
|
|
952
|
+
El = SHA512_IV[9] | 0;
|
|
953
|
+
Fh = SHA512_IV[10] | 0;
|
|
954
|
+
Fl = SHA512_IV[11] | 0;
|
|
955
|
+
Gh = SHA512_IV[12] | 0;
|
|
956
|
+
Gl = SHA512_IV[13] | 0;
|
|
957
|
+
Hh = SHA512_IV[14] | 0;
|
|
958
|
+
Hl = SHA512_IV[15] | 0;
|
|
959
|
+
constructor() {
|
|
960
|
+
super(64);
|
|
961
|
+
}
|
|
962
|
+
};
|
|
963
|
+
/**
|
|
964
|
+
* SHA2-512 hash function from RFC 4634.
|
|
965
|
+
* @param msg - message bytes to hash
|
|
966
|
+
* @returns Digest bytes.
|
|
967
|
+
* @example
|
|
968
|
+
* Hash a message with SHA2-512.
|
|
969
|
+
* ```ts
|
|
970
|
+
* sha512(new Uint8Array([97, 98, 99]));
|
|
971
|
+
* ```
|
|
972
|
+
*/
|
|
973
|
+
const sha512 = /* @__PURE__ */ createHasher(() => new _SHA512(), /* @__PURE__ */ oidNist(3));
|
|
974
|
+
|
|
975
|
+
//#endregion
|
|
976
|
+
//#region packages/wiki/src/crypto.ts
|
|
977
|
+
/**
|
|
978
|
+
* Ed25519 key generation, persistence, and challenge signing for CLI auth.
|
|
979
|
+
* Mirrors @abraca/mcp/src/crypto.ts — standalone to avoid MCP SDK dependency.
|
|
980
|
+
*/
|
|
981
|
+
ed.hashes.sha512 = sha512;
|
|
982
|
+
ed.hashes.sha512Async = (m) => Promise.resolve(sha512(m));
|
|
983
|
+
const DEFAULT_KEY_PATH = join(homedir(), ".abracadabra", "cli.key");
|
|
984
|
+
function toBase64url(bytes) {
|
|
985
|
+
return Buffer.from(bytes).toString("base64url");
|
|
986
|
+
}
|
|
987
|
+
function fromBase64url(b64) {
|
|
988
|
+
return new Uint8Array(Buffer.from(b64, "base64url"));
|
|
989
|
+
}
|
|
990
|
+
/**
|
|
991
|
+
* Load an existing Ed25519 keypair from disk, or generate and persist a new one.
|
|
992
|
+
* The file stores the raw 32-byte private key seed.
|
|
993
|
+
*/
|
|
994
|
+
async function loadOrCreateKeypair(keyPath) {
|
|
995
|
+
const path = keyPath || DEFAULT_KEY_PATH;
|
|
996
|
+
if (existsSync(path)) {
|
|
997
|
+
const seed = await readFile(path);
|
|
998
|
+
if (seed.length !== 32) throw new Error(`Invalid key file at ${path}: expected 32 bytes, got ${seed.length}`);
|
|
999
|
+
const privateKey = new Uint8Array(seed);
|
|
1000
|
+
return {
|
|
1001
|
+
privateKey,
|
|
1002
|
+
publicKeyB64: toBase64url(ed.getPublicKey(privateKey))
|
|
1003
|
+
};
|
|
1004
|
+
}
|
|
1005
|
+
const privateKey = ed.utils.randomSecretKey();
|
|
1006
|
+
const publicKey = ed.getPublicKey(privateKey);
|
|
1007
|
+
const dir = dirname(path);
|
|
1008
|
+
if (!existsSync(dir)) await mkdir(dir, {
|
|
1009
|
+
recursive: true,
|
|
1010
|
+
mode: 448
|
|
1011
|
+
});
|
|
1012
|
+
await writeFile(path, Buffer.from(privateKey), { mode: 384 });
|
|
1013
|
+
console.error(`[abracadabra] Generated new keypair at ${path}`);
|
|
1014
|
+
console.error(`[abracadabra] Public key: ${toBase64url(publicKey)}`);
|
|
1015
|
+
return {
|
|
1016
|
+
privateKey,
|
|
1017
|
+
publicKeyB64: toBase64url(publicKey)
|
|
1018
|
+
};
|
|
1019
|
+
}
|
|
1020
|
+
/** Sign a base64url challenge with the private key; returns base64url signature. */
|
|
1021
|
+
function signChallenge(challengeB64, privateKey) {
|
|
1022
|
+
const challenge = fromBase64url(challengeB64);
|
|
1023
|
+
return toBase64url(ed.sign(challenge, privateKey));
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
//#endregion
|
|
1027
|
+
//#region packages/wiki/src/connect.ts
|
|
1028
|
+
/**
|
|
1029
|
+
* Open a DocumentManager session for the wiki command, mirroring the
|
|
1030
|
+
* auth/register flow that CLIConnection uses but using the modern public API.
|
|
1031
|
+
*
|
|
1032
|
+
* Reuses the CLI's Ed25519 keypair handling (loadOrCreateKeypair, signChallenge)
|
|
1033
|
+
* so the wiki command authenticates with the same identity as every other
|
|
1034
|
+
* subcommand.
|
|
1035
|
+
*/
|
|
1036
|
+
async function openSession(config) {
|
|
1037
|
+
const keypair = await loadOrCreateKeypair(config.keyFile);
|
|
1038
|
+
const sign = (challenge) => Promise.resolve(signChallenge(challenge, keypair.privateKey));
|
|
1039
|
+
const dm = new DocumentManager({
|
|
1040
|
+
url: config.url,
|
|
1041
|
+
name: config.name ?? "Wiki Extractor",
|
|
1042
|
+
color: config.color,
|
|
1043
|
+
quiet: config.quiet
|
|
1044
|
+
});
|
|
1045
|
+
try {
|
|
1046
|
+
await dm.client.loginWithKey(keypair.publicKeyB64, sign);
|
|
1047
|
+
} catch (err) {
|
|
1048
|
+
const status = err?.status ?? err?.response?.status;
|
|
1049
|
+
if (status === 404 || status === 422) {
|
|
1050
|
+
if (!config.quiet) console.error("[abracadabra] Key not registered, creating new account...");
|
|
1051
|
+
await dm.client.registerWithKey({
|
|
1052
|
+
publicKey: keypair.publicKeyB64,
|
|
1053
|
+
username: (config.name ?? "wiki-extractor").replace(/\s+/g, "-").toLowerCase(),
|
|
1054
|
+
displayName: config.name ?? "Wiki Extractor",
|
|
1055
|
+
deviceName: "CLI Wiki",
|
|
1056
|
+
inviteCode: config.inviteCode
|
|
1057
|
+
});
|
|
1058
|
+
await dm.client.loginWithKey(keypair.publicKeyB64, sign);
|
|
1059
|
+
} else throw err;
|
|
1060
|
+
}
|
|
1061
|
+
await dm.connect();
|
|
1062
|
+
const rootDocId = dm.rootDocId;
|
|
1063
|
+
if (!rootDocId) throw new Error("Connected but no rootDocId — server has no spaces.");
|
|
1064
|
+
return {
|
|
1065
|
+
dm,
|
|
1066
|
+
rootDocId
|
|
1067
|
+
};
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
//#endregion
|
|
1071
|
+
//#region packages/wiki/src/index.ts
|
|
1072
|
+
const USAGE = [
|
|
1073
|
+
"abracadabra-wiki \"<Article Title>\" user-agent=\"<name (email)>\" [options]",
|
|
1074
|
+
"",
|
|
1075
|
+
"Options:",
|
|
1076
|
+
" mode=single|split single doc per article OR split into sections+infobox [split]",
|
|
1077
|
+
" depth=<n> follow internal links to depth N [1]",
|
|
1078
|
+
" category-depth=<n> recurse into sub-categories [1]",
|
|
1079
|
+
" lang=<code> wiki language [en]",
|
|
1080
|
+
" domain=<host> 3rd-party MediaWiki host (overrides lang)",
|
|
1081
|
+
" parent=<docId> parent doc for the new graph [active space root]",
|
|
1082
|
+
" user-agent=<str> Api-User-Agent header (REQUIRED by Wikimedia etiquette)",
|
|
1083
|
+
" rate=<rps> max wikipedia requests per second [3]",
|
|
1084
|
+
" --include-categories expand each article's categories into nested graphs",
|
|
1085
|
+
" --dry-run fetch only the entry article, print outline, no writes",
|
|
1086
|
+
"",
|
|
1087
|
+
"Environment: ABRA_URL (required unless --dry-run), ABRA_KEY_FILE, ABRA_NAME,",
|
|
1088
|
+
" ABRA_COLOR, ABRA_INVITE_CODE, ABRA_WIKI_USER_AGENT."
|
|
1089
|
+
].join("\n");
|
|
1090
|
+
/**
|
|
1091
|
+
* Run a Wikipedia import for already-parsed args. Returns a human-readable
|
|
1092
|
+
* summary (or an error/usage string). Exported for programmatic use.
|
|
1093
|
+
*/
|
|
1094
|
+
async function runWiki(args) {
|
|
1095
|
+
const opts = parseOptions(args);
|
|
1096
|
+
if (typeof opts === "string") return opts;
|
|
1097
|
+
const log = (msg) => {
|
|
1098
|
+
if (!args.flags.has("quiet") && !args.flags.has("q")) console.error(`[wiki] ${msg}`);
|
|
1099
|
+
};
|
|
1100
|
+
const wp = new WikipediaClient({
|
|
1101
|
+
lang: opts.lang,
|
|
1102
|
+
domain: opts.domain,
|
|
1103
|
+
userAgent: opts.userAgent,
|
|
1104
|
+
rate: opts.rate
|
|
1105
|
+
});
|
|
1106
|
+
if (opts.dryRun) {
|
|
1107
|
+
log(`fetch ${opts.title}`);
|
|
1108
|
+
const doc = await wp.fetchArticle(opts.title);
|
|
1109
|
+
if (!doc) return `Article not found: "${opts.title}"`;
|
|
1110
|
+
const snap = snapshotArticle(doc, canonicalTitle(doc.title?.() ?? opts.title));
|
|
1111
|
+
return [
|
|
1112
|
+
`Entry: ${snap.title}`,
|
|
1113
|
+
`URL: ${snap.url ?? "(none)"}`,
|
|
1114
|
+
`Internal links: ${snap.linkTitles.length}`,
|
|
1115
|
+
`Categories: ${snap.categories.length}`,
|
|
1116
|
+
`Sections: ${snap.sections.length}`,
|
|
1117
|
+
`Has infobox: ${snap.infobox && snap.infobox.length > 0 ? "yes" : "no"}`,
|
|
1118
|
+
"",
|
|
1119
|
+
"── Sections ──",
|
|
1120
|
+
printSections(snap.sections, "")
|
|
1121
|
+
].join("\n");
|
|
1122
|
+
}
|
|
1123
|
+
const env = globalThis.process?.env ?? {};
|
|
1124
|
+
const url = env["ABRA_URL"];
|
|
1125
|
+
if (!url) return "ABRA_URL is required to write to the server. Set it or pass --dry-run.";
|
|
1126
|
+
const { dm } = await openSession({
|
|
1127
|
+
url,
|
|
1128
|
+
name: env["ABRA_NAME"],
|
|
1129
|
+
color: env["ABRA_COLOR"],
|
|
1130
|
+
inviteCode: env["ABRA_INVITE_CODE"],
|
|
1131
|
+
keyFile: env["ABRA_KEY_FILE"],
|
|
1132
|
+
quiet: args.flags.has("quiet") || args.flags.has("q")
|
|
1133
|
+
});
|
|
1134
|
+
try {
|
|
1135
|
+
const result = await runStreaming(dm, wp, opts, log);
|
|
1136
|
+
return [`Done. Created ${result.articleCount} articles${result.categoryCount > 0 ? ` + ${result.categoryCount} categories` : ""}.`, `Root: ${result.rootDocId}`].join("\n");
|
|
1137
|
+
} finally {
|
|
1138
|
+
await dm.destroy().catch(() => {});
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1141
|
+
async function runStreaming(dm, wp, opts, log) {
|
|
1142
|
+
const titleToDocId = /* @__PURE__ */ new Map();
|
|
1143
|
+
const fetched = /* @__PURE__ */ new Map();
|
|
1144
|
+
const childrenCreated = /* @__PURE__ */ new Set();
|
|
1145
|
+
const categoryToDocId = /* @__PURE__ */ new Map();
|
|
1146
|
+
let categoriesContainerId = null;
|
|
1147
|
+
log(`fetch ${opts.title}`);
|
|
1148
|
+
const entryDoc = await wp.fetchArticle(opts.title);
|
|
1149
|
+
if (!entryDoc) throw new Error(`Article not found: "${opts.title}"`);
|
|
1150
|
+
const entryTitle = canonicalTitle(entryDoc.title?.() ?? opts.title);
|
|
1151
|
+
const entrySnap = snapshotArticle(entryDoc, entryTitle);
|
|
1152
|
+
fetched.set(entryTitle, entrySnap);
|
|
1153
|
+
const rootEntry = dm.tree.create({
|
|
1154
|
+
parentId: opts.parentDocId ?? null,
|
|
1155
|
+
label: entryTitle,
|
|
1156
|
+
type: "graph",
|
|
1157
|
+
meta: { icon: ICONS.graph }
|
|
1158
|
+
});
|
|
1159
|
+
log(`+ ${rootEntry.id.slice(0, 8)}… ${entryTitle} (graph)`);
|
|
1160
|
+
const entryArticleId = createArticleShell(dm, entrySnap, rootEntry.id, log);
|
|
1161
|
+
titleToDocId.set(entryTitle, entryArticleId);
|
|
1162
|
+
const queue = [{
|
|
1163
|
+
title: entryTitle,
|
|
1164
|
+
depth: 0
|
|
1165
|
+
}];
|
|
1166
|
+
let articleCount = 0;
|
|
1167
|
+
while (queue.length > 0) {
|
|
1168
|
+
const { title, depth } = queue.shift();
|
|
1169
|
+
const articleDocId = titleToDocId.get(title);
|
|
1170
|
+
let snap = fetched.get(title);
|
|
1171
|
+
if (!snap) {
|
|
1172
|
+
log(`fetch [d${depth}] ${title}`);
|
|
1173
|
+
try {
|
|
1174
|
+
const doc = await wp.fetchArticle(title);
|
|
1175
|
+
if (!doc) {
|
|
1176
|
+
log(` not found — leaving stub`);
|
|
1177
|
+
continue;
|
|
1178
|
+
}
|
|
1179
|
+
snap = snapshotArticle(doc, canonicalTitle(doc.title?.() ?? title));
|
|
1180
|
+
fetched.set(title, snap);
|
|
1181
|
+
} catch (err) {
|
|
1182
|
+
log(`! fetch failed: ${err?.message ?? err}`);
|
|
1183
|
+
continue;
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
if (opts.mode === "split" && !childrenCreated.has(title)) {
|
|
1187
|
+
createArticleChildren(dm, snap, articleDocId, log);
|
|
1188
|
+
childrenCreated.add(title);
|
|
1189
|
+
}
|
|
1190
|
+
if (depth < opts.depth) for (const linkTitle of snap.linkTitles) {
|
|
1191
|
+
if (titleToDocId.has(linkTitle)) continue;
|
|
1192
|
+
const shell = dm.tree.create({
|
|
1193
|
+
parentId: rootEntry.id,
|
|
1194
|
+
label: linkTitle,
|
|
1195
|
+
type: "doc",
|
|
1196
|
+
meta: { icon: ICONS.article }
|
|
1197
|
+
});
|
|
1198
|
+
titleToDocId.set(linkTitle, shell.id);
|
|
1199
|
+
queue.push({
|
|
1200
|
+
title: linkTitle,
|
|
1201
|
+
depth: depth + 1
|
|
1202
|
+
});
|
|
1203
|
+
log(`+ ${shell.id.slice(0, 8)}… ${linkTitle} (doc, shell)`);
|
|
1204
|
+
}
|
|
1205
|
+
if (opts.includeCategories && snap.categories.length > 0) {
|
|
1206
|
+
if (!categoriesContainerId) {
|
|
1207
|
+
const c = dm.tree.create({
|
|
1208
|
+
parentId: rootEntry.id,
|
|
1209
|
+
label: "Categories",
|
|
1210
|
+
type: "graph",
|
|
1211
|
+
meta: { icon: ICONS.categories }
|
|
1212
|
+
});
|
|
1213
|
+
categoriesContainerId = c.id;
|
|
1214
|
+
log(`+ ${c.id.slice(0, 8)}… Categories (graph)`);
|
|
1215
|
+
}
|
|
1216
|
+
for (const catTitle of snap.categories) {
|
|
1217
|
+
if (categoryToDocId.has(catTitle)) continue;
|
|
1218
|
+
const cat = dm.tree.create({
|
|
1219
|
+
parentId: categoriesContainerId,
|
|
1220
|
+
label: prettyCategoryLabel(catTitle),
|
|
1221
|
+
type: "graph",
|
|
1222
|
+
meta: { icon: ICONS.category }
|
|
1223
|
+
});
|
|
1224
|
+
categoryToDocId.set(catTitle, cat.id);
|
|
1225
|
+
log(`+ ${cat.id.slice(0, 8)}… ${prettyCategoryLabel(catTitle)} (graph, cat)`);
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1228
|
+
const body = opts.mode === "split" ? renderArticleLead(snap) : renderArticleSingleDoc(snap);
|
|
1229
|
+
if (body.trim().length > 0) {
|
|
1230
|
+
const rewritten = rewriteLinks(body, titleToDocId);
|
|
1231
|
+
try {
|
|
1232
|
+
await dm.content.write(articleDocId, rewritten);
|
|
1233
|
+
log(`✓ body ${title}`);
|
|
1234
|
+
} catch (err) {
|
|
1235
|
+
log(`! body write failed for ${title}: ${err?.message ?? err}`);
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
if (opts.mode === "split") await writeChildrenBodies(dm, snap, articleDocId, titleToDocId, log);
|
|
1239
|
+
articleCount++;
|
|
1240
|
+
}
|
|
1241
|
+
let categoryCount = 0;
|
|
1242
|
+
if (opts.includeCategories && categoryToDocId.size > 0) for (const [catTitle, catDocId] of categoryToDocId) {
|
|
1243
|
+
log(`category ${catTitle}`);
|
|
1244
|
+
try {
|
|
1245
|
+
const members = await wp.fetchCategoryPages(catTitle, opts.categoryDepth > 0, Math.max(0, opts.categoryDepth));
|
|
1246
|
+
const memberArticles = [];
|
|
1247
|
+
const subcats = [];
|
|
1248
|
+
for (const m of members) if (m.type === "subcat") subcats.push(prettyCategoryLabel(m.title));
|
|
1249
|
+
else memberArticles.push(m.title);
|
|
1250
|
+
const rewritten = rewriteLinks(renderCategoryBody(memberArticles, subcats), titleToDocId);
|
|
1251
|
+
if (rewritten.trim().length > 0) {
|
|
1252
|
+
await dm.content.write(catDocId, rewritten);
|
|
1253
|
+
log(`✓ body category ${catTitle}`);
|
|
1254
|
+
}
|
|
1255
|
+
categoryCount++;
|
|
1256
|
+
} catch (err) {
|
|
1257
|
+
log(`! category ${catTitle}: ${err?.message ?? err}`);
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
return {
|
|
1261
|
+
rootDocId: rootEntry.id,
|
|
1262
|
+
articleCount,
|
|
1263
|
+
categoryCount
|
|
1264
|
+
};
|
|
1265
|
+
}
|
|
1266
|
+
function createArticleShell(dm, article, parentId, log) {
|
|
1267
|
+
const meta = { icon: ICONS.article };
|
|
1268
|
+
if (article.url) meta.url = article.url;
|
|
1269
|
+
const entry = dm.tree.create({
|
|
1270
|
+
parentId,
|
|
1271
|
+
label: article.title,
|
|
1272
|
+
type: "doc",
|
|
1273
|
+
meta
|
|
1274
|
+
});
|
|
1275
|
+
log(`+ ${entry.id.slice(0, 8)}… ${article.title} (doc)`);
|
|
1276
|
+
return entry.id;
|
|
1277
|
+
}
|
|
1278
|
+
/**
|
|
1279
|
+
* Create section + infobox child docs for a split-mode article. Returns nothing
|
|
1280
|
+
* — children get bodies written later in writeChildrenBodies.
|
|
1281
|
+
*/
|
|
1282
|
+
function createArticleChildren(dm, article, articleDocId, log) {
|
|
1283
|
+
if (article.infobox && article.infobox.length > 0) {
|
|
1284
|
+
const ib = dm.tree.create({
|
|
1285
|
+
parentId: articleDocId,
|
|
1286
|
+
label: "Infobox",
|
|
1287
|
+
type: "outline",
|
|
1288
|
+
meta: { icon: ICONS.infobox }
|
|
1289
|
+
});
|
|
1290
|
+
log(` + ${ib.id.slice(0, 8)}… Infobox (outline)`);
|
|
1291
|
+
article._infoboxDocId = ib.id;
|
|
1292
|
+
}
|
|
1293
|
+
for (const section of article.sections) createSectionShell(dm, section, articleDocId, log);
|
|
1294
|
+
}
|
|
1295
|
+
function createSectionShell(dm, section, parentDocId, log) {
|
|
1296
|
+
const hasChildren = section.children.length > 0;
|
|
1297
|
+
if (!section.body.trim() && !hasChildren) return;
|
|
1298
|
+
const { type, icon } = pickSectionType(section);
|
|
1299
|
+
const entry = dm.tree.create({
|
|
1300
|
+
parentId: parentDocId,
|
|
1301
|
+
label: section.title || "Untitled section",
|
|
1302
|
+
type,
|
|
1303
|
+
meta: { icon }
|
|
1304
|
+
});
|
|
1305
|
+
log(` + ${entry.id.slice(0, 8)}… ${entry.label} (${type})`);
|
|
1306
|
+
section._docId = entry.id;
|
|
1307
|
+
for (const child of section.children) createSectionShell(dm, child, entry.id, log);
|
|
1308
|
+
}
|
|
1309
|
+
async function writeChildrenBodies(dm, article, _articleDocId, titleToDocId, log) {
|
|
1310
|
+
const infoboxDocId = article._infoboxDocId;
|
|
1311
|
+
if (infoboxDocId && article.infobox && article.infobox.length > 0) try {
|
|
1312
|
+
await dm.content.write(infoboxDocId, renderInfoboxBody(article.infobox));
|
|
1313
|
+
} catch (err) {
|
|
1314
|
+
log(`! infobox body write failed: ${err?.message ?? err}`);
|
|
1315
|
+
}
|
|
1316
|
+
for (const section of article.sections) await writeSectionBody(dm, section, titleToDocId, log);
|
|
1317
|
+
}
|
|
1318
|
+
async function writeSectionBody(dm, section, titleToDocId, log) {
|
|
1319
|
+
const docId = section._docId;
|
|
1320
|
+
if (docId && section.body.trim().length > 0) try {
|
|
1321
|
+
await dm.content.write(docId, rewriteLinks(section.body, titleToDocId));
|
|
1322
|
+
} catch (err) {
|
|
1323
|
+
log(`! section body write failed for ${section.title}: ${err?.message ?? err}`);
|
|
1324
|
+
}
|
|
1325
|
+
for (const child of section.children) await writeSectionBody(dm, child, titleToDocId, log);
|
|
1326
|
+
}
|
|
1327
|
+
function parseOptions(args) {
|
|
1328
|
+
const title = args.positional[0]?.trim() || args.params["title"];
|
|
1329
|
+
if (!title) return "Missing required positional argument: <title>. Example: abracadabra-wiki \"Toronto Raptors\"";
|
|
1330
|
+
const env = globalThis.process?.env ?? {};
|
|
1331
|
+
const userAgent = args.params["user-agent"] || args.params["userAgent"] || env["ABRA_WIKI_USER_AGENT"];
|
|
1332
|
+
if (!userAgent) return ["Missing required parameter: user-agent=\"your-name (you@example.com)\"", "(Wikimedia etiquette requires an Api-User-Agent header. Pass user-agent=... or set ABRA_WIKI_USER_AGENT.)"].join("\n");
|
|
1333
|
+
const mode = args.params["mode"] ?? "split";
|
|
1334
|
+
if (mode !== "single" && mode !== "split") return `Invalid mode "${mode}". Use mode=single or mode=split.`;
|
|
1335
|
+
const depth = parseIntOr(args.params["depth"], 1);
|
|
1336
|
+
const categoryDepth = parseIntOr(args.params["category-depth"] ?? args.params["categoryDepth"], 1);
|
|
1337
|
+
const rate = parseFloatOr(args.params["rate"], 3);
|
|
1338
|
+
return {
|
|
1339
|
+
title,
|
|
1340
|
+
mode,
|
|
1341
|
+
depth,
|
|
1342
|
+
categoryDepth,
|
|
1343
|
+
includeCategories: args.flags.has("include-categories") || args.flags.has("includeCategories"),
|
|
1344
|
+
lang: args.params["lang"] ?? "en",
|
|
1345
|
+
domain: args.params["domain"],
|
|
1346
|
+
parentDocId: args.params["parent"],
|
|
1347
|
+
userAgent,
|
|
1348
|
+
rate,
|
|
1349
|
+
dryRun: args.flags.has("dry-run") || args.flags.has("dryRun")
|
|
1350
|
+
};
|
|
1351
|
+
}
|
|
1352
|
+
function parseIntOr(s, fallback) {
|
|
1353
|
+
if (!s) return fallback;
|
|
1354
|
+
const n = Number.parseInt(s, 10);
|
|
1355
|
+
return Number.isFinite(n) && n >= 0 ? n : fallback;
|
|
1356
|
+
}
|
|
1357
|
+
function parseFloatOr(s, fallback) {
|
|
1358
|
+
if (!s) return fallback;
|
|
1359
|
+
const n = Number.parseFloat(s);
|
|
1360
|
+
return Number.isFinite(n) && n > 0 ? n : fallback;
|
|
1361
|
+
}
|
|
1362
|
+
function printSections(sections, indent) {
|
|
1363
|
+
const lines = [];
|
|
1364
|
+
for (const s of sections) {
|
|
1365
|
+
const hint = s.body ? ` (${s.body.length}b)` : "";
|
|
1366
|
+
lines.push(`${indent}- ${s.title}${hint}${s.children.length > 0 ? ` [${s.children.length} sub]` : ""}`);
|
|
1367
|
+
if (s.children.length > 0) lines.push(printSections(s.children, indent + " "));
|
|
1368
|
+
}
|
|
1369
|
+
return lines.join("\n");
|
|
1370
|
+
}
|
|
1371
|
+
async function main() {
|
|
1372
|
+
const args = parseArgs(process.argv);
|
|
1373
|
+
if (args.flags.has("help") || args.flags.has("h") || !args.positional[0]?.trim() && !args.params["title"]) {
|
|
1374
|
+
console.log(USAGE);
|
|
1375
|
+
return;
|
|
1376
|
+
}
|
|
1377
|
+
const output = await runWiki(args);
|
|
1378
|
+
if (output) console.log(output);
|
|
1379
|
+
}
|
|
1380
|
+
main().catch((err) => {
|
|
1381
|
+
console.error(`Fatal: ${err?.message ?? err}`);
|
|
1382
|
+
process.exit(1);
|
|
1383
|
+
});
|
|
1384
|
+
|
|
1385
|
+
//#endregion
|
|
1386
|
+
export { runWiki };
|
|
1387
|
+
//# sourceMappingURL=abracadabra-wiki.esm.js.map
|