@coywolf/emdash 0.35.13 → 0.35.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/README.md +14 -4
  2. package/docs/README.md +45 -0
  3. package/docs/ai.md +39 -0
  4. package/docs/backups.md +88 -0
  5. package/docs/breadcrumbs.md +59 -0
  6. package/docs/code-blocks.md +21 -0
  7. package/docs/content-urls.md +41 -0
  8. package/docs/custom-blocks.md +27 -0
  9. package/docs/discovery.md +32 -0
  10. package/docs/files.md +44 -0
  11. package/docs/form-uploads.md +59 -0
  12. package/docs/headings.md +37 -0
  13. package/docs/images.md +141 -0
  14. package/docs/internals.md +13 -0
  15. package/docs/links.md +34 -0
  16. package/docs/performance.md +72 -0
  17. package/docs/redirects.md +84 -0
  18. package/docs/releasing.md +21 -0
  19. package/docs/reviews.md +89 -0
  20. package/docs/robots.md +81 -0
  21. package/docs/schema.md +79 -0
  22. package/docs/search.md +114 -0
  23. package/docs/videos.md +48 -0
  24. package/docs/wordpress-import.md +198 -0
  25. package/package.json +11 -2
  26. package/src/admin/features.tsx +4 -0
  27. package/src/admin/index.tsx +7 -1
  28. package/src/admin/update-notice.tsx +198 -0
  29. package/src/core/features.ts +10 -0
  30. package/src/core/module.ts +2 -0
  31. package/src/index.ts +11 -3
  32. package/src/modules.ts +3 -0
  33. package/src/options.ts +2 -0
  34. package/src/pageCache/lib.ts +4 -2
  35. package/src/updates/check.ts +135 -0
  36. package/src/updates/core.ts +56 -0
  37. package/src/updates/pack.ts +54 -0
  38. package/src/version.ts +1 -1
  39. package/src/ai/logic.test.ts +0 -187
  40. package/src/ai/queue.test.ts +0 -97
  41. package/src/ai/wikidata.test.ts +0 -184
  42. package/src/backups/dump.test.mjs +0 -287
  43. package/src/backups/restore.test.mjs +0 -103
  44. package/src/codeBlocks/render.test.ts +0 -152
  45. package/src/files/counts.test.ts +0 -114
  46. package/src/files/format.test.ts +0 -93
  47. package/src/files/r2-rules.test.ts +0 -30
  48. package/src/files/sigv4.test.ts +0 -64
  49. package/src/headings/headings.test.ts +0 -159
  50. package/src/pageCache/lib.test.mjs +0 -224
  51. package/src/pageCache/warm.test.mjs +0 -670
  52. package/src/redirects/slug-core.test.ts +0 -50
  53. package/src/robots/rep.test.ts +0 -335
  54. package/src/robots/rules.test.ts +0 -251
  55. package/src/schema/graph.test.mjs +0 -522
@@ -1,187 +0,0 @@
1
- /**
2
- * Unit tests for AI Enrichment's pure logic.
3
- * Run: node --test test/*.test.mjs (or node --test src/ai/*.test.ts)
4
- */
5
- import assert from "node:assert/strict";
6
- import { test } from "node:test";
7
-
8
- // @ts-ignore -- Node runs the .ts source directly; tsc doesn't need to resolve it.
9
- import * as L from "./logic.ts";
10
-
11
- const logic = L as typeof import("./logic.js");
12
-
13
- test("decodeJson tolerates fences, prose, and trailing text", () => {
14
- assert.deepEqual(logic.decodeJson('```json\n[{"a":1}]\n```'), [{ a: 1 }]);
15
- assert.deepEqual(logic.decodeJson('Here you go: {"a":2} hope that helps'), { a: 2 });
16
- assert.equal(logic.decodeJson("no json here"), null);
17
- assert.deepEqual(logic.decodeJson({ already: true }), { already: true });
18
- });
19
-
20
- test("parseMentions validates shape, types, duplicates, and the 12 cap", () => {
21
- const rows = [
22
- { surface: "NYC", name: "New York City", type: "Place", description: "A city", primary: true },
23
- { name: "new york city", type: "Place" }, // duplicate (case-insensitive)
24
- { name: "Widget", type: "Gadget", description: "<b>thing</b>" }, // unknown type → Thing, tags stripped
25
- { type: "Person" }, // no name
26
- "junk",
27
- ...Array.from({ length: 20 }, (_, i) => ({ name: `E${i}`, type: "Organization" })),
28
- ];
29
- const out = logic.parseMentions(JSON.stringify(rows));
30
- assert.equal(out.length, 12);
31
- assert.deepEqual(out[0], { surface: "NYC", name: "New York City", type: "Place", description: "A city", primary: true });
32
- assert.equal(out[1].type, "Thing");
33
- assert.equal(out[1].description, "thing");
34
- assert.equal(out[1].primary, false);
35
- assert.deepEqual(logic.parseMentions("not json"), []);
36
- assert.equal(logic.parseMentions('{"entities":[{"name":"X","type":"Thing"}]}').length, 1);
37
- });
38
-
39
- test("parseChoices keeps only QID strings", () => {
40
- assert.deepEqual(logic.parseChoices('{"A":"q42","B":null,"C":"https://wikidata.org/Q1","D":7}'), { A: "Q42", B: null, C: null, D: null });
41
- assert.deepEqual(logic.parseChoices("[1,2]"), {});
42
- });
43
-
44
- test("cleanDescription strips quotes and labels and caps at 155 on a word", () => {
45
- assert.equal(logic.cleanDescription('"Meta description: A short summary."'), "A short summary.");
46
- const long = "word ".repeat(60);
47
- const out = logic.cleanDescription(long);
48
- assert.ok([...out].length <= 155, `length ${[...out].length}`);
49
- assert.ok(out.endsWith("…"));
50
- assert.ok(!out.includes("wor…"));
51
- assert.equal(logic.cleanDescription(""), "");
52
- });
53
-
54
- test("parseImageText validates and normalizes", () => {
55
- const r = logic.parseImageText('{"alt_text":"Image of a dog running.","title":"Dog Running","caption":"A dog.","description":"x"}');
56
- assert.deepEqual(r, { alt: "A dog running.", caption: "A dog.", title: "Dog Running" });
57
- assert.equal(logic.parseImageText('{"alt_text":"","title":"Divider","caption":""}').alt, "");
58
- assert.throws(() => logic.parseImageText(""), /empty response/);
59
- assert.throws(() => logic.parseImageText("nope"), /could not be parsed/);
60
- assert.throws(() => logic.parseImageText('{"caption":"only"}'), /usable/);
61
- });
62
-
63
- const human = { p31: ["Q5"], wikipedia: "", website: "" };
64
- const city = { p31: ["Q515"], wikipedia: "https://en.wikipedia.org/wiki/Paris", website: "https://paris.fr" };
65
- const disamb = { p31: ["Q4167410"], wikipedia: "", website: "" };
66
-
67
- test("grounding: candidates, choices restricted to real candidates", () => {
68
- const mentions = [
69
- { surface: "", name: "Paris", type: "Place", description: "", primary: true },
70
- { surface: "", name: "Mercury", type: "Thing", description: "planet", primary: false },
71
- { surface: "", name: "Nobody", type: "Person", description: "", primary: false },
72
- ] as const;
73
- const { mentions: grounded, ambiguous } = logic.attachCandidates([...mentions], [
74
- [{ id: "Q90", label: "Paris", description: "capital of France" }],
75
- [
76
- { id: "Q308", label: "Mercury", description: "planet" },
77
- { id: "Q925", label: "mercury", description: "element" },
78
- ],
79
- [],
80
- ]);
81
- assert.equal(grounded[0].qid, "Q90");
82
- assert.deepEqual(ambiguous, [1]);
83
- assert.equal(grounded[2].qid, "");
84
- // A hallucinated QID is ignored; a listed one is accepted.
85
- assert.equal(logic.applyChoices(grounded, ambiguous, { Mercury: "Q999" })[1].qid, "");
86
- assert.equal(logic.applyChoices(grounded, ambiguous, { Mercury: "Q308" })[1].qid, "Q308");
87
- });
88
-
89
- test("verifyEntities drops disambiguation pages and Person/human mismatches", () => {
90
- const m = (name: string, type: string, qid: string) => ({ surface: "", name, type, description: "", primary: false, qid, candidates: [] }) as never;
91
- const out = logic.verifyEntities(
92
- [m("Ada Lovelace", "Person", "Q7259"), m("Paris", "Place", "Q90"), m("Paris Hilton", "Place", "Q47899"), m("A Band", "Person", "Q1"), m("Mercury", "Thing", "Q2"), m("Unresolved", "Thing", ""), m("Dup", "Place", "Q90")],
93
- { Q7259: human, Q90: city, Q47899: human, Q1: { p31: ["Q215380"], wikipedia: "", website: "" }, Q2: disamb },
94
- );
95
- assert.deepEqual(
96
- out.map((e) => e.qid),
97
- ["Q7259", "Q90"],
98
- );
99
- assert.equal(out[1].website, "https://paris.fr");
100
- });
101
-
102
- test("parseSearch reads a wbsearchentities response", () => {
103
- assert.deepEqual(logic.parseSearch({ search: [{ id: "Q1", label: "Universe", description: "all" }, { id: "P31" }, {}] }), [{ id: "Q1", label: "Universe", description: "all" }]);
104
- });
105
-
106
- test("entityNodes builds about/mentions with sameAs", () => {
107
- const nodes = logic.entityNodes([
108
- { name: "Paris", type: "Place", description: "City", qid: "Q90", wikipedia: "https://en.wikipedia.org/wiki/Paris", website: "", primary: true },
109
- { name: "X", type: "Thing", description: "", qid: "Q1", wikipedia: "", website: "", primary: false },
110
- { name: "Bad", type: "Thing", description: "", qid: "nope", wikipedia: "", website: "", primary: false },
111
- ]);
112
- assert.deepEqual(nodes.about, [{ "@type": "Place", name: "Paris", sameAs: ["https://www.wikidata.org/wiki/Q90", "https://en.wikipedia.org/wiki/Paris"], description: "City" }]);
113
- assert.deepEqual(nodes.mentions, [{ "@type": "Thing", name: "X", sameAs: "https://www.wikidata.org/wiki/Q1" }]);
114
- });
115
-
116
- test("Portable Text → plain text", () => {
117
- const pt = [
118
- { _type: "block", children: [{ _type: "span", text: "Hello " }, { _type: "span", text: "world." }] },
119
- { _type: "image", asset: { _ref: "x" }, caption: "A caption" },
120
- { _type: "columns", columns: [{ content: [{ _type: "block", children: [{ text: "Nested." }] }] }] },
121
- ];
122
- assert.equal(logic.portableTextToPlain(pt), "Hello world.\n\nA caption\n\nNested.");
123
- const text = logic.entryPlainText({ body: pt, excerpt: "<p>Intro</p>", count: 3 }, [
124
- { slug: "excerpt", type: "text" },
125
- { slug: "body", type: "portableText" },
126
- { slug: "count", type: "number" },
127
- ]);
128
- assert.ok(text.startsWith(" Intro") || text.startsWith("Intro"));
129
- assert.ok(text.includes("Hello world."));
130
- assert.equal(logic.entryPlainText({ body: pt }, [{ slug: "body", type: "portableText" }], 5), "Hello");
131
- });
132
-
133
- test("normalizeName and nameMatches ignore case, diacritics, and punctuation", () => {
134
- assert.equal(logic.normalizeName(" Café-Müller, Inc. "), "cafe muller inc");
135
- const c = { id: "Q1", label: "Beyoncé", description: "", aliases: ["Beyoncé Knowles"] };
136
- assert.equal(logic.nameMatches({ name: "beyonce", surface: "" }, c), true);
137
- assert.equal(logic.nameMatches({ name: "Queen B", surface: "Beyoncé Knowles" }, c), true);
138
- assert.equal(logic.nameMatches({ name: "Beyoncé tour", surface: "" }, c), false);
139
- });
140
-
141
- test("attachCandidates drops search hits whose label/alias doesn't match, even a single one", () => {
142
- const m = [
143
- { surface: "", name: "Acme Widgets", type: "Organization", description: "", primary: true },
144
- { surface: "the Met", name: "Metropolitan Museum of Art", type: "Organization", description: "", primary: false },
145
- ] as never;
146
- const { mentions, ambiguous } = logic.attachCandidates(m, [
147
- [{ id: "Q5", label: "Acme Corporation", description: "fictional company" }],
148
- [
149
- { id: "Q160236", label: "Metropolitan Museum of Art", description: "art museum" },
150
- { id: "Q99", label: "Metropolitan Opera", description: "opera company", aliases: ["the Met"] },
151
- ],
152
- ]);
153
- assert.equal(mentions[0].qid, "");
154
- assert.equal(mentions[0].candidates.length, 0);
155
- assert.deepEqual(ambiguous, [1]); // both match (label and alias): the model must choose
156
- });
157
-
158
- test("parseSearch keeps aliases and the matched text", () => {
159
- const out = logic.parseSearch({ search: [{ id: "Q1", label: "A", aliases: ["Alpha"], match: { type: "alias", text: "Alef" } }] });
160
- assert.deepEqual(out[0].aliases, ["Alpha", "Alef"]);
161
- });
162
-
163
- test("verifyEntities uses the Wikidata label as the name and rejects unverifiable items", () => {
164
- const g = (name: string, type: string, qid: string, label: string) =>
165
- ({ surface: "", name, type, description: "", primary: true, qid, candidates: [{ id: qid, label, description: "" }] }) as never;
166
- const out = logic.verifyEntities([g("nyc", "Place", "Q60", "New York City"), g("Ghost", "Thing", "Q404", "Ghost")], {
167
- Q60: { p31: ["Q515"], wikipedia: "", website: "" },
168
- });
169
- assert.deepEqual(
170
- out.map((e) => e.name),
171
- ["New York City"],
172
- ); // Q404 wasn't returned by Wikidata: dropped
173
- });
174
-
175
- test("typeCompatible: Person needs Q5; Organization/Place reject creative works; Organization rejects places", () => {
176
- assert.equal(logic.typeCompatible("Person", []), false);
177
- assert.equal(logic.typeCompatible("Person", ["Q5"]), true);
178
- assert.equal(logic.typeCompatible("Organization", ["Q482994"]), false); // album, not the band
179
- assert.equal(logic.typeCompatible("Organization", ["Q515"]), false); // city
180
- assert.equal(logic.typeCompatible("Organization", ["Q4830453"]), true); // business
181
- assert.equal(logic.typeCompatible("Organization", []), false);
182
- assert.equal(logic.typeCompatible("Place", ["Q11424"]), false); // film
183
- assert.equal(logic.typeCompatible("Place", ["Q515"]), true);
184
- assert.equal(logic.typeCompatible("Thing", ["Q11424"]), true); // a film is a fine Thing
185
- assert.equal(logic.typeCompatible("Thing", ["Q5"]), false);
186
- assert.equal(logic.typeCompatible("Thing", ["Q4167410"]), false);
187
- });
@@ -1,97 +0,0 @@
1
- /**
2
- * Unit tests for AI Enrichment queue batching.
3
- * Run: node --test test/*.test.mjs (or node --test src/ai/*.test.ts)
4
- */
5
- import assert from "node:assert/strict";
6
- import { test } from "node:test";
7
-
8
- // @ts-ignore -- Node runs the .ts source directly; tsc doesn't need to resolve it.
9
- import * as L from "./logic.ts";
10
-
11
- const logic = L as typeof import("./logic.js");
12
- type Job = import("./logic.js").QueueJob;
13
-
14
- const job = (id: string, due: number, kind: Job["kind"] = "entry"): { id: string; data: Job } => ({
15
- id,
16
- data: { kind, due, attempts: 0, enqueuedAt: 0, ...(kind === "media" ? { mediaId: id } : { collection: "posts", entryId: id }) },
17
- });
18
-
19
- test("planBatch takes due jobs in due order up to the per-tick limit", () => {
20
- const jobs = [job("c", 300), job("a", 100), job("future", 10_000), job("b", 200)];
21
- const out = logic.planBatch(jobs, { now: 1000, perTick: 2, remainingCalls: 100, callsPerJob: () => 3 });
22
- assert.deepEqual(
23
- out.map((j) => j.id),
24
- ["a", "b"],
25
- );
26
- });
27
-
28
- test("planBatch respects the daily call budget and keeps each kind in order", () => {
29
- const jobs = [job("entry1", 1), job("img", 2, "media"), job("entry2", 3)];
30
- const cost = (j: Job) => (j.kind === "media" ? 1 : 3);
31
- assert.deepEqual(
32
- logic.planBatch(jobs, { now: 10, perTick: 10, remainingCalls: 4, callsPerJob: cost }).map((j) => j.id),
33
- ["entry1", "img"],
34
- );
35
- // Not enough budget for the first entry: no entry runs (entry2 doesn't jump the line), but the image can.
36
- assert.deepEqual(
37
- logic.planBatch(jobs, { now: 10, perTick: 10, remainingCalls: 2, callsPerJob: cost }).map((j) => j.id),
38
- ["img"],
39
- );
40
- assert.deepEqual(logic.planBatch(jobs, { now: 10, perTick: 10, remainingCalls: -5, callsPerJob: cost }), []);
41
- });
42
-
43
- test("deferJob pushes the job to Retry-After without an attempt, and gives up deferring after MAX_DEFERRALS in a row", () => {
44
- const j = job("a", 0).data;
45
- const once = logic.deferJob(j, 1_000, 30_000);
46
- assert.deepEqual(once, { ...j, deferrals: 1, due: 31_000 });
47
- assert.equal(once!.attempts, 0);
48
- const tenth = logic.deferJob({ ...j, deferrals: logic.MAX_DEFERRALS - 1 }, 1_000, 30_000);
49
- assert.equal(tenth?.deferrals, logic.MAX_DEFERRALS);
50
- assert.equal(logic.deferJob({ ...j, deferrals: logic.MAX_DEFERRALS }, 1_000, 30_000), null, "the next one counts as a failed attempt");
51
- assert.equal(logic.MAX_DEFERRALS, 10);
52
- });
53
-
54
- test("retryAt backs off and gives up after MAX_ATTEMPTS", () => {
55
- const j = job("a", 0).data;
56
- assert.equal(logic.retryAt({ ...j, attempts: 0 }, 0), 5 * 60_000);
57
- assert.equal(logic.retryAt({ ...j, attempts: 1 }, 0), 20 * 60_000);
58
- assert.equal(logic.retryAt({ ...j, attempts: logic.MAX_ATTEMPTS - 1 }, 0), null);
59
- });
60
-
61
- test("mergeJob debounces: later due wins, force sticks, attempts reset", () => {
62
- const first: Job = { kind: "entry", collection: "posts", entryId: "1", due: 100, attempts: 2, enqueuedAt: 50, force: true };
63
- const next: Job = { kind: "entry", collection: "posts", entryId: "1", due: 500, attempts: 0, enqueuedAt: 400 };
64
- assert.deepEqual(logic.mergeJob(first, next), { ...next, force: true, enqueuedAt: 50, attempts: 0 });
65
- assert.deepEqual(logic.mergeJob(null, next), next);
66
- });
67
-
68
- test("queueId and dayKey", () => {
69
- assert.equal(logic.queueId({ kind: "entry", collection: "posts", entryId: "x" }), "entry:posts:x");
70
- assert.equal(logic.queueId({ kind: "media", mediaId: "m" }), "media:m");
71
- assert.equal(logic.dayKey(Date.UTC(2026, 9, 3, 23, 59)), "2026-10-03");
72
- });
73
-
74
- test("planBatch alternates kinds so a backlog of entries can't starve images", () => {
75
- const jobs = [job("e1", 1), job("e2", 2), job("e3", 3), job("e4", 4), job("m1", 50, "media"), job("m2", 60, "media")];
76
- const out = logic.planBatch(jobs, { now: 100, perTick: 4, remainingCalls: 100, callsPerJob: () => 1 });
77
- assert.deepEqual(
78
- out.map((j) => j.id),
79
- ["e1", "m1", "e2", "m2"],
80
- );
81
- });
82
-
83
- test("planBatch stays within the subrequest budget; an expensive kind stops without blocking a cheap one", () => {
84
- const jobs = [job("e1", 1), job("e2", 2), job("m1", 3, "media"), job("m2", 4, "media")];
85
- const out = logic.planBatch(jobs, {
86
- now: 100,
87
- perTick: 10,
88
- remainingCalls: 100,
89
- callsPerJob: () => 1,
90
- subrequests: 20,
91
- subrequestsPerJob: (j: Job) => (j.kind === "entry" ? 16 : 1),
92
- });
93
- assert.deepEqual(
94
- out.map((j) => j.id),
95
- ["e1", "m1", "m2"],
96
- );
97
- });
@@ -1,184 +0,0 @@
1
- /**
2
- * Unit tests for the Wikidata client: the SPARQL details query, parsing its
3
- * results, and how non-OK answers (429, maxlag) are handled.
4
- * Run: node --test test/*.test.mjs
5
- */
6
- import assert from "node:assert/strict";
7
- import { readFileSync } from "node:fs";
8
- import { test } from "node:test";
9
-
10
- // @ts-ignore -- Node runs the .ts source directly; tsc doesn't need to resolve it.
11
- import * as L from "./logic.ts";
12
- // @ts-ignore -- as above.
13
- import * as W from "./wikidata.ts";
14
-
15
- const logic = L as typeof import("./logic.js");
16
- const wikidata = W as typeof import("./wikidata.js");
17
- const fixture = (name: string) => JSON.parse(readFileSync(new URL(`../../test/fixtures/${name}`, import.meta.url), "utf8"));
18
- const UA = "CoywolfPack/0.0.0 (https://example.com) EmDash";
19
-
20
- /** A fetcher that records requests and answers with the given responses in turn. */
21
- function fakeFetch(...responses: Response[]) {
22
- const calls: Array<{ url: string; init?: RequestInit }> = [];
23
- const fetcher = async (url: string, init?: RequestInit) => {
24
- calls.push({ url, init });
25
- const next = responses.shift();
26
- if (!next) throw new Error("unexpected request");
27
- return next;
28
- };
29
- return { fetcher, calls };
30
- }
31
-
32
- /** Run `fn` with console.warn captured. */
33
- async function warnings<T>(fn: () => Promise<T>): Promise<{ result?: T; error?: unknown; warned: unknown[][] }> {
34
- const warned: unknown[][] = [];
35
- const original = console.warn;
36
- console.warn = (...args: unknown[]) => void warned.push(args);
37
- try {
38
- return { result: await fn(), warned };
39
- } catch (error) {
40
- return { error, warned };
41
- } finally {
42
- console.warn = original;
43
- }
44
- }
45
-
46
- const json = (body: unknown, init: ResponseInit = {}) => new Response(JSON.stringify(body), { ...init, headers: { "content-type": "application/json", ...(init.headers ?? {}) } });
47
-
48
- test("detailsQuery takes only Q-ids, once each, and the sitelink only for a plain language code", () => {
49
- const q = logic.detailsQuery(["Q90", "Q90", "q5", "P31", "Q1 } . ?x ?y ?z", "Q42"], "en");
50
- assert.match(q, /VALUES \?item \{ wd:Q90 wd:Q42 \}/);
51
- assert.doesNotMatch(q, /\?x|P31 \}|q5/);
52
- assert.match(q, /schema:isPartOf <https:\/\/en\.wikipedia\.org\/>/);
53
- assert.match(q, /FILTER\(\?r != wikibase:DeprecatedRank\)/);
54
- assert.match(q, /SELECT \?item \?class \?website \?rank \?title/);
55
- assert.match(q, /ps:P856 \?website ; wikibase:rank \?rank \. FILTER\(\?rank != wikibase:DeprecatedRank\)/);
56
- assert.doesNotMatch(logic.detailsQuery(["Q1"], "e>n"), /wikipedia/);
57
- });
58
-
59
- test("parseSparqlDetails gives the same details the wbgetentities parser did (recorded Q90, Q42, Q95, Q1, Q2)", () => {
60
- const parsed = logic.parseSparqlDetails(fixture("wikidata-details.sparql.json"), "en");
61
- const expected: Record<string, { p31: string[] }> = fixture("wikidata-details.expected.json");
62
- // SPARQL rows have no statement order, and P31 order doesn't matter to verification.
63
- const sorted = (o: Record<string, { p31: string[] }>) => Object.fromEntries(Object.entries(o).map(([k, v]) => [k, { ...v, p31: [...v.p31].sort() }]));
64
- assert.deepEqual(sorted(parsed!), sorted(expected));
65
- });
66
-
67
- test("parseSparqlDetails keeps Q-ids and http(s) websites only, and rejects non-SPARQL bodies", () => {
68
- const row = (fields: Record<string, string>) => Object.fromEntries(Object.entries(fields).map(([k, value]) => [k, { type: "uri", value }]));
69
- const E = "http://www.wikidata.org/entity/";
70
- const out = logic.parseSparqlDetails(
71
- {
72
- results: {
73
- bindings: [
74
- row({ item: `${E}Q7`, class: `${E}Q5` }),
75
- row({ item: `${E}Q7`, class: `${E}Q5` }),
76
- row({ item: `${E}Q7`, class: "http://www.wikidata.org/.well-known/genid/abc" }),
77
- row({ item: `${E}Q7`, website: "ftp://example.com/" }),
78
- row({ item: `${E}Q7`, website: "https://example.com/" }),
79
- row({ item: `${E}Q7`, title: "A/B: C d" }),
80
- row({ item: `${E}L1`, class: `${E}Q5` }),
81
- row({ item: `${E}Q8` }),
82
- ],
83
- },
84
- },
85
- "en",
86
- );
87
- assert.deepEqual(out, {
88
- Q7: { p31: ["Q5"], wikipedia: "https://en.wikipedia.org/wiki/A/B:_C_d", website: "https://example.com/" },
89
- Q8: { p31: [], wikipedia: "", website: "" },
90
- });
91
- assert.equal(logic.parseSparqlDetails({ error: "nope" }, "en"), null);
92
- assert.equal(logic.parseSparqlDetails(null, "en"), null);
93
- });
94
-
95
- test("parseSparqlDetails picks one website regardless of row order: preferred rank, then https, then the smallest URL", () => {
96
- const E = "http://www.wikidata.org/entity/";
97
- const R = "http://wikiba.se/ontology#";
98
- const site = (qid: string, website: string, rank = "NormalRank") => ({ item: { type: "uri", value: `${E}${qid}` }, website: { type: "uri", value: website }, rank: { type: "uri", value: `${R}${rank}` } });
99
- const rows = [
100
- site("Q1", "https://zzz.example/"),
101
- site("Q1", "http://preferred.example/", "PreferredRank"),
102
- site("Q1", "https://aaa.example/"),
103
- site("Q2", "http://aaa.example/"),
104
- site("Q2", "https://zzz.example/"),
105
- site("Q3", "https://b.example/"),
106
- site("Q3", "https://a.example/"),
107
- ];
108
- const pick = (bindings: unknown[]) => Object.fromEntries(Object.entries(logic.parseSparqlDetails({ results: { bindings } }, "en")!).map(([k, v]) => [k, v.website]));
109
- const expected = { Q1: "http://preferred.example/", Q2: "https://zzz.example/", Q3: "https://a.example/" };
110
- assert.deepEqual(pick(rows), expected);
111
- assert.deepEqual(pick([...rows].reverse()), expected);
112
- // The recorded Q90 lists two websites with the http one first: the https one is kept.
113
- assert.equal(logic.parseSparqlDetails(fixture("wikidata-details.sparql.json"), "en")!.Q90.website, "https://www.paris.fr/");
114
- });
115
-
116
- test("userAgent follows the Wikimedia policy, with the repository as contact when the site has no URL", () => {
117
- assert.equal(logic.userAgent("1.2.3", "https://example.com"), "CoywolfPack/1.2.3 (https://example.com) EmDash");
118
- assert.equal(logic.userAgent("1.2.3", undefined), "CoywolfPack/1.2.3 (https://github.com/coywolf-llc/coywolf-pack) EmDash");
119
- assert.equal(logic.userAgent("1.2.3", ""), "CoywolfPack/1.2.3 (https://github.com/coywolf-llc/coywolf-pack) EmDash");
120
- });
121
-
122
- test("entityDetails makes one SPARQL request to query.wikidata.org", async () => {
123
- const { fetcher, calls } = fakeFetch(json(fixture("wikidata-details.sparql.json")));
124
- const details = await wikidata.entityDetails(fetcher, ["Q90", "Q42", "Q95", "Q1", "Q2", "bogus"], "en", UA);
125
- assert.equal(calls.length, 1);
126
- const url = new URL(calls[0].url);
127
- assert.equal(url.origin + url.pathname, `https://${wikidata.WIKIDATA_QUERY_HOST}/sparql`);
128
- assert.equal(url.searchParams.get("format"), "json");
129
- assert.match(url.searchParams.get("query")!, /wd:Q90 wd:Q42 wd:Q95 wd:Q1 wd:Q2 \}/);
130
- const headers = calls[0].init?.headers as Record<string, string>;
131
- assert.equal(headers["user-agent"], UA);
132
- assert.equal(headers.accept, "application/sparql-results+json");
133
- assert.deepEqual(Object.keys(details!).sort(), ["Q1", "Q2", "Q42", "Q90", "Q95"]);
134
- // Nothing to look up: no request.
135
- assert.deepEqual(await wikidata.entityDetails(fakeFetch().fetcher, ["P31"], "en", UA), {});
136
- });
137
-
138
- test("a non-OK answer is logged with its status and Retry-After; a 500 means 'didn't answer'", async () => {
139
- const { result, warned } = await warnings(() => wikidata.entityDetails(fakeFetch(new Response("down", { status: 503, headers: { "retry-after": "30" } })).fetcher, ["Q1"], "en", UA));
140
- assert.equal(result, null);
141
- assert.equal(warned.length, 1);
142
- assert.match(String(warned[0][0]), /query\.wikidata\.org.*HTTP 503/);
143
- assert.deepEqual(warned[0][1], { retryAfter: "30" });
144
- // Search: an empty result that isn't cached, so the next run asks again.
145
- const search = fakeFetch(new Response("down", { status: 502 }), json({ search: [{ id: "Q64", label: "Berlin" }] }));
146
- const first = await warnings(() => wikidata.searchCandidates(search.fetcher, "Berlin-uncached", "en", UA));
147
- assert.deepEqual(first.result, []);
148
- assert.match(String(first.warned[0][0]), /www\.wikidata\.org.*HTTP 502/);
149
- assert.deepEqual(await wikidata.searchCandidates(search.fetcher, "Berlin-uncached", "en", UA), [{ id: "Q64", label: "Berlin", description: "" }]);
150
- });
151
-
152
- test("HTTP 429 defers the job until Retry-After instead of failing it", async () => {
153
- const sparql = await warnings(() => wikidata.entityDetails(fakeFetch(new Response("slow down", { status: 429, headers: { "retry-after": "120" } })).fetcher, ["Q1"], "en", UA));
154
- assert.ok(sparql.error instanceof wikidata.WikidataBusyError);
155
- assert.equal((sparql.error as InstanceType<typeof wikidata.WikidataBusyError>).retryAfterMs, 120_000);
156
- assert.deepEqual(sparql.warned[0][1], { retryAfter: "120" });
157
- const search = await warnings(() => wikidata.searchCandidates(fakeFetch(new Response("", { status: 429 })).fetcher, "Paris-429", "en", UA));
158
- assert.ok(search.error instanceof wikidata.WikidataBusyError);
159
- assert.equal((search.error as InstanceType<typeof wikidata.WikidataBusyError>).retryAfterMs, 60_000, "no Retry-After: a minute");
160
- });
161
-
162
- test("api.php calls send maxlag=5, and a maxlag answer defers like a 429", async () => {
163
- const lagged = json({ error: { code: "maxlag", info: "Waiting for a database server: 6 seconds lagged." } }, { headers: { "retry-after": "5" } });
164
- const { fetcher, calls } = fakeFetch(lagged);
165
- const { error, warned } = await warnings(() => wikidata.searchCandidates(fetcher, "Paris-lag", "fr", UA));
166
- const url = new URL(calls[0].url);
167
- assert.equal(url.host, wikidata.WIKIDATA_HOST);
168
- assert.equal(url.searchParams.get("maxlag"), "5");
169
- assert.equal(url.searchParams.get("action"), "wbsearchentities");
170
- assert.equal(url.searchParams.get("language"), "fr");
171
- assert.ok(error instanceof wikidata.WikidataBusyError);
172
- assert.equal((error as InstanceType<typeof wikidata.WikidataBusyError>).retryAfterMs, 5_000);
173
- assert.match(String(warned[0][0]), /maxlag/);
174
- });
175
-
176
- test("retryAfterMs reads seconds or an HTTP date, within 5 seconds to an hour", () => {
177
- const now = Date.parse("2026-10-07T12:00:00Z");
178
- assert.equal(wikidata.retryAfterMs("30", now), 30_000);
179
- assert.equal(wikidata.retryAfterMs("Wed, 07 Oct 2026 12:02:00 GMT", now), 120_000);
180
- assert.equal(wikidata.retryAfterMs(null, now), 60_000);
181
- assert.equal(wikidata.retryAfterMs("soon", now), 60_000);
182
- assert.equal(wikidata.retryAfterMs("0", now), 5_000);
183
- assert.equal(wikidata.retryAfterMs("86400", now), 3_600_000);
184
- });