@refract-org/ingestion 0.2.1

package/src/__tests__/mediawiki-client.test.ts

@@ -0,0 +1,113 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { MediaWikiClient } from "../mediawiki-client.js";
+
+const MOCK_PROTECT_RESPONSE = {
+  query: {
+    logevents: [
+      {
+        logid: 100,
+        title: "Test",
+        timestamp: "2026-01-15T00:00:00Z",
+        comment: "semi-protected",
+        action: "protect",
+      },
+      {
+        logid: 101,
+        title: "Test",
+        timestamp: "2026-02-01T00:00:00Z",
+        comment: "extended",
+        action: "modify",
+      },
+    ],
+  },
+};
+
+const MOCK_EMPTY_RESPONSE = { query: { logevents: [] } };
+
+const MOCK_PAGE_INFO_RESPONSE = {
+  query: {
+    pages: {
+      "100": {
+        pageid: 100,
+        title: "Test",
+        revisions: [
+          {
+            revid: 1,
+            parentid: 0,
+            timestamp: "2026-01-01T00:00:00Z",
+            comment: "first",
+            size: 100,
+          },
+        ],
+      },
+    },
+  },
+};
+
+describe("MediaWikiClient", () => {
+  let client: MediaWikiClient;
+
+  beforeEach(() => {
+    vi.useFakeTimers();
+    client = new MediaWikiClient({ apiUrl: "https://en.wikipedia.org/w/api.php", minDelayMs: 0 });
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it("fetchProtectionLogs returns parsed logs", async () => {
+    const mockFetch = vi.fn().mockResolvedValue({
+      ok: true,
+      json: () => Promise.resolve(MOCK_PROTECT_RESPONSE),
+    });
+    vi.stubGlobal("fetch", mockFetch);
+
+    const logs = await client.fetchProtectionLogs("Test");
+    expect(logs).toHaveLength(2);
+    expect(logs[0].logId).toBe(100);
+    expect(logs[0].action).toBe("protect");
+    expect(logs[1].logId).toBe(101);
+    expect(logs[1].action).toBe("modify");
+  });
+
+  it("fetchProtectionLogs returns empty for no logs", async () => {
+    const mockFetch = vi.fn().mockResolvedValue({
+      ok: true,
+      json: () => Promise.resolve(MOCK_EMPTY_RESPONSE),
+    });
+    vi.stubGlobal("fetch", mockFetch);
+
+    const logs = await client.fetchProtectionLogs("Nonexistent");
+    expect(logs).toEqual([]);
+  });
+
+  it("revisions async iterator yields revisions", async () => {
+    const mockFetch = vi.fn().mockResolvedValue({
+      ok: true,
+      json: () => Promise.resolve(MOCK_PAGE_INFO_RESPONSE),
+    });
+    vi.stubGlobal("fetch", mockFetch);
+
+    const results: Array<{ revId: number }> = [];
+    for await (const rev of client.revisions("Test", { limit: 1 })) {
+      results.push(rev);
+    }
+    expect(results).toHaveLength(1);
+    expect(results[0].revId).toBe(1);
+  });
+
+  it("revisions returns empty when page is missing", async () => {
+    const mockFetch = vi.fn().mockResolvedValue({
+      ok: true,
+      json: () => Promise.resolve({ query: { pages: { "-1": { missing: "" } } } }),
+    });
+    vi.stubGlobal("fetch", mockFetch);
+
+    const revs: Array<{ revId: number }> = [];
+    for await (const rev of client.revisions("MissingPage", { limit: 1 })) {
+      revs.push(rev);
+    }
+    expect(revs).toEqual([]);
+  });
+});

package/src/__tests__/page-move.test.ts

@@ -0,0 +1,31 @@
+import { describe, expect, it } from "vitest";
+import { MediaWikiClient } from "../mediawiki-client.js";
+
+describe("Page move fetcher", () => {
+  it("fetches page moves and returns correctly shaped results", { timeout: 30000 }, async () => {
+    const client = new MediaWikiClient({ minDelayMs: 100 });
+    const moves = await client.fetchPageMoves("Earth");
+
+    expect(Array.isArray(moves)).toBe(true);
+
+    for (const move of moves) {
+      expect(move).toHaveProperty("oldTitle");
+      expect(move).toHaveProperty("newTitle");
+      expect(move).toHaveProperty("timestamp");
+      expect(move).toHaveProperty("revId");
+      expect(move).toHaveProperty("comment");
+
+      expect(typeof move.oldTitle).toBe("string");
+      expect(typeof move.newTitle).toBe("string");
+      expect(typeof move.timestamp).toBe("string");
+      expect(typeof move.revId).toBe("number");
+      expect(typeof move.comment).toBe("string");
+    }
+  });
+
+  it("returns empty array for a non-existent page", { timeout: 30000 }, async () => {
+    const client = new MediaWikiClient({ minDelayMs: 100 });
+    const moves = await client.fetchPageMoves("ThisPageDoesNotExistXYZ123!!!");
+    expect(moves).toEqual([]);
+  });
+});

package/src/__tests__/rate-limiter.test.ts

@@ -0,0 +1,30 @@
+import { describe, expect, it } from "vitest";
+import { RateLimiter } from "../rate-limiter.js";
+
+describe("RateLimiter", () => {
+  it("acquires without error", async () => {
+    const limiter = new RateLimiter(1);
+    await limiter.acquire();
+  });
+
+  it("acquires multiple times sequentially", async () => {
+    const limiter = new RateLimiter(1);
+    await limiter.acquire();
+    await limiter.acquire();
+    await limiter.acquire();
+  });
+
+  it("enforces minimum delay between acquires", async () => {
+    const limiter = new RateLimiter(50);
+    const start = Date.now();
+    await limiter.acquire();
+    await limiter.acquire();
+    const elapsed = Date.now() - start;
+    expect(elapsed).toBeGreaterThanOrEqual(45);
+  });
+
+  it("accepts custom delay", async () => {
+    const limiter = new RateLimiter(10);
+    await limiter.acquire();
+  });
+});

package/src/__tests__/talk-page.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, it } from "vitest";
+import { MediaWikiClient } from "../mediawiki-client.js";
+
+describe("Talk page fetcher", () => {
+  it("fetches talk page revisions for Earth (Talk:Earth)", { timeout: 60000 }, async () => {
+    const client = new MediaWikiClient({ minDelayMs: 100 });
+    const revisions = await client.fetchTalkRevisions("Earth", {
+      limit: 5,
+      direction: "newer",
+    });
+
+    expect(revisions.length).toBeGreaterThanOrEqual(1);
+
+    for (const rev of revisions) {
+      expect(rev).toHaveProperty("revId");
+      expect(rev).toHaveProperty("pageId");
+      expect(rev).toHaveProperty("pageTitle");
+      expect(rev).toHaveProperty("timestamp");
+      expect(rev).toHaveProperty("comment");
+      expect(rev).toHaveProperty("content");
+      expect(rev).toHaveProperty("size");
+      expect(rev).toHaveProperty("minor");
+
+      expect(typeof rev.revId).toBe("number");
+      expect(rev.revId).toBeGreaterThan(0);
+      expect(typeof rev.pageId).toBe("number");
+      expect(rev.pageId).toBeGreaterThan(0);
+      expect(rev.pageTitle).toContain("Talk:");
+      expect(typeof rev.timestamp).toBe("string");
+      expect(typeof rev.size).toBe("number");
+      expect(typeof rev.minor).toBe("boolean");
+
+      expect(rev.content.length).toBeGreaterThan(0);
+    }
+  });
+
+  it("accepts custom talk prefix for Project namespace talk pages", { timeout: 60000 }, async () => {
+    const client = new MediaWikiClient({ minDelayMs: 100 });
+    const revisions = await client.fetchTalkRevisions("About", { limit: 1, direction: "newer" }, "Wikipedia talk:");
+
+    expect(revisions.length).toBeGreaterThanOrEqual(1);
+    for (const rev of revisions) {
+      expect(rev.pageTitle).toContain("Wikipedia talk:");
+    }
+  });
+});

package/src/__tests__/wikidata-mapper.test.ts

@@ -0,0 +1,134 @@
+import { describe, expect, it, vi } from "vitest";
+import { fetchWikidataId, mapPagesToEntities, mapPageToEntity, wikidataEntityToEvents } from "../wikidata-mapper.js";
+
+describe("fetchWikidataId", () => {
+  it("returns null when page has no Wikidata ID", async () => {
+    globalThis.fetch = vi.fn().mockResolvedValue({
+      ok: true,
+      json: async () => ({ query: { pages: { "-1": { title: "NoSuchPage" } } } }),
+    });
+
+    const result = await fetchWikidataId("NoSuchPage");
+    expect(result).toBeNull();
+  });
+
+  it("returns QID when pageprops contains wikibase_item", async () => {
+    globalThis.fetch = vi.fn().mockResolvedValue({
+      ok: true,
+      json: async () => ({
+        query: { pages: { "123": { pageprops: { wikibase_item: "Q42" } } } },
+      }),
+    });
+
+    const result = await fetchWikidataId("Douglas_Adams");
+    expect(result).toBe("Q42");
+  });
+
+  it("returns null on fetch error", async () => {
+    globalThis.fetch = vi.fn().mockResolvedValue({ ok: false, status: 404 });
+
+    const result = await fetchWikidataId("SomePage");
+    expect(result).toBeNull();
+  });
+});
+
+describe("mapPageToEntity", () => {
+  it("returns empty qid when no Wikidata ID found", async () => {
+    globalThis.fetch = vi.fn().mockResolvedValue({
+      ok: true,
+      json: async () => ({ query: { pages: { "-1": {} } } }),
+    });
+
+    const result = await mapPageToEntity("NoPage");
+    expect(result.qid).toBe("");
+    expect(result.pageTitle).toBe("NoPage");
+    expect(result.entity).toBeUndefined();
+  });
+
+  it("returns entity when Wikidata ID is found", async () => {
+    const fetchMock = vi.fn();
+    fetchMock
+      .mockResolvedValueOnce({
+        ok: true,
+        json: async () => ({
+          query: { pages: { "123": { pageprops: { wikibase_item: "Q1" } } } },
+        }),
+      })
+      .mockResolvedValueOnce({
+        ok: true,
+        json: async () => ({
+          entities: {
+            Q1: {
+              labels: { en: { value: "Universe" } },
+              descriptions: { en: { value: "everything" } },
+              aliases: {},
+              claims: {
+                P31: [
+                  {
+                    mainsnak: {
+                      snaktype: "value",
+                      datavalue: { type: "wikibase-item", value: { id: "Q2" } },
+                    },
+                  },
+                ],
+                P569: [
+                  {
+                    mainsnak: {
+                      snaktype: "value",
+                      datavalue: { type: "time", value: { time: "+1800-01-01T00:00:00Z" } },
+                    },
+                  },
+                ],
+              },
+            },
+          },
+        }),
+      });
+
+    globalThis.fetch = fetchMock;
+
+    const result = await mapPageToEntity("Universe");
+    expect(result.qid).toBe("Q1");
+    expect(result.entity?.label).toBe("Universe");
+    expect(result.entity?.description).toBe("everything");
+    expect(result.entity?.instanceOf).toEqual(["Q2"]);
+    expect(Object.keys(result.entity?.claims ?? {})).toHaveLength(2); // P31 (instance of) + P569 (date of birth)
+  });
+});
+
+describe("mapPagesToEntities", () => {
+  it("maps multiple pages concurrently", async () => {
+    globalThis.fetch = vi.fn().mockResolvedValue({
+      ok: true,
+      json: async () => ({ query: { pages: { "-1": {} } } }),
+    });
+
+    const results = await mapPagesToEntities(["PageA", "PageB", "PageC"], 2);
+    expect(results).toHaveLength(3);
+    expect(results.every((r) => r.qid === "")).toBe(true);
+  });
+});
+
+describe("wikidataEntityToEvents", () => {
+  it("generates events from entity data", () => {
+    const entity = {
+      qid: "Q42",
+      label: "Test",
+      description: "A test entity",
+      aliases: [],
+      instanceOf: ["Q5"],
+      claims: {
+        P31: {
+          property: "P31",
+          propertyLabel: "instance of",
+          values: [{ type: "wikibase-item" as const, value: "Q5" }],
+        },
+      },
+    };
+
+    const events = wikidataEntityToEvents(entity, "Test_Page");
+    expect(events.length).toBeGreaterThanOrEqual(1);
+    expect(events[0].eventType).toBe("sentence_first_seen");
+    expect(events[0].deterministicFacts[0].fact).toBe("wikidata_entity_linked");
+  });
+});

package/src/__tests__/xml-dump-source.test.ts

@@ -0,0 +1,151 @@
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import { XmlDumpRevisionSource } from "../xml-dump-source.js";
+
+const SAMPLE_DUMP = `<?xml version="1.0"?>
+<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.11/">
+  <page>
+    <title>Earth</title>
+    <ns>0</ns>
+    <id>1</id>
+    <revision>
+      <id>10</id>
+      <timestamp>2026-01-01T00:00:00Z</timestamp>
+      <contributor><username>Alice</username><id>100</id></contributor>
+      <comment>first revision</comment>
+      <text>Hello world</text>
+      <sha1>abc</sha1>
+    </revision>
+    <revision>
+      <id>20</id>
+      <timestamp>2026-02-01T00:00:00Z</timestamp>
+      <contributor><username>Bob</username><id>200</id></contributor>
+      <comment>added content</comment>
+      <text>Hello world. More content here.</text>
+      <sha1>def</sha1>
+    </revision>
+  </page>
+  <page>
+    <title>Mars</title>
+    <ns>0</ns>
+    <id>2</id>
+    <revision>
+      <id>30</id>
+      <timestamp>2026-03-01T00:00:00Z</timestamp>
+      <contributor><username>Carol</username><id>300</id></contributor>
+      <comment>mars revision</comment>
+      <text>Mars content</text>
+      <sha1>ghi</sha1>
+    </revision>
+  </page>
+</mediawiki>`;
+
+const SAMPLE_ENTITIES = `<?xml version="1.0"?>
+<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.11/">
+  <page>
+    <title>Foo &amp; Bar</title>
+    <ns>0</ns>
+    <id>3</id>
+    <revision>
+      <id>40</id>
+      <timestamp>2026-04-01T00:00:00Z</timestamp>
+      <contributor><username>Dave</username><id>400</id></contributor>
+      <comment>with &amp; entities</comment>
+      <text>Text with &lt;tags&gt; &amp; entities.</text>
+      <sha1>jkl</sha1>
+    </revision>
+  </page>
+</mediawiki>`;
+
+describe("XmlDumpRevisionSource", () => {
+  it("yields revisions for matching page title", async () => {
+    const dir = mkdtempSync(join(tmpdir(), "varia-xml-test-"));
+    const filePath = join(dir, "dump.xml");
+    writeFileSync(filePath, SAMPLE_DUMP, "utf-8");
+
+    const source = new XmlDumpRevisionSource(filePath);
+    const revs: Array<{ revId: number; pageTitle: string; content: string }> = [];
+    for await (const rev of source.revisions("Earth")) {
+      revs.push(rev);
+    }
+
+    rmSync(dir, { recursive: true });
+
+    expect(revs).toHaveLength(2);
+    expect(revs[0].revId).toBe(10);
+    expect(revs[0].pageTitle).toBe("Earth");
+    expect(revs[0].content).toBe("Hello world");
+    expect(revs[1].revId).toBe(20);
+    expect(revs[1].content).toBe("Hello world. More content here.");
+  });
+
+  it("returns empty for non-existent page", async () => {
+    const dir = mkdtempSync(join(tmpdir(), "varia-xml-test-"));
+    const filePath = join(dir, "dump.xml");
+    writeFileSync(filePath, SAMPLE_DUMP, "utf-8");
+
+    const source = new XmlDumpRevisionSource(filePath);
+    const revs: Array<unknown> = [];
+    for await (const rev of source.revisions("Venus")) {
+      revs.push(rev);
+    }
+
+    rmSync(dir, { recursive: true });
+    expect(revs).toEqual([]);
+  });
+
+  it("respects limit option", async () => {
+    const dir = mkdtempSync(join(tmpdir(), "varia-xml-test-"));
+    const filePath = join(dir, "dump.xml");
+    writeFileSync(filePath, SAMPLE_DUMP, "utf-8");
+
+    const source = new XmlDumpRevisionSource(filePath);
+    const revs: Array<{ revId: number }> = [];
+    for await (const rev of source.revisions("Earth", { limit: 1 })) {
+      revs.push(rev);
+    }
+
+    rmSync(dir, { recursive: true });
+    expect(revs).toHaveLength(1);
+    expect(revs[0].revId).toBe(10);
+  });
+
+  it("respects timestamp filtering", async () => {
+    const dir = mkdtempSync(join(tmpdir(), "varia-xml-test-"));
+    const filePath = join(dir, "dump.xml");
+    writeFileSync(filePath, SAMPLE_DUMP, "utf-8");
+
+    const source = new XmlDumpRevisionSource(filePath);
+    const revs: Array<{ revId: number }> = [];
+    for await (const rev of source.revisions("Earth", {
+      start: new Date("2026-01-15T00:00:00Z"),
+    })) {
+      revs.push(rev);
+    }
+
+    rmSync(dir, { recursive: true });
+    expect(revs).toHaveLength(1);
+    expect(revs[0].revId).toBe(20);
+  });
+
+  it("handles XML entities in title and content", async () => {
+    const dir = mkdtempSync(join(tmpdir(), "varia-xml-test-"));
+    const filePath = join(dir, "dump.xml");
+    writeFileSync(filePath, SAMPLE_ENTITIES, "utf-8");
+
+    const source = new XmlDumpRevisionSource(filePath);
+    const revs: Array<{ revId: number; pageTitle: string; content: string; comment: string }> = [];
+    for await (const rev of source.revisions("Foo & Bar")) {
+      revs.push(rev);
+    }
+
+    rmSync(dir, { recursive: true });
+
+    expect(revs).toHaveLength(1);
+    expect(revs[0].revId).toBe(40);
+    expect(revs[0].content).toBe("Text with <tags> & entities.");
+    expect(revs[0].comment).toBe("with & entities");
+  });
+});

package/src/index.ts

@@ -0,0 +1,63 @@
+import type { DiffResult, Revision } from "@refract-org/evidence-graph";
+
+export interface AuthConfig {
+  apiKey?: string;
+  apiUser?: string;
+  apiPassword?: string;
+  oauthClientId?: string;
+  oauthClientSecret?: string;
+}
+
+export interface RevisionFetcher {
+  fetchRevisions(pageTitle: string, options?: RevisionOptions): Promise<Revision[]>;
+}
+
+export interface RevisionSource {
+  revisions(pageTitle: string, options?: RevisionOptions): AsyncIterable<Revision>;
+}
+
+export interface DiffFetcher {
+  fetchDiff(fromRevId: number, toRevId: number): Promise<DiffResult>;
+}
+
+export interface MoveFetcher {
+  fetchPageMoves(pageTitle: string): Promise<PageMove[]>;
+}
+
+export interface ProtectionLogEvent {
+  logId: number;
+  pageTitle: string;
+  timestamp: string;
+  comment: string;
+  action: "protect" | "unprotect" | "modify";
+  level?: string;
+}
+
+export interface PageMove {
+  oldTitle: string;
+  newTitle: string;
+  timestamp: string;
+  revId: number;
+  comment: string;
+}
+
+export interface RevisionOptions {
+  limit?: number;
+  start?: Date;
+  end?: Date;
+  direction?: "newer" | "older";
+  startRevId?: number;
+  endRevId?: number;
+}
+
+export { MediaWikiClient } from "./mediawiki-client.js";
+export { RateLimiter } from "./rate-limiter.js";
+export type { PageToEntityMap, WikidataClaim, WikidataEntity, WikidataValue } from "./wikidata-mapper.js";
+export {
+  fetchWikidataEntity,
+  fetchWikidataId,
+  mapPagesToEntities,
+  mapPageToEntity,
+  wikidataEntityToEvents,
+} from "./wikidata-mapper.js";
+export { XmlDumpRevisionSource } from "./xml-dump-source.js";