@refract-org/ingestion 0.2.1

package/README.md

@@ -0,0 +1,27 @@
+# @refract-org/ingestion
+
+Wikimedia API adapters — revision fetching, diffing, rate limiting.
+
+```bash
+bun add @refract-org/ingestion
+```
+
+## Exports
+
+### Interfaces
+
+- `RevisionFetcher` — fetch revisions by page title
+- `RevisionSource` — async iterable revision stream
+- `DiffFetcher` — fetch diff between two revisions
+
+### Classes
+
+- `MediaWikiClient` — Wikipedia REST API client with pagination and error handling
+- `RateLimiter` — configurable request throttling
+
+```ts
+import { MediaWikiClient, RateLimiter } from "@refract-org/ingestion";
+import type { RevisionFetcher, RevisionOptions } from "@refract-org/ingestion";
+```
+
+[Refract](https://github.com/refract-org/sequent) · [Docs](https://github.com/refract-org/sequent-docs) · [npm](https://www.npmjs.com/package/@refract-org/ingestion)

package/dist/src/index.d.ts

@@ -0,0 +1,49 @@
+import type { DiffResult, Revision } from "@refract-org/evidence-graph";
+export interface AuthConfig {
+    apiKey?: string;
+    apiUser?: string;
+    apiPassword?: string;
+    oauthClientId?: string;
+    oauthClientSecret?: string;
+}
+export interface RevisionFetcher {
+    fetchRevisions(pageTitle: string, options?: RevisionOptions): Promise<Revision[]>;
+}
+export interface RevisionSource {
+    revisions(pageTitle: string, options?: RevisionOptions): AsyncIterable<Revision>;
+}
+export interface DiffFetcher {
+    fetchDiff(fromRevId: number, toRevId: number): Promise<DiffResult>;
+}
+export interface MoveFetcher {
+    fetchPageMoves(pageTitle: string): Promise<PageMove[]>;
+}
+export interface ProtectionLogEvent {
+    logId: number;
+    pageTitle: string;
+    timestamp: string;
+    comment: string;
+    action: "protect" | "unprotect" | "modify";
+    level?: string;
+}
+export interface PageMove {
+    oldTitle: string;
+    newTitle: string;
+    timestamp: string;
+    revId: number;
+    comment: string;
+}
+export interface RevisionOptions {
+    limit?: number;
+    start?: Date;
+    end?: Date;
+    direction?: "newer" | "older";
+    startRevId?: number;
+    endRevId?: number;
+}
+export { MediaWikiClient } from "./mediawiki-client.js";
+export { RateLimiter } from "./rate-limiter.js";
+export type { PageToEntityMap, WikidataClaim, WikidataEntity, WikidataValue } from "./wikidata-mapper.js";
+export { fetchWikidataEntity, fetchWikidataId, mapPagesToEntities, mapPageToEntity, wikidataEntityToEvents, } from "./wikidata-mapper.js";
+export { XmlDumpRevisionSource } from "./xml-dump-source.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAExE,MAAM,WAAW,UAAU;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;CACnF;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;CAClF;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CACpE;AAED,MAAM,WAAW,WAAW;IAC1B,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;CACxD;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IAC3C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,IAAI,CAAC;IACb,GAAG,CAAC,EAAE,IAAI,CAAC;IACX,SAAS,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,eAAe,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1G,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,eAAe,EACf,sBAAsB,GACvB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC"}

package/dist/src/index.js

@@ -0,0 +1,5 @@
+export { MediaWikiClient } from "./mediawiki-client.js";
+export { RateLimiter } from "./rate-limiter.js";
+export { fetchWikidataEntity, fetchWikidataId, mapPagesToEntities, mapPageToEntity, wikidataEntityToEvents, } from "./wikidata-mapper.js";
+export { XmlDumpRevisionSource } from "./xml-dump-source.js";
+//# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAoDA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,eAAe,EACf,sBAAsB,GACvB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC"}

package/dist/src/mediawiki-client.d.ts

@@ -0,0 +1,24 @@
+import type { DiffResult, Revision } from "@refract-org/evidence-graph";
+import type { AuthConfig, DiffFetcher, MoveFetcher, PageMove, ProtectionLogEvent, RevisionFetcher, RevisionOptions, RevisionSource } from "./index.js";
+export declare class MediaWikiClient implements RevisionFetcher, RevisionSource, DiffFetcher, MoveFetcher {
+    private rateLimiter;
+    private userAgent;
+    private apiUrl;
+    private auth?;
+    constructor(options?: {
+        apiUrl?: string;
+        userAgent?: string;
+        minDelayMs?: number;
+        auth?: AuthConfig;
+    });
+    fetchTalkRevisions(pageTitle: string, options?: RevisionOptions, talkPrefix?: string): Promise<Revision[]>;
+    fetchRevisions(pageTitle: string, options?: RevisionOptions): Promise<Revision[]>;
+    fetchPageMoves(pageTitle: string): Promise<PageMove[]>;
+    fetchProtectionLogs(pageTitle: string): Promise<ProtectionLogEvent[]>;
+    fetchDiff(fromRevId: number, toRevId: number): Promise<DiffResult>;
+    private fetch;
+    private sleep;
+    revisions(pageTitle: string, options?: RevisionOptions): AsyncIterable<Revision>;
+    private mapRevision;
+}
+//# sourceMappingURL=mediawiki-client.d.ts.map

package/dist/src/mediawiki-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mediawiki-client.d.ts","sourceRoot":"","sources":["../../src/mediawiki-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAY,UAAU,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAClF,OAAO,KAAK,EACV,UAAU,EACV,WAAW,EACX,WAAW,EACX,QAAQ,EACR,kBAAkB,EAClB,eAAe,EACf,eAAe,EACf,cAAc,EACf,MAAM,YAAY,CAAC;AAqEpB,qBAAa,eAAgB,YAAW,eAAe,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW;IAC/F,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,IAAI,CAAC,CAAa;gBAEd,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,UAAU,CAAA;KAAE;IAO/F,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAM1G,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAwEjF,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IA6CtD,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IA4DrE,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;YA6B1D,KAAK;IAgDnB,OAAO,CAAC,KAAK;IAIN,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,aAAa,CAAC,QAAQ,CAAC;IAOvF,OAAO,CAAC,WAAW;CAcpB"}

package/dist/src/mediawiki-client.js

@@ -0,0 +1,292 @@
+import { RateLimiter } from "./rate-limiter.js";
+const DEFAULT_API_URL = "https://en.wikipedia.org/w/api.php";
+const DEFAULT_USER_AGENT = "Refract/0.1.0 (https://github.com/refract-org/var-ia; sequent@nextconsensus.com)";
+const MAX_REVISIONS_PER_REQUEST = 500;
+export class MediaWikiClient {
+    rateLimiter;
+    userAgent;
+    apiUrl;
+    auth;
+    constructor(options) {
+        this.apiUrl = options?.apiUrl ?? DEFAULT_API_URL;
+        this.userAgent = options?.userAgent ?? DEFAULT_USER_AGENT;
+        this.rateLimiter = new RateLimiter(options?.minDelayMs ?? 100);
+        this.auth = options?.auth;
+    }
+    async fetchTalkRevisions(pageTitle, options, talkPrefix) {
+        const prefix = talkPrefix ?? "Talk:";
+        const talkTitle = `${prefix}${pageTitle}`;
+        return this.fetchRevisions(talkTitle, options);
+    }
+    async fetchRevisions(pageTitle, options) {
+        const revisions = [];
+        const limit = Math.min(options?.limit ?? MAX_REVISIONS_PER_REQUEST, MAX_REVISIONS_PER_REQUEST);
+        let rvcontinue;
+        let pageInfo = null;
+        while (true) {
+            const params = new URLSearchParams({
+                action: "query",
+                prop: "revisions",
+                titles: pageTitle,
+                rvprop: "content|ids|timestamp|flags|comment|size|user",
+                rvslots: "main",
+                rvlimit: String(limit),
+                format: "json",
+                formatversion: "2",
+            });
+            const isNewer = options?.direction === "newer";
+            params.set("rvdir", isNewer ? "newer" : "older");
+            if (options?.start && options?.end) {
+                params.set("rvstart", formatTimestamp(isNewer ? options.start : options.end));
+                params.set("rvend", formatTimestamp(isNewer ? options.end : options.start));
+            }
+            else if (options?.start) {
+                params.set("rvstart", formatTimestamp(options.start));
+            }
+            else if (options?.end) {
+                params.set("rvend", formatTimestamp(options.end));
+            }
+            if (options?.startRevId) {
+                params.set("rvstartid", String(options.startRevId));
+            }
+            if (options?.endRevId) {
+                params.set("rvendid", String(options.endRevId));
+            }
+            if (rvcontinue) {
+                params.set("rvcontinue", rvcontinue);
+            }
+            const url = `${this.apiUrl}?${params.toString()}`;
+            const response = await this.fetch(url);
+            const data = await response.json();
+            if (!data.query?.pages) {
+                break;
+            }
+            for (const page of Object.values(data.query.pages)) {
+                if (page.missing)
+                    continue;
+                if (!pageInfo) {
+                    pageInfo = { pageId: page.pageid, title: page.title };
+                }
+                if (page.revisions) {
+                    for (const rev of page.revisions) {
+                        revisions.push(this.mapRevision(rev, pageInfo));
+                    }
+                }
+            }
+            if (data.continue?.rvcontinue) {
+                rvcontinue = data.continue.rvcontinue;
+                if (revisions.length >= (options?.limit ?? MAX_REVISIONS_PER_REQUEST))
+                    break;
+            }
+            else {
+                break;
+            }
+        }
+        return revisions;
+    }
+    async fetchPageMoves(pageTitle) {
+        const moves = [];
+        let lecontinue;
+        while (true) {
+            const params = new URLSearchParams({
+                action: "query",
+                list: "logevents",
+                letype: "move",
+                letitle: pageTitle,
+                lelimit: "50",
+                format: "json",
+                formatversion: "2",
+            });
+            if (lecontinue)
+                params.set("lecontinue", lecontinue);
+            const url = `${this.apiUrl}?${params.toString()}`;
+            const response = await this.fetch(url);
+            const data = (await response.json());
+            if (!data.query?.logevents)
+                break;
+            for (const entry of data.query.logevents) {
+                moves.push({
+                    oldTitle: entry.title,
+                    newTitle: entry.params?.target_title ?? "",
+                    timestamp: entry.timestamp,
+                    revId: entry.logid,
+                    comment: entry.comment ?? "",
+                });
+            }
+            if (data.continue?.lecontinue) {
+                lecontinue = data.continue.lecontinue;
+            }
+            else {
+                break;
+            }
+        }
+        return moves;
+    }
+    async fetchProtectionLogs(pageTitle) {
+        const events = [];
+        let lecontinue;
+        while (true) {
+            const params = new URLSearchParams({
+                action: "query",
+                list: "logevents",
+                letype: "protect",
+                letitle: pageTitle,
+                lelimit: "50",
+                leprop: "details",
+                format: "json",
+                formatversion: "2",
+            });
+            if (lecontinue)
+                params.set("lecontinue", lecontinue);
+            const url = `${this.apiUrl}?${params.toString()}`;
+            const response = await this.fetch(url);
+            const data = (await response.json());
+            if (data.query?.logevents) {
+                for (const entry of data.query.logevents) {
+                    const level = entry.params?.detail?.[0]?.level;
+                    events.push({
+                        logId: entry.logid,
+                        pageTitle: entry.title,
+                        timestamp: entry.timestamp,
+                        comment: entry.comment ?? "",
+                        action: entry.action,
+                        level,
+                    });
+                }
+            }
+            if (data.continue?.lecontinue) {
+                lecontinue = data.continue.lecontinue;
+            }
+            else {
+                break;
+            }
+        }
+        return events;
+    }
+    async fetchDiff(fromRevId, toRevId) {
+        const params = new URLSearchParams({
+            action: "compare",
+            fromrev: String(fromRevId),
+            torev: String(toRevId),
+            format: "json",
+            formatversion: "2",
+        });
+        const url = `${this.apiUrl}?${params.toString()}`;
+        const response = await this.fetch(url);
+        const data = await response.json();
+        if (!data.compare) {
+            throw new Error(`Failed to fetch diff for revisions ${fromRevId} -> ${toRevId}`);
+        }
+        const sizeDelta = data.compare.tosize - data.compare.fromsize;
+        const lines = data.compare["*"] ? parseUnifiedDiff(data.compare["*"]) : [];
+        return {
+            fromRevId: data.compare.fromrevid,
+            toRevId: data.compare.torevid,
+            lines,
+            sections: [],
+            sizeDelta,
+        };
+    }
+    async fetch(url, retries = 3) {
+        for (let attempt = 0; attempt < retries; attempt++) {
+            await this.rateLimiter.acquire();
+            const headers = {
+                "User-Agent": this.userAgent,
+                Accept: "application/json",
+                "Accept-Encoding": "gzip",
+            };
+            if (this.auth?.apiKey) {
+                headers.Authorization = `Bearer ${this.auth.apiKey}`;
+            }
+            else if (this.auth?.apiUser && this.auth?.apiPassword) {
+                const encoded = btoa(`${this.auth.apiUser}:${this.auth.apiPassword}`);
+                headers.Authorization = `Basic ${encoded}`;
+            }
+            if (this.auth?.oauthClientId && this.auth?.oauthClientSecret) {
+                headers["X-OAuth-Client-Id"] = this.auth.oauthClientId;
+                headers["X-OAuth-Client-Secret"] = this.auth.oauthClientSecret;
+            }
+            const response = await fetch(url, {
+                headers,
+                signal: AbortSignal.timeout(30000),
+            });
+            if (response.ok)
+                return response;
+            if (response.status === 429) {
+                const retryAfter = response.headers.get("Retry-After");
+                const waitMs = retryAfter ? parseInt(retryAfter, 10) * 1000 : 1000;
+                if (attempt < retries - 1) {
+                    await this.sleep(waitMs);
+                    continue;
+                }
+            }
+            if (response.status >= 500 && attempt < retries - 1) {
+                await this.sleep(2 ** attempt * 1000);
+                continue;
+            }
+            throw new Error(`MediaWiki API error: ${response.status} ${response.statusText} for ${url}`);
+        }
+        throw new Error(`MediaWiki API request failed after ${retries} retries for ${url}`);
+    }
+    sleep(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    async *revisions(pageTitle, options) {
+        const revs = await this.fetchRevisions(pageTitle, options);
+        for (const rev of revs) {
+            yield rev;
+        }
+    }
+    mapRevision(raw, page) {
+        const content = raw.slots?.main?.content ?? "";
+        return {
+            revId: raw.revid,
+            pageId: page.pageId,
+            pageTitle: page.title,
+            timestamp: raw.timestamp,
+            user: raw.userhidden ? undefined : raw.user,
+            comment: raw.comment ?? "",
+            content,
+            size: raw.size,
+            minor: raw.minor ?? false,
+        };
+    }
+}
+function formatTimestamp(date) {
+    const iso = date.toISOString();
+    return `${iso.slice(0, -5)}Z`;
+}
+function parseUnifiedDiff(diffText) {
+    const lines = [];
+    const textLines = diffText.split("\n");
+    let fromLine = 0;
+    let toLine = 0;
+    for (const line of textLines) {
+        if (line.startsWith("@@")) {
+            const match = line.match(/@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@/);
+            if (match) {
+                fromLine = parseInt(match[1], 10);
+                toLine = parseInt(match[2], 10);
+            }
+            continue;
+        }
+        if (line.startsWith("---") || line.startsWith("+++")) {
+            continue;
+        }
+        if (line.startsWith(" ")) {
+            lines.push({ type: "unchanged", content: line.slice(1), lineNumber: toLine });
+            fromLine++;
+            toLine++;
+        }
+        else if (line.startsWith("-")) {
+            lines.push({ type: "removed", content: line.slice(1), lineNumber: fromLine });
+            fromLine++;
+        }
+        else if (line.startsWith("+")) {
+            lines.push({ type: "added", content: line.slice(1), lineNumber: toLine });
+            toLine++;
+        }
+    }
+    return lines;
+}
+//# sourceMappingURL=mediawiki-client.js.map

package/dist/src/mediawiki-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mediawiki-client.js","sourceRoot":"","sources":["../../src/mediawiki-client.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,eAAe,GAAG,oCAAoC,CAAC;AAC7D,MAAM,kBAAkB,GAAG,kFAAkF,CAAC;AAC9G,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAgEtC,MAAM,OAAO,eAAe;IAClB,WAAW,CAAc;IACzB,SAAS,CAAS;IAClB,MAAM,CAAS;IACf,IAAI,CAAc;IAE1B,YAAY,OAAyF;QACnG,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,eAAe,CAAC;QACjD,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,kBAAkB,CAAC;QAC1D,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,UAAU,IAAI,GAAG,CAAC,CAAC;QAC/D,IAAI,CAAC,IAAI,GAAG,OAAO,EAAE,IAAI,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,SAAiB,EAAE,OAAyB,EAAE,UAAmB;QACxF,MAAM,MAAM,GAAG,UAAU,IAAI,OAAO,CAAC;QACrC,MAAM,SAAS,GAAG,GAAG,MAAM,GAAG,SAAS,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,SAAiB,EAAE,OAAyB;QAC/D,MAAM,SAAS,GAAe,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,IAAI,yBAAyB,EAAE,yBAAyB,CAAC,CAAC;QAC/F,IAAI,UAA8B,CAAC;QAEnC,IAAI,QAAQ,GAAoB,IAAI,CAAC;QAErC,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;gBACjC,MAAM,EAAE,OAAO;gBACf,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,SAAS;gBACjB,MAAM,EAAE,+CAA+C;gBACvD,OAAO,EAAE,MAAM;gBACf,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC;gBACtB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,GAAG;aACnB,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,OAAO,EAAE,SAAS,KAAK,OAAO,CAAC;YAC/C,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAEjD,IAAI,OAAO,EAAE,KAAK,IAAI,OAAO,EAAE,GAAG,EAAE,CAAC;gBACnC,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC9E,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YAC9E,CAAC;iBAAM,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;gBAC1B,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YACxD,CAAC;iBAAM,IAAI,OAAO,EAAE,GAAG,EAAE,CAAC;gBACxB,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YACpD,CAAC;YACD,IAAI,OAAO,EAAE,UAAU,EAAE,CAAC;gBACxB,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YACtD,CAAC;YACD,IAAI,OAAO,EAAE,QAAQ,EAAE,CAAC;gBACtB,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;YACvC,CAAC;YAED,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;YAClD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACvC,MAAM,IAAI,GAA0B,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAE1D,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC;gBACvB,MAAM;YACR,CAAC;YAED,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnD,IAAI,IAAI,CAAC,OAAO;oBAAE,SAAS;gBAC3B,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,QAAQ,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;gBACxD,CAAC;gBACD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBACnB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;wBACjC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC;oBAClD,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC;gBAC9B,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACtC,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,yBAAyB,CAAC;oBAAE,MAAM;YAC/E,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,SAAiB;QACpC,MAAM,KAAK,GAAe,EAAE,CAAC;QAC7B,IAAI,UAA8B,CAAC;QAEnC,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;gBACjC,MAAM,EAAE,OAAO;gBACf,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,SAAS;gBAClB,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,GAAG;aACnB,CAAC,CAAC;YAEH,IAAI,UAAU;gBAAE,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;YAErD,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;YAClD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACvC,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAElC,CAAC;YAEF,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS;gBAAE,MAAM;YAElC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBACzC,KAAK,CAAC,IAAI,CAAC;oBACT,QAAQ,EAAE,KAAK,CAAC,KAAK;oBACrB,QAAQ,EAAE,KAAK,CAAC,MAAM,EAAE,YAAY,IAAI,EAAE;oBAC1C,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,EAAE;iBAC7B,CAAC,CAAC;YACL,CAAC;YAED,IAAI,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC;gBAC9B,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,SAAiB;QACzC,MAAM,MAAM,GAAyB,EAAE,CAAC;QACxC,IAAI,UAA8B,CAAC;QAEnC,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;gBACjC,MAAM,EAAE,OAAO;gBACf,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,SAAS;gBACjB,OAAO,EAAE,SAAS;gBAClB,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,SAAS;gBACjB,MAAM,EAAE,MAAM;gBACd,aAAa,EAAE,GAAG;aACnB,CAAC,CAAC;YAEH,IAAI,UAAU;gBAAE,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;YAErD,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;YAClD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACvC,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAclC,CAAC;YAEF,IAAI,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC;gBAC1B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;oBACzC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC;oBAC/C,MAAM,CAAC,IAAI,CAAC;wBACV,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,SAAS,EAAE,KAAK,CAAC,KAAK;wBACtB,SAAS,EAAE,KAAK,CAAC,SAAS;wBAC1B,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,EAAE;wBAC5B,MAAM,EAAE,KAAK,CAAC,MAA4C;wBAC1D,KAAK;qBACN,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,IAAI,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC;gBAC9B,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,SAAiB,EAAE,OAAe;QAChD,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC;YAC1B,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC;YACtB,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,GAAG;SACnB,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;QAClD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,IAAI,GAAoB,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEpD,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,sCAAsC,SAAS,OAAO,OAAO,EAAE,CAAC,CAAC;QACnF,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAE3E,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS;YACjC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;YAC7B,KAAK;YACL,QAAQ,EAAE,EAAE;YACZ,SAAS;SACV,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,OAAO,GAAG,CAAC;QAC1C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YACnD,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;YACjC,MAAM,OAAO,GAA2B;gBACtC,YAAY,EAAE,IAAI,CAAC,SAAS;gBAC5B,MAAM,EAAE,kBAAkB;gBAC1B,iBAAiB,EAAE,MAAM;aAC1B,CAAC;YAEF,IAAI,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;gBACtB,OAAO,CAAC,aAAa,GAAG,UAAU,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACvD,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,EAAE,OAAO,IAAI,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;gBACxD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;gBACtE,OAAO,CAAC,aAAa,GAAG,SAAS,OAAO,EAAE,CAAC;YAC7C,CAAC;YAED,IAAI,IAAI,CAAC,IAAI,EAAE,aAAa,IAAI,IAAI,CAAC,IAAI,EAAE,iBAAiB,EAAE,CAAC;gBAC7D,OAAO,CAAC,mBAAmB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC;gBACvD,OAAO,CAAC,uBAAuB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC;YACjE,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,OAAO;gBACP,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;aACnC,CAAC,CAAC;YAEH,IAAI,QAAQ,CAAC,EAAE;gBAAE,OAAO,QAAQ,CAAC;YAEjC,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;gBACnE,IAAI,OAAO,GAAG,OAAO,GAAG,CAAC,EAAE,CAAC;oBAC1B,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;oBACzB,SAAS;gBACX,CAAC;YACH,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,OAAO,GAAG,OAAO,GAAG,CAAC,EAAE,CAAC;gBACpD,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,OAAO,GAAG,IAAI,CAAC,CAAC;gBACtC,SAAS;YACX,CAAC;YAED,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,QAAQ,GAAG,EAAE,CAAC,CAAC;QAC/F,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,sCAAsC,OAAO,gBAAgB,GAAG,EAAE,CAAC,CAAC;IACtF,CAAC;IAEO,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,CAAC,SAAS,CAAC,SAAiB,EAAE,OAAyB;QAC3D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC3D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,GAAgB,EAAE,IAAc;QAClD,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC;QAC/C,OAAO;YACL,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,SAAS,EAAE,IAAI,CAAC,KAAK;YACrB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,IAAI,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI;YAC3C,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE;YAC1B,OAAO;YACP,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,KAAK;SAC1B,CAAC;IACJ,CAAC;CACF;AAED,SAAS,eAAe,CAAC,IAAU;IACjC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAC/B,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;AAChC,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAgB;IACxC,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEvC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;YACnE,IAAI,KAAK,EAAE,CAAC;gBACV,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAClC,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAClC,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACrD,SAAS;QACX,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;YAC9E,QAAQ,EAAE,CAAC;YACX,MAAM,EAAE,CAAC;QACX,CAAC;aAAM,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC9E,QAAQ,EAAE,CAAC;QACb,CAAC;aAAM,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1E,MAAM,EAAE,CAAC;QACX,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/src/rate-limiter.d.ts

@@ -0,0 +1,8 @@
+export declare class RateLimiter {
+    private minDelayMs;
+    private nextSlot;
+    constructor(minDelayMs?: number);
+    acquire(): Promise<void>;
+    private sleep;
+}
+//# sourceMappingURL=rate-limiter.d.ts.map

package/dist/src/rate-limiter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rate-limiter.d.ts","sourceRoot":"","sources":["../../src/rate-limiter.ts"],"names":[],"mappings":"AAAA,qBAAa,WAAW;IAGV,OAAO,CAAC,UAAU;IAF9B,OAAO,CAAC,QAAQ,CAAS;gBAEL,UAAU,GAAE,MAAY;IAItC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAkB9B,OAAO,CAAC,KAAK;CAGd"}

package/dist/src/rate-limiter.js

@@ -0,0 +1,28 @@
+export class RateLimiter {
+    minDelayMs;
+    nextSlot;
+    constructor(minDelayMs = 100) {
+        this.minDelayMs = minDelayMs;
+        this.nextSlot = Date.now();
+    }
+    async acquire() {
+        const now = Date.now();
+        let slot;
+        if (this.nextSlot <= now) {
+            slot = now;
+            this.nextSlot = now + this.minDelayMs;
+        }
+        else {
+            slot = this.nextSlot;
+            this.nextSlot += this.minDelayMs;
+        }
+        const waitMs = slot - now;
+        if (waitMs > 0) {
+            await this.sleep(waitMs);
+        }
+    }
+    sleep(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+}
+//# sourceMappingURL=rate-limiter.js.map

package/dist/src/rate-limiter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rate-limiter.js","sourceRoot":"","sources":["../../src/rate-limiter.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,WAAW;IAGF;IAFZ,QAAQ,CAAS;IAEzB,YAAoB,aAAqB,GAAG;QAAxB,eAAU,GAAV,UAAU,CAAc;QAC1C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,IAAY,CAAC;QAEjB,IAAI,IAAI,CAAC,QAAQ,IAAI,GAAG,EAAE,CAAC;YACzB,IAAI,GAAG,GAAG,CAAC;YACX,IAAI,CAAC,QAAQ,GAAG,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC;YACrB,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,UAAU,CAAC;QACnC,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,GAAG,GAAG,CAAC;QAC1B,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;CACF"}

package/dist/src/wikidata-mapper.d.ts

@@ -0,0 +1,29 @@
+import type { EvidenceEvent } from "@refract-org/evidence-graph";
+export interface WikidataEntity {
+    qid: string;
+    label: string;
+    description: string;
+    aliases: string[];
+    instanceOf: string[];
+    claims: Record<string, WikidataClaim>;
+}
+export interface WikidataClaim {
+    property: string;
+    propertyLabel: string;
+    values: WikidataValue[];
+}
+export interface WikidataValue {
+    type: "wikibase-item" | "string" | "time" | "quantity" | "url";
+    value: string;
+}
+export interface PageToEntityMap {
+    pageTitle: string;
+    qid: string;
+    entity?: WikidataEntity;
+}
+export declare function fetchWikidataId(pageTitle: string): Promise<string | null>;
+export declare function fetchWikidataEntity(qid: string): Promise<WikidataEntity | null>;
+export declare function mapPageToEntity(pageTitle: string): Promise<PageToEntityMap>;
+export declare function mapPagesToEntities(pageTitles: string[], concurrency?: number): Promise<PageToEntityMap[]>;
+export declare function wikidataEntityToEvents(entity: WikidataEntity, _pageTitle: string): EvidenceEvent[];
+//# sourceMappingURL=wikidata-mapper.d.ts.map

package/dist/src/wikidata-mapper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"wikidata-mapper.d.ts","sourceRoot":"","sources":["../../src/wikidata-mapper.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAKjE,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,aAAa,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,eAAe,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,GAAG,KAAK,CAAC;IAC/D,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB;AAED,wBAAsB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAoB/E;AAED,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAgBrF;AAED,wBAAsB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAKjF;AAED,wBAAsB,kBAAkB,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE,WAAW,SAAI,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAQ1G;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE,CAgClG"}

package/dist/src/wikidata-mapper.js

@@ -0,0 +1,138 @@
+const WIKIPEDIA_API = "https://en.wikipedia.org/w/api.php";
+const WIKIDATA_ENTITY_API = "https://www.wikidata.org/wiki/Special:EntityData";
+export async function fetchWikidataId(pageTitle) {
+    const params = new URLSearchParams({
+        action: "query",
+        prop: "pageprops",
+        titles: pageTitle,
+        format: "json",
+        origin: "*",
+    });
+    const url = `${WIKIPEDIA_API}?${params}`;
+    const res = await fetch(url, { signal: AbortSignal.timeout(15000) });
+    if (!res.ok)
+        return null;
+    const data = (await res.json());
+    const pages = data.query?.pages;
+    if (!pages)
+        return null;
+    for (const id of Object.keys(pages)) {
+        if (id === "-1")
+            continue;
+        return pages[id].pageprops?.wikibase_item ?? null;
+    }
+    return null;
+}
+export async function fetchWikidataEntity(qid) {
+    const url = `${WIKIDATA_ENTITY_API}/${encodeURIComponent(qid)}.json`;
+    const res = await fetch(url, { signal: AbortSignal.timeout(15000) });
+    if (!res.ok)
+        return null;
+    const data = (await res.json());
+    const entity = data.entities?.[qid];
+    if (!entity)
+        return null;
+    return {
+        qid,
+        label: entity.labels?.en?.value ?? qid,
+        description: entity.descriptions?.en?.value ?? "",
+        aliases: Object.values(entity.aliases?.en ?? {}).map((a) => a.value),
+        instanceOf: extractInstanceOf(entity),
+        claims: extractClaims(entity),
+    };
+}
+export async function mapPageToEntity(pageTitle) {
+    const qid = await fetchWikidataId(pageTitle);
+    if (!qid)
+        return { pageTitle, qid: "" };
+    const entity = await fetchWikidataEntity(qid);
+    return { pageTitle, qid, entity: entity ?? undefined };
+}
+export async function mapPagesToEntities(pageTitles, concurrency = 3) {
+    const results = [];
+    for (let i = 0; i < pageTitles.length; i += concurrency) {
+        const batch = pageTitles.slice(i, i + concurrency);
+        const mapped = await Promise.all(batch.map((title) => mapPageToEntity(title)));
+        results.push(...mapped);
+    }
+    return results;
+}
+export function wikidataEntityToEvents(entity, _pageTitle) {
+    const events = [];
+    const props = Object.keys(entity.claims).join(", ");
+    const instanceOf = entity.instanceOf.join(", ");
+    events.push({
+        eventType: "sentence_first_seen",
+        fromRevisionId: 0,
+        toRevisionId: 0,
+        section: "",
+        before: "",
+        after: `Wikidata entity: ${entity.label}`,
+        deterministicFacts: [{ fact: "wikidata_entity_linked", detail: `qid=${entity.qid} label=${entity.label}` }],
+        layer: "observed",
+        timestamp: new Date().toISOString(),
+    });
+    if (instanceOf) {
+        events.push({
+            eventType: "category_added",
+            fromRevisionId: 0,
+            toRevisionId: 0,
+            section: "",
+            before: "",
+            after: instanceOf,
+            deterministicFacts: [{ fact: "wikidata_instance_of", detail: `types=${instanceOf} properties=${props}` }],
+            layer: "observed",
+            timestamp: new Date().toISOString(),
+        });
+    }
+    return events;
+}
+function extractInstanceOf(entity) {
+    const p31 = entity?.claims?.P31;
+    if (!p31)
+        return [];
+    return p31
+        .filter((c) => c.mainsnak?.snaktype === "value" && c.mainsnak?.datavalue?.type === "wikibase-item")
+        .map((c) => {
+        const dt = c.mainsnak?.datavalue;
+        if (!dt || dt.type !== "wikibase-item")
+            return "";
+        return dt.value.id;
+    })
+        .filter(Boolean);
+}
+function extractClaims(entity) {
+    const result = {};
+    if (!entity?.claims)
+        return result;
+    for (const [prop, statements] of Object.entries(entity.claims)) {
+        const values = [];
+        for (const stmt of statements) {
+            if (stmt.mainsnak?.snaktype !== "value" || !stmt.mainsnak?.datavalue)
+                continue;
+            const dt = stmt.mainsnak.datavalue;
+            switch (dt.type) {
+                case "wikibase-item":
+                    values.push({ type: "wikibase-item", value: dt.value.id });
+                    break;
+                case "string":
+                    values.push({ type: "string", value: dt.value });
+                    break;
+                case "time":
+                    values.push({ type: "time", value: dt.value.time });
+                    break;
+                case "quantity":
+                    values.push({ type: "quantity", value: String(dt.value.amount) });
+                    break;
+                case "url":
+                    values.push({ type: "url", value: dt.value });
+                    break;
+            }
+        }
+        if (values.length > 0) {
+            result[prop] = { property: prop, propertyLabel: prop, values };
+        }
+    }
+    return result;
+}
+//# sourceMappingURL=wikidata-mapper.js.map