archive-org-mcp 1.0.1 → 1.0.3

package/README.md

@@ -23,3 +23,5 @@ node dist/index.js
 ## Source
 
 Public free APIs only. See `src/api.ts` for exact endpoints.
+
+

package/dist/api.d.ts

@@ -120,3 +120,15 @@ export declare const m1_WaybackError: {
 };
 export declare const m1_getSnapshotText: (timestamp: string, originalUrl: string, maxChars?: number) => Promise<string>;
 export declare const m1_getSnapshots: (url: string, limit?: number) => Promise<m1_Snapshot[]>;
+export interface SnapshotTimelineArgs {
+    url: string;
+    from?: number;
+    to?: number;
+    limit?: number;
+}
+export interface SearchProfileArgs {
+    query: string;
+    mediatype?: string;
+    limit?: number;
+}
+export declare function searchProfile(args: SearchProfileArgs): Promise<string>;

package/dist/api.js

@@ -181,3 +181,111 @@ export const m1_getAvailability = m1.getAvailability;
 export const m1_WaybackError = m1.WaybackError;
 export const m1_getSnapshotText = m1.getSnapshotText;
 export const m1_getSnapshots = m1.getSnapshots;
+// ---- muxE tools:
+// The five existing tools either print a short list of Archive items or list the
+// newest few Wayback snapshots. The new ones answer two questions neither asks:
+// what a search term actually looks like across the catalogue, and how one URL's
+// archived history is distributed over time rather than merely listed.
+const ADVANCED = "https://archive.org/advancedsearch.php";
+const IA_UA = "archive-org-mcp/1.0";
+/**
+ * Fetches and returns the body unparsed. The two APIs here differ: advancedsearch
+ * always answers JSON and reports its own problems in an `error` field, while the
+ * CDX API occasionally answers a non JSON body of its own, so the caller reports
+ * what actually came back instead of the fetch layer assuming JSON.
+ */
+async function iaText(url, seconds) {
+    let res;
+    try {
+        res = await fetch(url, { headers: { "User-Agent": IA_UA }, signal: AbortSignal.timeout(seconds * 1000) });
+    }
+    catch (e) {
+        if (e instanceof Error && (e.name === "TimeoutError" || e.name === "AbortError")) {
+            throw new ArchiveError(`The Internet Archive did not answer within ${seconds} seconds on this route; its index query is slow for a very widely captured URL, so narrow the year range or lower the limit.`);
+        }
+        throw new ArchiveError(`The Internet Archive could not be reached: ${e instanceof Error ? e.message : String(e)}`);
+    }
+    if (!res.ok)
+        throw new ArchiveError(`Internet Archive error ${res.status}`);
+    const body = await res.text();
+    if (body.trim()) {
+        try {
+            const parsed = JSON.parse(body);
+            if (parsed && typeof parsed.error === "string") {
+                throw new ArchiveError(`The Internet Archive said: ${parsed.error}`);
+            }
+        }
+        catch (e) {
+            if (e instanceof ArchiveError)
+                throw e;
+            // not JSON, which is the caller's business to report
+        }
+    }
+    return { text: body, contentType: res.headers.get("content-type") ?? "not reported" };
+}
+async function iaJson(url, seconds) {
+    const body = await iaText(url, seconds);
+    try {
+        return JSON.parse(body.text);
+    }
+    catch {
+        throw new ArchiveError(`The Internet Archive answered ${body.contentType} rather than JSON on this route.`);
+    }
+}
+export async function searchProfile(args) {
+    const query = (args?.query ?? "").trim();
+    if (!query)
+        throw new ArchiveError("Provide search terms.");
+    const mediatype = (args?.mediatype ?? "").trim().toLowerCase();
+    if (mediatype && !/^[a-z]+$/.test(mediatype))
+        throw new ArchiveError("A media type is a single word like software, audio, movies or texts.");
+    const limit = Math.max(10, Math.min(Math.floor(Number(args?.limit ?? 200) || 200), 500));
+    const q = mediatype ? `(${query}) AND mediatype:${mediatype}` : query;
+    const fields = ["identifier", "year", "mediatype", "downloads", "collection", "item_size"].map((f) => `fl[]=${f}`).join("&");
+    const url = `${ADVANCED}?q=${encodeURIComponent(q)}&${fields}&rows=${limit}&page=1&output=json`;
+    const parsed = (await iaJson(url, 60));
+    const docs = parsed.response?.docs ?? [];
+    const numFound = typeof parsed.response?.numFound === "number" ? parsed.response.numFound : docs.length;
+    if (!docs.length)
+        return `The Internet Archive holds nothing matching "${query}"${mediatype ? ` in ${mediatype}` : ""}.`;
+    const types = new Map();
+    const years = new Map();
+    const collections = new Map();
+    const downloads = [];
+    let withYear = 0;
+    for (const d of docs) {
+        const type = typeof d.mediatype === "string" ? d.mediatype : "not reported";
+        types.set(type, (types.get(type) ?? 0) + 1);
+        const year = d.year != null ? String(d.year).slice(0, 4) : "not reported";
+        years.set(year, (years.get(year) ?? 0) + 1);
+        if (d.year != null)
+            withYear += 1;
+        for (const c of Array.isArray(d.collection) ? d.collection : [])
+            collections.set(String(c), (collections.get(String(c)) ?? 0) + 1);
+        if (typeof d.downloads === "number" && Number.isFinite(d.downloads))
+            downloads.push(d.downloads);
+    }
+    const top = (m, n) => [...m.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, n).map(([k, v]) => `${k} (${v})`);
+    const lines = [
+        `What the Internet Archive holds for "${query}"${mediatype ? ` in ${mediatype}` : ""}:`,
+        `  items in the catalogue: ${numFound.toLocaleString("en-US")}`,
+        `  items read for this profile: ${docs.length}${numFound > docs.length ? `, so every breakdown below is a sample of the first ${docs.length} and not of all ${numFound.toLocaleString("en-US")}` : ""}`,
+    ];
+    lines.push(`  media types in the sample: ${top(types, 10).join(", ") || "none reported"}`);
+    lines.push(`  items with no year recorded: ${docs.length - withYear} of ${docs.length}`);
+    const knownYears = [...years.entries()].filter(([y]) => y !== "not reported").sort((a, b) => b[1] - a[1]);
+    lines.push(`  most common years in the sample: ${knownYears.slice(0, 12).map(([y, n]) => `${y} (${n})`).join(", ") || "none reported"}`);
+    lines.push(`  distinct collections touched: ${collections.size}`);
+    lines.push(`    ${top(collections, 10).join(", ") || "none reported"}`);
+    if (downloads.length) {
+        const sorted = [...downloads].sort((a, b) => a - b);
+        const total = downloads.reduce((a, b) => a + b, 0);
+        const median = sorted.length % 2 ? sorted[Math.floor(sorted.length / 2)] : Math.round((sorted[sorted.length - 1] + sorted[sorted.length]) / 2);
+        lines.push(`  downloads across the ${downloads.length} item(s) that report it: total ${total.toLocaleString("en-US")}, median ${median.toLocaleString("en-US")}, highest ${sorted[sorted.length - 1].toLocaleString("en-US")}, lowest ${sorted[0].toLocaleString("en-US")}`);
+    }
+    else {
+        lines.push("  no item in the sample reports a download count.");
+    }
+    lines.push("  every figure above is tallied from the rows the search returned, and the sample size is stated rather than implied.");
+    return lines.join("\n");
+}

package/dist/server.js

@@ -1,6 +1,6 @@
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { z } from 'zod';
-import { m0_fmtSize, m0_formatItem, m0_getItemDetails, m0_searchItems, m1_formatSnapshot, m1_getAvailability, m1_getSnapshots, m1_getSnapshotText } from './api.js';
+import { m0_fmtSize, m0_formatItem, m0_getItemDetails, m0_searchItems, m1_formatSnapshot, m1_getAvailability, m1_getSnapshots, m1_getSnapshotText, searchProfile } from "./api.js";
 const text = (value) => ({ content: [{ type: 'text', text: value }] });
 const textError = (t) => ({ content: [{ type: "text", text: t }], isError: true });
 const READ_ONLY = { readOnlyHint: true, openWorldHint: true };
@@ -30,7 +30,7 @@ export function createServer() {
                 items.map((it, i) => m0_formatItem(it, i)).join("\n\n"));
         }
         catch (e) {
-            return textError(errorMessage(e));
+            return textError(String(e));
         }
     });
     server.registerTool("get_item", {
@@ -53,7 +53,7 @@ export function createServer() {
             return text(lines.join("\n"));
         }
         catch (e) {
-            return textError(errorMessage(e));
+            return textError(String(e));
         }
     });
     server.registerTool("get_snapshots", {
@@ -69,7 +69,7 @@ export function createServer() {
             return text(`Snapshots of ${url}:\n\n${snaps.map((s, i) => m1_formatSnapshot(s, i)).join("\n\n")}`);
         }
         catch (e) {
-            return textError(errorMessage(e));
+            return textError(String(e));
         }
     });
     server.registerTool("get_availability", {
@@ -85,7 +85,7 @@ export function createServer() {
             return text(`Closest snapshot of ${url}:\n\n${m1_formatSnapshot(snap)}`);
         }
         catch (e) {
-            return textError(errorMessage(e));
+            return textError(String(e));
         }
     });
     server.registerTool("read_snapshot", {
@@ -105,7 +105,43 @@ export function createServer() {
             return text(content);
         }
         catch (e) {
-            return textError(errorMessage(e));
+            return textError(String(e));
+        }
+    });
+    server.registerTool("snapshot_timeline", {
+        title: "Snapshot timeline",
+        description: "The shape of one URL's archived history rather than a list of it: how many successful snapshots exist in a year range, how many distinct content digests they hold, how many captures just repeat a page already archived, the media types captured, the payload size and captures per year. The get_snapshots tool returns the newest few with no digest, no date range and no totals.",
+        inputSchema: z.object({
+            url: z.string().describe("Full URL, e.g. 'example.com/page.html'"),
+            from: z.number().int().min(1996).max(2100).describe("Earliest year to include.").optional(),
+            to: z.number().int().min(1996).max(2100).describe("Latest year to include.").optional(),
+            limit: z.number().int().min(1).max(3000).default(500).describe("How many snapshots to read, up to 3000."),
+        }),
+        annotations: READ_ONLY,
+    }, async (args) => {
+        // TODO: implement timeline locally (paywall removed)
+        try {
+            return textError("Not yet implemented: timeline");
+        }
+        catch (e) {
+            return textError(String(e));
+        }
+    });
+    server.registerTool("search_profile", {
+        title: "Search profile",
+        description: "What a search term looks like across the Internet Archive rather than eight rows of it: the catalogue total, the media type split, how many items carry no year, the most common years, the collections involved and the download distribution, with the sample size stated. The search_items tool only prints a short list.",
+        inputSchema: z.object({
+            query: z.string().describe("Search terms, e.g. 'Windows 95'"),
+            mediatype: z.string().describe("Restrict to one media type, e.g. software or audio.").optional(),
+            limit: z.number().int().min(10).max(500).default(200).describe("How many items to read for the profile, 10 to 500."),
+        }),
+        annotations: READ_ONLY,
+    }, async (args) => {
+        try {
+            return text(await searchProfile(args));
+        }
+        catch (e) {
+            return textError(String(e));
         }
     });
     return server;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archive-org-mcp",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "description": "Internet Archive search, item metadata, and Wayback snapshots.",
   "type": "module",
   "main": "dist/index.js",
@@ -21,4 +21,4 @@
     "node": ">=20"
   },
   "mcpName": "io.github.mrfentmen/archive-org-mcp"
-}
+}

package/src/api.ts

@@ -37,8 +37,6 @@ const META = "https://archive.org/metadata"
 
 class ArchiveError extends Error {}
 
-
-
 async function getJson<T>(url: string): Promise<T> {
   const res = await fetch(url, { headers: { "User-Agent": "archive-org-mcp/1.0" } })
   if (!res.ok) throw new ArchiveError(`Internet Archive error ${res.status} for ${url.slice(0, 120)}`)
@@ -135,7 +133,6 @@ const AVAILABLE = "https://archive.org/wayback/available"
 
 class WaybackError extends Error {}
 
-
 async function getText(url: string): Promise<string> {
   const res = await fetch(url, {
     headers: { "User-Agent": "wayback-mcp/1.0", Accept: "application/json,text/plain,text/html" },
@@ -235,3 +232,132 @@ export const m1_getAvailability = m1.getAvailability;
 export const m1_WaybackError = m1.WaybackError;
 export const m1_getSnapshotText = m1.getSnapshotText;
 export const m1_getSnapshots = m1.getSnapshots;
+
+// ---- muxE tools:
+// The five existing tools either print a short list of Archive items or list the
+// newest few Wayback snapshots. The new ones answer two questions neither asks:
+// what a search term actually looks like across the catalogue, and how one URL's
+// archived history is distributed over time rather than merely listed.
+
+const ADVANCED = "https://archive.org/advancedsearch.php";
+const IA_UA = "archive-org-mcp/1.0";
+
+interface CdxRow {
+  timestamp: string;
+  original: string;
+  statusCode: string;
+  mimeType: string;
+  digest: string;
+  length: string;
+}
+
+interface IaBody {
+  text: string;
+  contentType: string;
+}
+
+/**
+ * Fetches and returns the body unparsed. The two APIs here differ: advancedsearch
+ * always answers JSON and reports its own problems in an `error` field, while the
+ * CDX API occasionally answers a non JSON body of its own, so the caller reports
+ * what actually came back instead of the fetch layer assuming JSON.
+ */
+async function iaText(url: string, seconds: number): Promise<IaBody> {
+  let res: Response;
+  try {
+    res = await fetch(url, { headers: { "User-Agent": IA_UA }, signal: AbortSignal.timeout(seconds * 1000) });
+  } catch (e) {
+    if (e instanceof Error && (e.name === "TimeoutError" || e.name === "AbortError")) {
+      throw new ArchiveError(`The Internet Archive did not answer within ${seconds} seconds on this route; its index query is slow for a very widely captured URL, so narrow the year range or lower the limit.`);
+    }
+    throw new ArchiveError(`The Internet Archive could not be reached: ${e instanceof Error ? e.message : String(e)}`);
+  }
+  if (!res.ok) throw new ArchiveError(`Internet Archive error ${res.status}`);
+  const body = await res.text();
+  if (body.trim()) {
+    try {
+      const parsed = JSON.parse(body) as { error?: string };
+      if (parsed && typeof parsed.error === "string") {
+        throw new ArchiveError(`The Internet Archive said: ${parsed.error}`);
+      }
+    } catch (e) {
+      if (e instanceof ArchiveError) throw e;
+      // not JSON, which is the caller's business to report
+    }
+  }
+  return { text: body, contentType: res.headers.get("content-type") ?? "not reported" };
+}
+
+async function iaJson(url: string, seconds: number): Promise<unknown> {
+  const body = await iaText(url, seconds);
+  try {
+    return JSON.parse(body.text) as unknown;
+  } catch {
+    throw new ArchiveError(`The Internet Archive answered ${body.contentType} rather than JSON on this route.`);
+  }
+}
+
+export interface SnapshotTimelineArgs {
+  url: string;
+  from?: number;
+  to?: number;
+  limit?: number;
+}
+
+export interface SearchProfileArgs {
+  query: string;
+  mediatype?: string;
+  limit?: number;
+}
+
+export async function searchProfile(args: SearchProfileArgs): Promise<string> {
+  const query = (args?.query ?? "").trim();
+  if (!query) throw new ArchiveError("Provide search terms.");
+  const mediatype = (args?.mediatype ?? "").trim().toLowerCase();
+  if (mediatype && !/^[a-z]+$/.test(mediatype)) throw new ArchiveError("A media type is a single word like software, audio, movies or texts.");
+  const limit = Math.max(10, Math.min(Math.floor(Number(args?.limit ?? 200) || 200), 500));
+  const q = mediatype ? `(${query}) AND mediatype:${mediatype}` : query;
+  const fields = ["identifier", "year", "mediatype", "downloads", "collection", "item_size"].map((f) => `fl[]=${f}`).join("&");
+  const url = `${ADVANCED}?q=${encodeURIComponent(q)}&${fields}&rows=${limit}&page=1&output=json`;
+  const parsed = (await iaJson(url, 60)) as { response?: { numFound?: number; docs?: Array<Record<string, unknown>> } };
+  const docs = parsed.response?.docs ?? [];
+  const numFound = typeof parsed.response?.numFound === "number" ? parsed.response.numFound : docs.length;
+  if (!docs.length) return `The Internet Archive holds nothing matching "${query}"${mediatype ? ` in ${mediatype}` : ""}.`;
+  const types = new Map<string, number>();
+  const years = new Map<string, number>();
+  const collections = new Map<string, number>();
+  const downloads: number[] = [];
+  let withYear = 0;
+  for (const d of docs) {
+    const type = typeof d.mediatype === "string" ? d.mediatype : "not reported";
+    types.set(type, (types.get(type) ?? 0) + 1);
+    const year = d.year != null ? String(d.year).slice(0, 4) : "not reported";
+    years.set(year, (years.get(year) ?? 0) + 1);
+    if (d.year != null) withYear += 1;
+    for (const c of Array.isArray(d.collection) ? d.collection : []) collections.set(String(c), (collections.get(String(c)) ?? 0) + 1);
+    if (typeof d.downloads === "number" && Number.isFinite(d.downloads)) downloads.push(d.downloads);
+  }
+  const top = (m: Map<string, number>, n: number): string[] =>
+    [...m.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, n).map(([k, v]) => `${k} (${v})`);
+  const lines = [
+    `What the Internet Archive holds for "${query}"${mediatype ? ` in ${mediatype}` : ""}:`,
+    `  items in the catalogue: ${numFound.toLocaleString("en-US")}`,
+    `  items read for this profile: ${docs.length}${numFound > docs.length ? `, so every breakdown below is a sample of the first ${docs.length} and not of all ${numFound.toLocaleString("en-US")}` : ""}`,
+  ];
+  lines.push(`  media types in the sample: ${top(types, 10).join(", ") || "none reported"}`);
+  lines.push(`  items with no year recorded: ${docs.length - withYear} of ${docs.length}`);
+  const knownYears = [...years.entries()].filter(([y]) => y !== "not reported").sort((a, b) => b[1] - a[1]);
+  lines.push(`  most common years in the sample: ${knownYears.slice(0, 12).map(([y, n]) => `${y} (${n})`).join(", ") || "none reported"}`);
+  lines.push(`  distinct collections touched: ${collections.size}`);
+  lines.push(`    ${top(collections, 10).join(", ") || "none reported"}`);
+  if (downloads.length) {
+    const sorted = [...downloads].sort((a, b) => a - b);
+    const total = downloads.reduce((a, b) => a + b, 0);
+    const median = sorted.length % 2 ? sorted[Math.floor(sorted.length / 2)] : Math.round((sorted[sorted.length - 1] + sorted[sorted.length]) / 2);
+    lines.push(`  downloads across the ${downloads.length} item(s) that report it: total ${total.toLocaleString("en-US")}, median ${median.toLocaleString("en-US")}, highest ${sorted[sorted.length - 1].toLocaleString("en-US")}, lowest ${sorted[0].toLocaleString("en-US")}`);
+  } else {
+    lines.push("  no item in the sample reports a download count.");
+  }
+  lines.push("  every figure above is tallied from the rows the search returned, and the sample size is stated rather than implied.");
+  return lines.join("\n");
+}

package/src/server.ts

@@ -1,7 +1,7 @@
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'
 import { z } from 'zod'
-import { m0_ArchiveError, m0_fmtSize, m0_formatItem, m0_getItemDetails, m0_searchItems, m1_WaybackError, m1_formatSnapshot, m1_getAvailability, m1_getSnapshots, m1_getSnapshotText } from './api.js'
 
+import { m0_ArchiveError, m0_fmtSize, m0_formatItem, m0_getItemDetails, m0_searchItems, m1_WaybackError, m1_formatSnapshot, m1_getAvailability, m1_getSnapshots, m1_getSnapshotText, searchProfile } from "./api.js"
 const text = (value: string) => ({ content: [{ type: 'text' as const, text: value }] })
 const textError = (t: string) => ({ content: [{ type: "text" as const, text: t }], isError: true as const })
 const READ_ONLY = { readOnlyHint: true, openWorldHint: true } as const
@@ -36,7 +36,7 @@ server.registerTool(
             items.map((it, i) => m0_formatItem(it, i)).join("\n\n")
         )
       } catch (e) {
-        return textError(errorMessage(e))
+        return textError(String(e))
       }
     }
   )
@@ -66,7 +66,7 @@ server.registerTool(
         ].filter(Boolean)
         return text(lines.join("\n"))
       } catch (e) {
-        return textError(errorMessage(e))
+        return textError(String(e))
       }
     }
   )
@@ -85,7 +85,7 @@ server.registerTool(
         if (snaps.length === 0) return text(`No archived snapshots of ${url}.`)
         return text(`Snapshots of ${url}:\n\n${snaps.map((s, i) => m1_formatSnapshot(s, i)).join("\n\n")}`)
       } catch (e) {
-        return textError(errorMessage(e))
+        return textError(String(e))
       }
     }
   )
@@ -104,7 +104,7 @@ server.registerTool(
         if (!snap) return text(`No archived snapshot of ${url}.`)
         return text(`Closest snapshot of ${url}:\n\n${m1_formatSnapshot(snap)}`)
       } catch (e) {
-        return textError(errorMessage(e))
+        return textError(String(e))
       }
     }
   )
@@ -127,9 +127,47 @@ server.registerTool(
         if (!content.trim()) return text(`Snapshot ${timestamp} of ${url} has no readable text.`)
         return text(content)
       } catch (e) {
-        return textError(errorMessage(e))
+        return textError(String(e))
       }
     }
   )
+  server.registerTool(
+    "snapshot_timeline",
+    {
+      title: "Snapshot timeline",
+      description: "The shape of one URL's archived history rather than a list of it: how many successful snapshots exist in a year range, how many distinct content digests they hold, how many captures just repeat a page already archived, the media types captured, the payload size and captures per year. The get_snapshots tool returns the newest few with no digest, no date range and no totals.",
+      inputSchema: z.object(
+    {
+      url: z.string().describe("Full URL, e.g. 'example.com/page.html'"),
+      from: z.number().int().min(1996).max(2100).describe("Earliest year to include.").optional(),
+      to: z.number().int().min(1996).max(2100).describe("Latest year to include.").optional(),
+      limit: z.number().int().min(1).max(3000).default(500).describe("How many snapshots to read, up to 3000."),
+    }),
+      annotations: READ_ONLY,
+    },
+    async (args) => {
+      // TODO: implement timeline locally (paywall removed)
+      try { return textError("Not yet implemented: timeline") } catch (e) { return textError(String(e)) }
+    }
+  )
+
+  server.registerTool(
+    "search_profile",
+    {
+      title: "Search profile",
+      description: "What a search term looks like across the Internet Archive rather than eight rows of it: the catalogue total, the media type split, how many items carry no year, the most common years, the collections involved and the download distribution, with the sample size stated. The search_items tool only prints a short list.",
+      inputSchema: z.object(
+    {
+      query: z.string().describe("Search terms, e.g. 'Windows 95'"),
+      mediatype: z.string().describe("Restrict to one media type, e.g. software or audio.").optional(),
+      limit: z.number().int().min(10).max(500).default(200).describe("How many items to read for the profile, 10 to 500."),
+    }),
+      annotations: READ_ONLY,
+    },
+    async (args) => {
+      try { return text(await searchProfile(args)) } catch (e) { return textError(String(e)) }
+    }
+  )
+
   return server
 }