mcp-scraper 0.37.0 → 0.38.0

package/dist/{server-QKDDEBVQ.js → server-3Q6XIJPF.js}

@@ -56,7 +56,7 @@ import {
   sanitizeAttempts,
   sanitizeHarvestResult,
   transcribeMediaUrl
-} from "./chunk-JK2FRDAP.js";
+} from "./chunk-P64PZUCA.js";
 import {
   auditImageUrls,
   auditImages,
@@ -101,7 +101,7 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
 } from "./chunk-CB5C3BPB.js";
-import "./chunk-3LWYPAU5.js";
+import "./chunk-H3M3O457.js";
 import {
   abandonExtractSettlement,
   countSuccessfulPages,
@@ -14026,6 +14026,22 @@ var DiffPageBodySchema = z15.object({
   allowLocal: z15.boolean().optional(),
   resetBaseline: z15.boolean().optional()
 });
+var ArchiveReadBodySchema = z15.object({
+  url: z15.string().url("url must be a valid HTTPS ZIP URL"),
+  path: z15.string().trim().min(1).max(2e3).optional(),
+  offset: z15.number().int().min(0).optional(),
+  maxBytes: z15.number().int().min(1).max(2e5).optional(),
+  maxEntries: z15.number().int().min(1).max(1e3).optional(),
+  depositToLibrary: z15.boolean().optional()
+}).superRefine((value, ctx) => {
+  if (value.depositToLibrary && !value.path) {
+    ctx.addIssue({
+      code: z15.ZodIssueCode.custom,
+      path: ["path"],
+      message: "path is required when depositToLibrary is true"
+    });
+  }
+});
 var MapUrlsBodySchema = z15.object({
   url: z15.string().min(1, "url is required"),
   maxUrls: z15.number().int().min(1).max(2e3).optional(),
@@ -26845,7 +26861,14 @@ async function depositScrapeToVault(user, opts) {
     const title = (opts.title?.trim() || opts.source).slice(0, 200);
     const res = await memoryCall(
       "libraryIngestTool",
-      { title, content: clipped, source: opts.source, vault },
+      {
+        title,
+        content: clipped,
+        source: opts.source,
+        vault,
+        ...opts.capturedAt ? { capturedAt: opts.capturedAt } : {},
+        ...opts.summary ? { summary: opts.summary } : {}
+      },
       key
     );
     if (!res.ok) return { deposited: false, vault, error: res.error ?? "ingest failed" };
@@ -26878,6 +26901,414 @@ async function persistScrapeBody(user, opts) {
   return { ...deposit, ...fallback };
 }
 
+// src/api/archive-reader.ts
+import { basename, extname } from "path";
+import yauzl from "yauzl";
+var MAX_ARCHIVE_DOWNLOAD_BYTES = 50 * 1024 * 1024;
+var MAX_ARCHIVE_ENTRIES = 1e4;
+var MAX_ARCHIVE_EXPANDED_BYTES = 250 * 1024 * 1024;
+var MAX_ARCHIVE_ENTRY_BYTES = 10 * 1024 * 1024;
+var MAX_LIBRARY_ENTRY_BYTES = 5 * 1024 * 1024;
+var MAX_COMPRESSION_RATIO = 200;
+var MAX_ARCHIVE_REDIRECTS = 4;
+var READABLE_EXTENSIONS = /* @__PURE__ */ new Map([
+  [".txt", "text/plain"],
+  [".md", "text/markdown"],
+  [".markdown", "text/markdown"],
+  [".json", "application/json"],
+  [".jsonl", "application/x-ndjson"],
+  [".ndjson", "application/x-ndjson"],
+  [".csv", "text/csv"],
+  [".tsv", "text/tab-separated-values"],
+  [".html", "text/html"],
+  [".htm", "text/html"],
+  [".xml", "application/xml"],
+  [".yaml", "application/yaml"],
+  [".yml", "application/yaml"],
+  [".css", "text/css"],
+  [".js", "text/javascript"],
+  [".mjs", "text/javascript"],
+  [".cjs", "text/javascript"],
+  [".ts", "text/typescript"],
+  [".tsx", "text/typescript"],
+  [".jsx", "text/javascript"],
+  [".log", "text/plain"],
+  [".toml", "text/plain"],
+  [".ini", "text/plain"],
+  [".conf", "text/plain"],
+  [".sql", "text/plain"],
+  [".py", "text/x-python"],
+  [".rb", "text/plain"],
+  [".go", "text/plain"],
+  [".java", "text/plain"],
+  [".sh", "text/x-shellscript"],
+  [".graphql", "text/plain"]
+]);
+var READABLE_FILENAMES = /* @__PURE__ */ new Set(["readme", "license", "changelog", "makefile", "dockerfile", "gemfile"]);
+var ArchiveReadError = class extends Error {
+  constructor(code, message, status = 400, retryable = false) {
+    super(message);
+    this.code = code;
+    this.status = status;
+    this.retryable = retryable;
+  }
+  code;
+  status;
+  retryable;
+};
+function readableContentType(path5) {
+  const extensionType = READABLE_EXTENSIONS.get(extname(path5).toLowerCase());
+  if (extensionType) return extensionType;
+  return READABLE_FILENAMES.has(basename(path5).toLowerCase()) ? "text/plain" : null;
+}
+function safeEntryPath(path5) {
+  const normalized = path5.replaceAll("\\", "/");
+  if (!normalized || /[\u0000-\u001f\u007f]/.test(normalized) || normalized.startsWith("/") || /^[a-zA-Z]:\//.test(normalized) || normalized.split("/").some((segment) => segment === "..")) {
+    throw new ArchiveReadError("archive_unsafe_path", `ZIP entry has an unsafe path: ${path5}`);
+  }
+  return normalized;
+}
+function entryUnixMode(entry) {
+  return entry.externalFileAttributes >>> 16 & 65535;
+}
+function isSymlink(entry) {
+  return (entryUnixMode(entry) & 61440) === 40960;
+}
+function entryModifiedAt(entry) {
+  try {
+    const value = entry.getLastModDate();
+    return Number.isFinite(value.getTime()) ? value.toISOString() : null;
+  } catch {
+    return null;
+  }
+}
+function openZip(buffer) {
+  return new Promise((resolve, reject) => {
+    yauzl.fromBuffer(buffer, {
+      lazyEntries: true,
+      decodeStrings: true,
+      validateEntrySizes: true,
+      strictFileNames: true
+    }, (error, zip) => {
+      if (error || !zip) {
+        reject(new ArchiveReadError("archive_invalid_zip", "The downloaded file is not a valid ZIP archive."));
+        return;
+      }
+      resolve(zip);
+    });
+  });
+}
+async function scanZip(buffer, visibleEntryLimit) {
+  const zip = await openZip(buffer);
+  return new Promise((resolve, reject) => {
+    const entries = [];
+    const allEntries = [];
+    let totalUncompressedBytes = 0;
+    let settled = false;
+    const fail2 = (error) => {
+      if (settled) return;
+      settled = true;
+      try {
+        zip.close();
+      } catch {
+      }
+      reject(error instanceof ArchiveReadError ? error : new ArchiveReadError("archive_invalid_zip", error instanceof Error ? error.message : "Failed to read ZIP archive."));
+    };
+    zip.on("error", fail2);
+    zip.on("entry", (entry) => {
+      try {
+        if (allEntries.length >= MAX_ARCHIVE_ENTRIES) {
+          fail2(new ArchiveReadError("archive_entry_limit", `ZIP archive exceeds the ${MAX_ARCHIVE_ENTRIES.toLocaleString()} entry limit.`));
+          return;
+        }
+        const path5 = safeEntryPath(entry.fileName);
+        if (entry.isEncrypted()) {
+          fail2(new ArchiveReadError("archive_encrypted_entry", `Encrypted ZIP entries are not supported: ${path5}`));
+          return;
+        }
+        if (isSymlink(entry)) {
+          fail2(new ArchiveReadError("archive_symlink_entry", `Symbolic-link ZIP entries are not supported: ${path5}`));
+          return;
+        }
+        const directory = path5.endsWith("/");
+        totalUncompressedBytes += entry.uncompressedSize;
+        if (totalUncompressedBytes > MAX_ARCHIVE_EXPANDED_BYTES) {
+          fail2(new ArchiveReadError("archive_expanded_size_limit", `ZIP archive exceeds the ${Math.round(MAX_ARCHIVE_EXPANDED_BYTES / 1024 / 1024)} MB expanded-size limit.`));
+          return;
+        }
+        const ratio = entry.compressedSize > 0 ? entry.uncompressedSize / entry.compressedSize : entry.uncompressedSize === 0 ? 1 : Number.POSITIVE_INFINITY;
+        if (!directory && entry.uncompressedSize > 1024 * 1024 && ratio > MAX_COMPRESSION_RATIO) {
+          fail2(new ArchiveReadError("archive_compression_ratio_limit", `ZIP entry exceeds the ${MAX_COMPRESSION_RATIO}:1 compression-ratio limit: ${path5}`));
+          return;
+        }
+        const contentType = directory ? null : readableContentType(path5);
+        const supportedCompression = entry.compressionMethod === 0 || entry.compressionMethod === 8;
+        const info = {
+          path: path5,
+          directory,
+          compressedBytes: entry.compressedSize,
+          uncompressedBytes: entry.uncompressedSize,
+          contentType,
+          readable: contentType !== null && supportedCompression && entry.uncompressedSize <= MAX_ARCHIVE_ENTRY_BYTES,
+          modifiedAt: entryModifiedAt(entry)
+        };
+        allEntries.push(info);
+        if (entries.length < visibleEntryLimit) entries.push(info);
+        zip.readEntry();
+      } catch (error) {
+        fail2(error);
+      }
+    });
+    zip.on("end", () => {
+      if (settled) return;
+      settled = true;
+      resolve({ entries, allEntries, totalUncompressedBytes });
+    });
+    zip.readEntry();
+  });
+}
+function openEntryStream(zip, entry) {
+  return new Promise((resolve, reject) => {
+    zip.openReadStream(entry, (error, stream) => {
+      if (error || !stream) {
+        reject(new ArchiveReadError("archive_entry_read_failed", error?.message ?? "Failed to open ZIP entry."));
+        return;
+      }
+      resolve(stream);
+    });
+  });
+}
+async function collectEntry(stream, expectedBytes) {
+  const chunks = [];
+  let bytes = 0;
+  for await (const chunk of stream) {
+    const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+    bytes += buffer.length;
+    if (bytes > MAX_ARCHIVE_ENTRY_BYTES) {
+      stream.destroy();
+      throw new ArchiveReadError("archive_entry_size_limit", `ZIP entry exceeds the ${Math.round(MAX_ARCHIVE_ENTRY_BYTES / 1024 / 1024)} MB readable-size limit.`);
+    }
+    chunks.push(buffer);
+  }
+  if (bytes !== expectedBytes) {
+    throw new ArchiveReadError("archive_entry_size_mismatch", "ZIP entry expanded to a different size than declared.");
+  }
+  return Buffer.concat(chunks, bytes);
+}
+async function extractEntry(buffer, requestedPath) {
+  const zip = await openZip(buffer);
+  return new Promise((resolve, reject) => {
+    let settled = false;
+    const fail2 = (error) => {
+      if (settled) return;
+      settled = true;
+      try {
+        zip.close();
+      } catch {
+      }
+      reject(error instanceof ArchiveReadError ? error : new ArchiveReadError("archive_entry_read_failed", error instanceof Error ? error.message : "Failed to read ZIP entry."));
+    };
+    zip.on("error", fail2);
+    zip.on("entry", (entry) => {
+      let path5;
+      try {
+        path5 = safeEntryPath(entry.fileName);
+      } catch (error) {
+        fail2(error);
+        return;
+      }
+      if (path5 !== requestedPath) {
+        zip.readEntry();
+        return;
+      }
+      if (path5.endsWith("/")) {
+        fail2(new ArchiveReadError("archive_entry_is_directory", `ZIP entry is a directory: ${path5}`));
+        return;
+      }
+      if (entry.uncompressedSize > MAX_ARCHIVE_ENTRY_BYTES) {
+        fail2(new ArchiveReadError("archive_entry_size_limit", `ZIP entry exceeds the ${Math.round(MAX_ARCHIVE_ENTRY_BYTES / 1024 / 1024)} MB readable-size limit.`));
+        return;
+      }
+      if (!readableContentType(path5)) {
+        fail2(new ArchiveReadError("archive_binary_entry", `ZIP entry is not a supported text file: ${path5}`));
+        return;
+      }
+      void openEntryStream(zip, entry).then((stream) => collectEntry(stream, entry.uncompressedSize)).then((content) => {
+        if (settled) return;
+        settled = true;
+        try {
+          zip.close();
+        } catch {
+        }
+        resolve({ entry, content });
+      }).catch(fail2);
+    });
+    zip.on("end", () => fail2(new ArchiveReadError("archive_entry_not_found", `ZIP entry was not found: ${requestedPath}`, 404)));
+    zip.readEntry();
+  });
+}
+async function boundedResponseBuffer(response) {
+  const declaredLength = Number(response.headers.get("content-length"));
+  if (Number.isFinite(declaredLength) && declaredLength > MAX_ARCHIVE_DOWNLOAD_BYTES) {
+    throw new ArchiveReadError("archive_download_size_limit", `ZIP download exceeds the ${Math.round(MAX_ARCHIVE_DOWNLOAD_BYTES / 1024 / 1024)} MB limit.`);
+  }
+  if (!response.body) throw new ArchiveReadError("archive_empty_response", "ZIP download returned no response body.", 502, true);
+  const reader = response.body.getReader();
+  const chunks = [];
+  let bytes = 0;
+  try {
+    for (; ; ) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (!value) continue;
+      bytes += value.byteLength;
+      if (bytes > MAX_ARCHIVE_DOWNLOAD_BYTES) {
+        await reader.cancel();
+        throw new ArchiveReadError("archive_download_size_limit", `ZIP download exceeds the ${Math.round(MAX_ARCHIVE_DOWNLOAD_BYTES / 1024 / 1024)} MB limit.`);
+      }
+      chunks.push(value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  const buffer = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk)), bytes);
+  if (buffer.length < 4 || buffer[0] !== 80 || buffer[1] !== 75) {
+    throw new ArchiveReadError("archive_invalid_zip", "The downloaded file is not a ZIP archive.");
+  }
+  return buffer;
+}
+async function downloadPublicZip(rawUrl) {
+  let currentUrl = rawUrl;
+  for (let redirectCount = 0; redirectCount <= MAX_ARCHIVE_REDIRECTS; redirectCount++) {
+    const checked = await validatePublicHttpUrl(currentUrl, { field: "archive URL", requireHttps: true });
+    if (checked.error || !checked.parsed) {
+      throw new ArchiveReadError("archive_url_not_public", checked.error ?? "Invalid archive URL.");
+    }
+    if (checked.parsed.username || checked.parsed.password) {
+      throw new ArchiveReadError("archive_url_credentials_forbidden", "Archive URL must not contain embedded credentials.");
+    }
+    const response = await fetch(checked.parsed.href, {
+      method: "GET",
+      redirect: "manual",
+      headers: { accept: "application/zip, application/octet-stream;q=0.9" },
+      signal: AbortSignal.timeout(6e4)
+    });
+    if ([301, 302, 303, 307, 308].includes(response.status)) {
+      const location2 = response.headers.get("location");
+      try {
+        await response.body?.cancel();
+      } catch {
+      }
+      if (!location2) throw new ArchiveReadError("archive_redirect_missing_location", "Archive server returned a redirect without a location.", 502, true);
+      currentUrl = new URL(location2, checked.parsed).href;
+      continue;
+    }
+    if (!response.ok) {
+      try {
+        await response.body?.cancel();
+      } catch {
+      }
+      throw new ArchiveReadError(
+        "archive_download_failed",
+        `Archive download returned HTTP ${response.status}.`,
+        response.status >= 500 ? 502 : 400,
+        response.status >= 500 || response.status === 429
+      );
+    }
+    return { archiveUrl: checked.parsed.href, buffer: await boundedResponseBuffer(response) };
+  }
+  throw new ArchiveReadError("archive_redirect_limit", `Archive download exceeded ${MAX_ARCHIVE_REDIRECTS} redirects.`);
+}
+async function listZipArchive(rawUrl, maxEntries) {
+  const { archiveUrl, buffer } = await downloadPublicZip(rawUrl);
+  const scanned = await scanZip(buffer, maxEntries);
+  return {
+    archiveUrl,
+    compressedBytes: buffer.length,
+    entryCount: scanned.allEntries.length,
+    totalUncompressedBytes: scanned.totalUncompressedBytes,
+    entries: scanned.entries,
+    entriesTruncated: scanned.allEntries.length > scanned.entries.length
+  };
+}
+function decodeUtf8(content, path5) {
+  try {
+    return new TextDecoder("utf-8", { fatal: true }).decode(content);
+  } catch {
+    throw new ArchiveReadError("archive_text_decode_failed", `ZIP entry is not valid UTF-8 text: ${path5}`);
+  }
+}
+function utf8Window(content, path5, offset, maxBytes) {
+  const start = Math.min(offset, content.length);
+  if (start < content.length && (content[start] & 192) === 128) {
+    throw new ArchiveReadError(
+      "archive_offset_invalid",
+      `Byte offset ${start} falls inside a UTF-8 character in ZIP entry: ${path5}`
+    );
+  }
+  let end = Math.min(content.length, start + maxBytes);
+  while (end > start && end < content.length && (content[end] & 192) === 128) end--;
+  if (end === start && start < content.length) {
+    throw new ArchiveReadError(
+      "archive_window_too_small",
+      `maxBytes is too small for the next UTF-8 character in ZIP entry: ${path5}`
+    );
+  }
+  return {
+    content: content.subarray(start, end).toString("utf8"),
+    offset: start,
+    nextOffset: end < content.length ? end : null
+  };
+}
+async function readZipArchiveFile(rawUrl, rawPath, offset, maxBytes) {
+  const requestedPath = safeEntryPath(rawPath);
+  const { archiveUrl, buffer } = await downloadPublicZip(rawUrl);
+  const scanned = await scanZip(buffer, 0);
+  const info = scanned.allEntries.find((entry) => entry.path === requestedPath);
+  if (!info) throw new ArchiveReadError("archive_entry_not_found", `ZIP entry was not found: ${requestedPath}`, 404);
+  if (!info.readable) {
+    if (info.directory) throw new ArchiveReadError("archive_entry_is_directory", `ZIP entry is a directory: ${requestedPath}`);
+    if (info.uncompressedBytes > MAX_ARCHIVE_ENTRY_BYTES) {
+      throw new ArchiveReadError("archive_entry_size_limit", `ZIP entry exceeds the ${Math.round(MAX_ARCHIVE_ENTRY_BYTES / 1024 / 1024)} MB readable-size limit.`);
+    }
+    throw new ArchiveReadError("archive_binary_entry", `ZIP entry is not a supported text file: ${requestedPath}`);
+  }
+  const { content } = await extractEntry(buffer, requestedPath);
+  const fullContent = decodeUtf8(content, requestedPath);
+  const textBytes = Buffer.from(fullContent);
+  const window2 = utf8Window(textBytes, requestedPath, offset, maxBytes);
+  return {
+    archiveUrl,
+    compressedBytes: buffer.length,
+    entryCount: scanned.allEntries.length,
+    totalUncompressedBytes: scanned.totalUncompressedBytes,
+    path: requestedPath,
+    contentType: info.contentType,
+    fileBytes: textBytes.length,
+    offset: window2.offset,
+    content: window2.content,
+    nextOffset: window2.nextOffset,
+    fullContent
+  };
+}
+function archiveLibrarySource(archiveUrl, path5) {
+  const source = new URL(archiveUrl);
+  source.username = "";
+  source.password = "";
+  source.search = "";
+  source.hash = `entry=${encodeURIComponent(path5)}`;
+  return source.href;
+}
+function archiveEntryTitle(path5) {
+  return basename(path5).slice(0, 200) || "Archive entry";
+}
+function inferredWaybackCapturedAt(content) {
+  const timestamp = content.slice(0, 4e3).match(/^Archive timestamp:\s*(\d{14})\s*$/m)?.[1];
+  if (!timestamp) return void 0;
+  const iso = `${timestamp.slice(0, 4)}-${timestamp.slice(4, 6)}-${timestamp.slice(6, 8)}T${timestamp.slice(8, 10)}:${timestamp.slice(10, 12)}:${timestamp.slice(12, 14)}Z`;
+  return Number.isFinite(Date.parse(iso)) ? iso : void 0;
+}
+
 // src/api/connection-memory-import.ts
 import { createHash as createHash10 } from "crypto";
 var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
@@ -31979,6 +32410,69 @@ app.post("/diff-page", auth2, async (c) => {
     await releaseConcurrencyGate(gate.lockId);
   }
 });
+app.post("/archive/read", auth2, async (c) => {
+  const raw = await c.req.json().catch(() => ({}));
+  const bodyResult = ArchiveReadBodySchema.safeParse(raw);
+  if (!bodyResult.success) {
+    return c.json({
+      error: "archive_invalid_request",
+      error_code: "archive_invalid_request",
+      retryable: false,
+      message: bodyResult.error.issues[0]?.message ?? "Invalid request"
+    }, 400);
+  }
+  const { url, path: path5, depositToLibrary } = bodyResult.data;
+  const user = c.get("user");
+  const gate = await acquireConcurrencyGate(user, "archive_read", {
+    reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"),
+    metadata: { url }
+  });
+  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
+  try {
+    if (!path5) {
+      const inventory = await listZipArchive(url, bodyResult.data.maxEntries ?? 200);
+      return c.json({ mode: "list", ...inventory });
+    }
+    const file = await readZipArchiveFile(
+      url,
+      path5,
+      bodyResult.data.offset ?? 0,
+      bodyResult.data.maxBytes ?? 5e4
+    );
+    if (depositToLibrary && file.fileBytes > MAX_LIBRARY_ENTRY_BYTES) {
+      throw new ArchiveReadError(
+        "archive_library_size_limit",
+        `ZIP entry exceeds the ${Math.round(MAX_LIBRARY_ENTRY_BYTES / 1024 / 1024)} MB Library-ingest limit.`
+      );
+    }
+    const librarySource = archiveLibrarySource(file.archiveUrl, file.path);
+    const memory = depositToLibrary ? await depositScrapeToVault(user, {
+      title: archiveEntryTitle(file.path),
+      content: file.fullContent,
+      source: librarySource,
+      vault: "Library",
+      capturedAt: inferredWaybackCapturedAt(file.fullContent),
+      summary: `Source file \`${file.path}\` preserved from ZIP archive ${librarySource.split("#")[0]}.`
+    }) : void 0;
+    const { fullContent: _fullContent, ...responseFile } = file;
+    return c.json({ mode: "read", ...responseFile, memory });
+  } catch (error) {
+    const known = error instanceof ArchiveReadError ? error : new ArchiveReadError(
+      "archive_read_failed",
+      error instanceof Error ? error.message : "Failed to read ZIP archive.",
+      500,
+      true
+    );
+    return c.json({
+      error: known.code,
+      error_code: known.code,
+      retryable: known.retryable,
+      message: known.message
+    }, known.status);
+  } finally {
+    await releaseConcurrencyGate(gate.lockId);
+  }
+});
 app.post("/map-urls", auth2, async (c) => {
   const raw = await c.req.json().catch(() => ({}));
   const bodyResult = MapUrlsBodySchema.safeParse(raw);
@@ -32931,4 +33425,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-QKDDEBVQ.js.map
+//# sourceMappingURL=server-3Q6XIJPF.js.map