mcp-scraper 0.88.2 → 0.89.0

package/dist/chunk-T3MZISOF.js

@@ -1,240 +0,0 @@
-// src/api/connected-data-artifacts.ts
-import { createHash, randomUUID } from "crypto";
-import { mkdir, readFile, stat, writeFile } from "fs/promises";
-import { homedir } from "os";
-import { dirname, join } from "path";
-var CONNECTED_DATA_ARTIFACT_PREFIX = "connected-data-exports/";
-var CONNECTED_DATA_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
-var CONNECTED_DATA_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
-function privateBlobToken() {
-  return process.env.CONNECTED_DATA_READ_WRITE_TOKEN?.trim() || process.env.CONNECTED_DATA_BLOB_READ_WRITE_TOKEN?.trim() || null;
-}
-function localBaseDir() {
-  return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join(homedir(), "Downloads", "mcp-scraper");
-}
-function isHosted() {
-  return process.env.VERCEL === "1" || process.env.NODE_ENV === "production";
-}
-function safeFilename(value) {
-  const safe = value.toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
-  return (safe || "connected-data-export").slice(0, 120);
-}
-function artifactTimestamp(artifactId) {
-  if (!artifactId.startsWith(CONNECTED_DATA_ARTIFACT_PREFIX)) return null;
-  const filename = artifactId.split("/").at(-1) ?? "";
-  const match = filename.match(/^(\d{13})-/);
-  if (!match) return null;
-  const timestamp = Number(match[1]);
-  return Number.isFinite(timestamp) ? timestamp : null;
-}
-function connectedDataArtifactOwnerId(artifactId) {
-  if (!artifactId.startsWith(CONNECTED_DATA_ARTIFACT_PREFIX)) return null;
-  const rest = artifactId.slice(CONNECTED_DATA_ARTIFACT_PREFIX.length);
-  const owner = rest.split("/")[0];
-  return owner || null;
-}
-function connectedDataArtifactExpiresAt(artifactId) {
-  const timestamp = artifactTimestamp(artifactId);
-  return timestamp === null ? null : new Date(timestamp + CONNECTED_DATA_ARTIFACT_TTL_MS);
-}
-function connectedDataArtifactIsExpired(artifactId, now = Date.now()) {
-  const timestamp = artifactTimestamp(artifactId);
-  return timestamp === null || timestamp + CONNECTED_DATA_ARTIFACT_TTL_MS <= now;
-}
-async function signedDownloadUrl(pathname, expiresAt) {
-  const token = privateBlobToken();
-  if (!token) return null;
-  const validUntil = Math.min(Date.now() + CONNECTED_DATA_DOWNLOAD_TTL_MS, expiresAt.getTime());
-  if (validUntil <= Date.now()) return null;
-  const { issueSignedToken, presignUrl } = await import("@vercel/blob");
-  const signedToken = await issueSignedToken({
-    token,
-    pathname,
-    operations: ["get"],
-    validUntil
-  });
-  const { presignedUrl } = await presignUrl(signedToken, {
-    access: "private",
-    operation: "get",
-    pathname,
-    validUntil
-  });
-  return { url: presignedUrl, expiresAt: new Date(validUntil).toISOString() };
-}
-async function createConnectedDataArtifact(args) {
-  return createConnectedDataBinaryArtifact({
-    ...args,
-    contentType: "application/x-ndjson"
-  });
-}
-async function createConnectedDataBinaryArtifact(args) {
-  const requestedCreatedAt = args.stableCreatedAt ? Date.parse(args.stableCreatedAt) : Number.NaN;
-  const createdAt = Number.isFinite(requestedCreatedAt) ? requestedCreatedAt : Date.now();
-  const filename = safeFilename(args.filename);
-  const extension = filename.includes(".") ? `.${filename.split(".").at(-1)}` : ".bin";
-  const stableSuffix = args.stableKey ? safeFilename(args.stableKey) : randomUUID();
-  const requestedPathname = `${CONNECTED_DATA_ARTIFACT_PREFIX}${args.ownerId}/${createdAt}-${safeFilename(args.exportId)}-${stableSuffix}--${filename}`;
-  const content = typeof args.content === "string" ? Buffer.from(args.content) : Buffer.from(args.content);
-  const bytes = content.byteLength;
-  const sha256 = createHash("sha256").update(content).digest("hex");
-  const expiresAt = new Date(createdAt + CONNECTED_DATA_ARTIFACT_TTL_MS);
-  const token = privateBlobToken();
-  let artifactId = requestedPathname;
-  if (token) {
-    const { put } = await import("@vercel/blob");
-    const stored = await put(requestedPathname, content, {
-      access: "private",
-      token,
-      contentType: args.contentType,
-      addRandomSuffix: !args.stableKey,
-      ...args.stableKey ? { allowOverwrite: true } : {},
-      cacheControlMaxAge: 60,
-      multipart: bytes > 100 * 1024 * 1024
-    });
-    artifactId = stored.pathname;
-  } else {
-    if (isHosted()) throw new Error("connected_data_private_blob_not_configured");
-    const path = join(localBaseDir(), "blobs", requestedPathname);
-    await mkdir(dirname(path), { recursive: true });
-    await writeFile(path, content);
-  }
-  const download = await signedDownloadUrl(artifactId, expiresAt);
-  return {
-    artifactId,
-    filename,
-    contentType: args.contentType,
-    bytes,
-    sha256,
-    createdAt: new Date(createdAt).toISOString(),
-    expiresAt: expiresAt.toISOString(),
-    downloadUrl: download?.url ?? null,
-    downloadUrlExpiresAt: download?.expiresAt ?? null,
-    readback: {
-      tool: "report_artifact_read",
-      arguments: { artifactId, offset: 0, maxBytes: 2e4 },
-      continuation: "Repeat with offset set to the previous result nextOffset until nextOffset is null."
-    }
-  };
-}
-async function renewConnectedDataArtifactDownload(args) {
-  if (connectedDataArtifactOwnerId(args.artifactId) !== args.ownerId) return null;
-  const expiresAt = connectedDataArtifactExpiresAt(args.artifactId);
-  if (!expiresAt || expiresAt.getTime() <= Date.now()) return null;
-  const download = await signedDownloadUrl(args.artifactId, expiresAt);
-  if (!download) return null;
-  return {
-    downloadUrl: download.url,
-    downloadUrlExpiresAt: download.expiresAt,
-    expiresAt: expiresAt.toISOString()
-  };
-}
-async function streamToBuffer(stream, maxBytes = Number.POSITIVE_INFINITY) {
-  const reader = stream.getReader();
-  const chunks = [];
-  let totalBytes = 0;
-  try {
-    for (; ; ) {
-      const { done, value } = await reader.read();
-      if (done) break;
-      if (value) {
-        const chunk = Buffer.from(value);
-        totalBytes += chunk.byteLength;
-        if (totalBytes > maxBytes) {
-          await reader.cancel("artifact_too_large").catch(() => void 0);
-          return null;
-        }
-        chunks.push(chunk);
-      }
-    }
-  } finally {
-    reader.releaseLock();
-  }
-  return Buffer.concat(chunks);
-}
-async function readConnectedDataArtifactWindow(artifactId, offset, maxBytes) {
-  if (connectedDataArtifactIsExpired(artifactId)) return null;
-  const token = privateBlobToken();
-  let buffer;
-  if (token) {
-    const { get } = await import("@vercel/blob");
-    const result = await get(artifactId, { access: "private", token, useCache: false });
-    if (!result || result.statusCode !== 200) return null;
-    const streamed = await streamToBuffer(result.stream);
-    if (!streamed) return null;
-    buffer = streamed;
-  } else {
-    if (isHosted()) return null;
-    try {
-      buffer = await readFile(join(localBaseDir(), "blobs", artifactId));
-    } catch {
-      return null;
-    }
-  }
-  const totalBytes = buffer.length;
-  const start = Math.max(0, offset);
-  const end = Math.min(totalBytes, start + maxBytes);
-  return {
-    text: buffer.subarray(start, end).toString("utf8"),
-    totalBytes,
-    nextOffset: end < totalBytes ? end : null
-  };
-}
-async function readConnectedDataArtifactBytes(args) {
-  if (connectedDataArtifactOwnerId(args.artifactId) !== args.ownerId || connectedDataArtifactIsExpired(args.artifactId)) return null;
-  const token = privateBlobToken();
-  let buffer;
-  if (token) {
-    const { get } = await import("@vercel/blob");
-    const result = await get(args.artifactId, { access: "private", token, useCache: false });
-    if (!result || result.statusCode !== 200) return null;
-    const streamed = await streamToBuffer(result.stream, args.maxBytes ?? 25 * 1024 * 1024);
-    if (!streamed) return null;
-    buffer = streamed;
-  } else {
-    if (isHosted()) return null;
-    try {
-      const path = join(localBaseDir(), "blobs", args.artifactId);
-      const info = await stat(path);
-      if (info.size > (args.maxBytes ?? 25 * 1024 * 1024)) return null;
-      buffer = await readFile(path);
-    } catch {
-      return null;
-    }
-  }
-  if (buffer.byteLength > (args.maxBytes ?? 25 * 1024 * 1024)) return null;
-  return buffer;
-}
-async function cleanupExpiredConnectedDataArtifacts(args = {}) {
-  const now = args.now ?? /* @__PURE__ */ new Date();
-  const token = privateBlobToken();
-  if (!token) return { deleted: 0, store: isHosted() ? "none" : "local" };
-  if (!args.force && !(now.getUTCHours() === 3 && now.getUTCMinutes() === 17)) {
-    return { deleted: 0, store: "private-vercel-blob", skipped: true };
-  }
-  const cutoff = now.getTime() - CONNECTED_DATA_ARTIFACT_TTL_MS;
-  const { list, del } = await import("@vercel/blob");
-  let cursor;
-  let deleted = 0;
-  for (let page = 0; page < 20; page++) {
-    const result = await list({ prefix: CONNECTED_DATA_ARTIFACT_PREFIX, token, limit: 1e3, cursor });
-    const expired = result.blobs.filter((blob) => new Date(blob.uploadedAt).getTime() <= cutoff);
-    if (expired.length > 0) {
-      await del(expired.map((blob) => blob.pathname), { token });
-      deleted += expired.length;
-    }
-    if (!result.hasMore || !result.cursor) break;
-    cursor = result.cursor;
-  }
-  return { deleted, store: "private-vercel-blob" };
-}
-
-export {
-  CONNECTED_DATA_ARTIFACT_PREFIX,
-  connectedDataArtifactOwnerId,
-  createConnectedDataArtifact,
-  createConnectedDataBinaryArtifact,
-  renewConnectedDataArtifactDownload,
-  readConnectedDataArtifactWindow,
-  readConnectedDataArtifactBytes,
-  cleanupExpiredConnectedDataArtifacts
-};