mcp-scraper 0.88.2 → 0.89.0

package/dist/chunk-DNM65UCK.js

@@ -1,299 +0,0 @@
-// src/api/private-artifacts.ts
-import { createHash } from "crypto";
-import { mkdir, readFile, writeFile } from "fs/promises";
-import { createWriteStream } from "fs";
-import { homedir } from "os";
-import { dirname, join } from "path";
-import { Transform } from "stream";
-import { pipeline } from "stream/promises";
-var DURABLE_ARTIFACT_EXPIRES_AT = "9999-12-31T23:59:59.999Z";
-function hostedByEnvironment() {
-  return process.env.VERCEL === "1" || process.env.NODE_ENV === "production";
-}
-function policyHosted(policy) {
-  return policy.hosted ?? hostedByEnvironment();
-}
-function policyLocalBaseDir(policy) {
-  return policy.localBaseDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join(homedir(), "Downloads", "mcp-scraper");
-}
-function normalizedPrefix(prefix) {
-  const normalized = prefix.replace(/^\/+/, "").replace(/\/+$/, "");
-  if (!normalized || normalized.includes("..") || !/^[a-zA-Z0-9/_-]+$/.test(normalized)) {
-    throw new Error("private artifact prefix is invalid");
-  }
-  return `${normalized}/`;
-}
-function requiredSafeSegment(value, field) {
-  const trimmed = value.trim();
-  if (!/^[a-zA-Z0-9_-]{1,160}$/.test(trimmed)) {
-    throw new Error(`${field} must contain only letters, numbers, underscores, or hyphens`);
-  }
-  return trimmed;
-}
-function requiredSafeArtifactKey(value) {
-  const trimmed = value.trim();
-  if (!/^[a-zA-Z0-9_.-]{1,200}$/.test(trimmed) || trimmed === "." || trimmed === "..") {
-    throw new Error("private artifact key is invalid");
-  }
-  return trimmed;
-}
-function safeDownloadFilename(value) {
-  const normalized = value.trim().replace(/[^a-zA-Z0-9_.-]+/g, "-").replace(/^-+|-+$/g, "");
-  return (normalized || "artifact").slice(0, 160);
-}
-function createdAtMs(value) {
-  const normalized = typeof value === "string" && /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(value) ? `${value.replace(" ", "T")}Z` : value;
-  const timestamp = normalized instanceof Date ? normalized.getTime() : typeof normalized === "number" ? normalized : new Date(normalized).getTime();
-  if (!Number.isSafeInteger(timestamp) || timestamp <= 0) throw new Error("private artifact createdAt is invalid");
-  return timestamp;
-}
-function validatePolicy(policy) {
-  if (policy.artifactTtlMs !== null && (!Number.isSafeInteger(policy.artifactTtlMs) || policy.artifactTtlMs <= 0)) {
-    throw new Error("private artifact TTL must be a positive integer");
-  }
-  if (!Number.isSafeInteger(policy.downloadTtlMs) || policy.downloadTtlMs <= 0) {
-    throw new Error("private artifact download TTL must be a positive integer");
-  }
-  return {
-    prefix: normalizedPrefix(policy.prefix),
-    artifactTtlMs: policy.artifactTtlMs,
-    downloadTtlMs: policy.downloadTtlMs
-  };
-}
-function artifactTimestamp(artifactId, prefix) {
-  const normalized = normalizedPrefix(prefix);
-  if (!artifactId.startsWith(normalized)) return null;
-  const filename = artifactId.split("/").at(-1) ?? "";
-  const match = filename.match(/^(\d{13})-/);
-  if (!match) return null;
-  const timestamp = Number(match[1]);
-  return Number.isSafeInteger(timestamp) && timestamp > 0 ? timestamp : null;
-}
-function privateArtifactOwnerId(artifactId, prefix) {
-  const normalized = normalizedPrefix(prefix);
-  if (!artifactId.startsWith(normalized) || artifactId.includes("..")) return null;
-  const rest = artifactId.slice(normalized.length);
-  const segments = rest.split("/");
-  if (segments.length < 2) return null;
-  try {
-    return requiredSafeSegment(segments[0] ?? "", "ownerId");
-  } catch {
-    return null;
-  }
-}
-function privateArtifactExpiresAt(artifactId, policy) {
-  const validated = validatePolicy(policy);
-  if (privateArtifactOwnerId(artifactId, validated.prefix) === null) return null;
-  if (validated.artifactTtlMs === null) return null;
-  const timestamp = artifactTimestamp(artifactId, validated.prefix);
-  return timestamp === null ? null : new Date(timestamp + validated.artifactTtlMs);
-}
-async function signedDownloadUrl(pathname, expiresAt, policy) {
-  const token = policy.token?.trim();
-  if (!token) return null;
-  const validUntil = expiresAt ? Math.min(Date.now() + policy.downloadTtlMs, expiresAt.getTime()) : Date.now() + policy.downloadTtlMs;
-  if (validUntil <= Date.now()) return null;
-  const { issueSignedToken, presignUrl } = await import("@vercel/blob");
-  const signedToken = await issueSignedToken({
-    token,
-    pathname,
-    operations: ["get"],
-    validUntil
-  });
-  const { presignedUrl } = await presignUrl(signedToken, {
-    access: "private",
-    operation: "get",
-    pathname,
-    validUntil
-  });
-  return { url: presignedUrl, expiresAt: new Date(validUntil).toISOString() };
-}
-async function createPrivateArtifact(args) {
-  const policy = { ...args.policy, ...validatePolicy(args.policy) };
-  const ownerId = requiredSafeSegment(args.ownerId, "ownerId");
-  const artifactKey = requiredSafeArtifactKey(args.artifactKey);
-  const scope = (args.scopeSegments ?? []).map((segment, index) => requiredSafeSegment(segment, `scopeSegments[${index}]`));
-  const timestamp = createdAtMs(args.createdAt);
-  const requestedPathname = `${policy.prefix}${ownerId}/${scope.length ? `${scope.join("/")}/` : ""}${timestamp}-${artifactKey}`;
-  const body = Buffer.isBuffer(args.content) ? args.content : Buffer.from(args.content);
-  const bytes = body.length;
-  const sha256 = createHash("sha256").update(body).digest("hex");
-  const expiresAt = policy.artifactTtlMs === null ? null : new Date(timestamp + policy.artifactTtlMs);
-  const token = policy.token?.trim() || null;
-  let artifactId = requestedPathname;
-  if (token) {
-    const { put } = await import("@vercel/blob");
-    const stored = await put(requestedPathname, body, {
-      access: "private",
-      token,
-      contentType: args.contentType,
-      addRandomSuffix: false,
-      allowOverwrite: true,
-      cacheControlMaxAge: 60,
-      multipart: bytes > 100 * 1024 * 1024
-    });
-    artifactId = stored.pathname;
-  } else {
-    if (policyHosted(policy)) throw new Error("private_artifact_blob_not_configured");
-    const path = join(policyLocalBaseDir(policy), "blobs", requestedPathname);
-    await mkdir(dirname(path), { recursive: true });
-    await writeFile(path, body);
-  }
-  const download = await signedDownloadUrl(artifactId, expiresAt, policy);
-  return {
-    artifactId,
-    filename: safeDownloadFilename(args.filename),
-    contentType: args.contentType,
-    bytes,
-    sha256,
-    expiresAt: expiresAt?.toISOString() ?? DURABLE_ARTIFACT_EXPIRES_AT,
-    downloadUrl: download?.url ?? null,
-    downloadUrlExpiresAt: download?.expiresAt ?? null
-  };
-}
-async function createPrivateArtifactFromStream(args) {
-  const policy = { ...args.policy, ...validatePolicy(args.policy) };
-  const ownerId = requiredSafeSegment(args.ownerId, "ownerId");
-  const artifactKey = requiredSafeArtifactKey(args.artifactKey);
-  const scope = (args.scopeSegments ?? []).map((segment, index) => requiredSafeSegment(segment, `scopeSegments[${index}]`));
-  const timestamp = createdAtMs(args.createdAt);
-  const requestedPathname = `${policy.prefix}${ownerId}/${scope.length ? `${scope.join("/")}/` : ""}${timestamp}-${artifactKey}`;
-  const expiresAt = policy.artifactTtlMs === null ? null : new Date(timestamp + policy.artifactTtlMs);
-  const token = policy.token?.trim() || null;
-  const hash = createHash("sha256");
-  let bytes = 0;
-  const meter = new Transform({
-    transform(chunk, _encoding, callback) {
-      bytes += chunk.length;
-      hash.update(chunk);
-      callback(null, chunk);
-    }
-  });
-  let artifactId = requestedPathname;
-  if (token) {
-    const { put } = await import("@vercel/blob");
-    const pumping = pipeline(args.content, meter);
-    try {
-      const stored = await put(requestedPathname, meter, {
-        access: "private",
-        token,
-        contentType: args.contentType,
-        addRandomSuffix: false,
-        allowOverwrite: true,
-        cacheControlMaxAge: 60,
-        multipart: true
-      });
-      await pumping;
-      artifactId = stored.pathname;
-    } catch (error) {
-      args.content.destroy(error instanceof Error ? error : new Error(String(error)));
-      meter.destroy(error instanceof Error ? error : new Error(String(error)));
-      await pumping.catch(() => void 0);
-      throw error;
-    }
-  } else {
-    if (policyHosted(policy)) throw new Error("private_artifact_blob_not_configured");
-    const path = join(policyLocalBaseDir(policy), "blobs", requestedPathname);
-    await mkdir(dirname(path), { recursive: true });
-    await pipeline(args.content, meter, createWriteStream(path));
-  }
-  const sha256 = hash.digest("hex");
-  const download = await signedDownloadUrl(artifactId, expiresAt, policy);
-  return {
-    artifactId,
-    filename: safeDownloadFilename(args.filename),
-    contentType: args.contentType,
-    bytes,
-    sha256,
-    expiresAt: expiresAt?.toISOString() ?? DURABLE_ARTIFACT_EXPIRES_AT,
-    downloadUrl: download?.url ?? null,
-    downloadUrlExpiresAt: download?.expiresAt ?? null
-  };
-}
-async function renewPrivateArtifactDownload(args) {
-  if (privateArtifactOwnerId(args.artifactId, args.policy.prefix) !== args.ownerId) return null;
-  const expiresAt = privateArtifactExpiresAt(args.artifactId, args.policy);
-  if (args.policy.artifactTtlMs !== null && (!expiresAt || expiresAt.getTime() <= Date.now())) return null;
-  const download = await signedDownloadUrl(args.artifactId, expiresAt, args.policy);
-  if (!download) return null;
-  return {
-    downloadUrl: download.url,
-    downloadUrlExpiresAt: download.expiresAt,
-    expiresAt: expiresAt?.toISOString() ?? DURABLE_ARTIFACT_EXPIRES_AT
-  };
-}
-async function streamToBuffer(stream) {
-  const reader = stream.getReader();
-  const chunks = [];
-  try {
-    for (; ; ) {
-      const { done, value } = await reader.read();
-      if (done) break;
-      if (value) chunks.push(Buffer.from(value));
-    }
-  } finally {
-    reader.releaseLock();
-  }
-  return Buffer.concat(chunks);
-}
-async function readPrivateArtifactWindow(args) {
-  if (privateArtifactOwnerId(args.artifactId, args.policy.prefix) === null) return null;
-  const expiresAt = privateArtifactExpiresAt(args.artifactId, args.policy);
-  if (args.policy.artifactTtlMs !== null && (!expiresAt || expiresAt.getTime() <= Date.now())) return null;
-  if (!Number.isSafeInteger(args.offset) || args.offset < 0) throw new Error("private artifact offset is invalid");
-  if (!Number.isSafeInteger(args.maxBytes) || args.maxBytes <= 0) throw new Error("private artifact maxBytes is invalid");
-  const token = args.policy.token?.trim() || null;
-  let buffer;
-  if (token) {
-    const { get } = await import("@vercel/blob");
-    const result = await get(args.artifactId, { access: "private", token, useCache: false });
-    if (!result || result.statusCode !== 200) return null;
-    buffer = await streamToBuffer(result.stream);
-  } else {
-    if (policyHosted(args.policy)) return null;
-    try {
-      buffer = await readFile(join(policyLocalBaseDir(args.policy), "blobs", args.artifactId));
-    } catch {
-      return null;
-    }
-  }
-  const totalBytes = buffer.length;
-  const end = Math.min(totalBytes, args.offset + args.maxBytes);
-  return {
-    text: buffer.subarray(args.offset, end).toString("utf8"),
-    totalBytes,
-    nextOffset: end < totalBytes ? end : null
-  };
-}
-async function readPrivateArtifactBuffer(args) {
-  if (privateArtifactOwnerId(args.artifactId, args.policy.prefix) === null) return null;
-  const expiresAt = privateArtifactExpiresAt(args.artifactId, args.policy);
-  if (args.policy.artifactTtlMs !== null && (!expiresAt || expiresAt.getTime() <= Date.now())) return null;
-  if (!Number.isSafeInteger(args.maxBytes) || args.maxBytes <= 0) throw new Error("private artifact maxBytes is invalid");
-  const token = args.policy.token?.trim() || null;
-  let buffer;
-  if (token) {
-    const { get } = await import("@vercel/blob");
-    const result = await get(args.artifactId, { access: "private", token, useCache: false });
-    if (!result || result.statusCode !== 200) return null;
-    buffer = await streamToBuffer(result.stream);
-  } else {
-    if (policyHosted(args.policy)) return null;
-    try {
-      buffer = await readFile(join(policyLocalBaseDir(args.policy), "blobs", args.artifactId));
-    } catch {
-      return null;
-    }
-  }
-  if (buffer.length > args.maxBytes) throw new Error("private artifact exceeds read limit");
-  return buffer;
-}
-
-export {
-  privateArtifactOwnerId,
-  createPrivateArtifact,
-  createPrivateArtifactFromStream,
-  renewPrivateArtifactDownload,
-  readPrivateArtifactWindow,
-  readPrivateArtifactBuffer
-};