@cedarjs/uploads 8.0.0-canary.3429 → 8.0.0-canary.3433

package/dist/variants.js

@@ -1,500 +0,0 @@
-import { randomUUID } from "node:crypto";
-import { setTimeout as sleep } from "node:timers/promises";
-import mime from "mime-types";
-import { UploadError } from "./errors.js";
-import { generateStorageKey } from "./keys.js";
-import { DB_MAX_FILE_SIZE } from "./profiles.js";
-import { toDataUri } from "./serialize.js";
-import { resolveTarget } from "./targets.js";
-const ONE_DAY = 24 * 60 * 60 * 1e3;
-const TEN_MINUTES = 10 * 60 * 1e3;
-const MIME_TYPE = /^[a-z0-9][a-z0-9!#$&^_.+-]*\/[a-z0-9][a-z0-9!#$&^_.+-]*$/i;
-const VARIANT_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
-function idOf(source) {
-  return typeof source === "string" ? source : source.id;
-}
-function isPrismaError(e, code) {
-  return typeof e === "object" && e !== null && "code" in e && e.code === code;
-}
-function defaultFilename(sourceFilename, name, mimeType) {
-  const base = sourceFilename.replace(/\.[^./\\]*$/, "") || "file";
-  const ext = mimeType ? mime.extension(mimeType.split(";")[0].trim()) : false;
-  return ext ? `${base}-${name}.${ext}` : `${base}-${name}`;
-}
-async function readBytes(db, targets, upload) {
-  const target = resolveTarget(targets, upload.target);
-  if (target.providerType === "db") {
-    const row = await db.upload.findUnique({ where: { id: upload.id } });
-    if (!row?.data) {
-      throw new UploadError(
-        "NOT_FOUND",
-        `Upload '${upload.id}' has no inline data.`
-      );
-    }
-    return Buffer.from(row.data);
-  }
-  if (!upload.storageKey) {
-    throw new UploadError(
-      "NOT_FOUND",
-      `Upload '${upload.id}' has no stored file.`
-    );
-  }
-  return Buffer.from(await target.read(upload.storageKey));
-}
-async function deleteVariantRow(db, targets, row) {
-  let current = row;
-  while (current) {
-    if (current.storageKey) {
-      await resolveTarget(targets, current.target).delete(current.storageKey);
-    }
-    const { count } = await db.upload.deleteMany({
-      where: {
-        id: current.id,
-        status: current.status,
-        tokenId: current.tokenId,
-        storageKey: current.storageKey
-      }
-    });
-    if (count === 1) {
-      return;
-    }
-    current = await db.upload.findUnique({
-      where: { id: current.id },
-      omit: { data: true }
-    });
-  }
-}
-async function deleteVariants(source, { db, targets }) {
-  const rows = await db.upload.findMany({
-    where: { sourceId: idOf(source) },
-    omit: { data: true }
-  });
-  for (const row of rows) {
-    await deleteVariantRow(db, targets, row);
-  }
-  return rows.length;
-}
-function defineUploadVariants(options) {
-  const {
-    db,
-    targets,
-    target: defaultTarget,
-    retryFailedAfter = ONE_DAY,
-    claimTimeout = TEN_MINUTES,
-    pollInterval = 250,
-    onError = (error, { sourceId, variant }) => console.error(
-      `[cedar uploads] could not make variant '${variant}' of ${sourceId}:`,
-      error
-    )
-  } = options;
-  const definitions = /* @__PURE__ */ new Map();
-  for (const [name, value] of Object.entries(options.variants)) {
-    if (!VARIANT_NAME.test(name)) {
-      throw new UploadError(
-        "CONFIGURATION",
-        `Invalid variant name '${name}'. Use letters, digits, '.', '_' and '-', starting with a letter or digit.`
-      );
-    }
-    const definition = typeof value === "function" ? { generate: value } : value;
-    const target = definition.target ?? defaultTarget;
-    if (!Object.hasOwn(targets, target.name) || targets[target.name] !== target) {
-      throw new UploadError(
-        "CONFIGURATION",
-        `Variant '${name}' uses a storage target that isn't in \`targets\`.`
-      );
-    }
-    definitions.set(name, { generate: definition.generate, target });
-  }
-  const names = [...definitions.keys()];
-  const inFlight = /* @__PURE__ */ new Map();
-  const waitingFor = /* @__PURE__ */ new Map();
-  const keyOf = (sourceId, name) => `${sourceId}\0${name}`;
-  const isWaitingFor = (from, to) => {
-    const seen = /* @__PURE__ */ new Set();
-    const stack = [from];
-    for (let key = stack.pop(); key !== void 0; key = stack.pop()) {
-      if (key === to) {
-        return true;
-      }
-      if (!seen.has(key)) {
-        seen.add(key);
-        stack.push(...waitingFor.get(key) ?? []);
-      }
-    }
-    return false;
-  };
-  const obtainFor = async (waiter, sourceId, name, chain) => {
-    const key = keyOf(sourceId, name);
-    if (!chain.includes(name) && inFlight.has(key) && isWaitingFor(key, waiter)) {
-      throw new UploadError(
-        "CONFIGURATION",
-        `Variant '${name}' and the variant reading it depend on each other.`
-      );
-    }
-    const waits = waitingFor.get(waiter) ?? [];
-    waitingFor.set(waiter, [...waits, key]);
-    try {
-      return await obtain(sourceId, name, chain);
-    } finally {
-      const rest = waitingFor.get(waiter) ?? [];
-      rest.splice(rest.indexOf(key), 1);
-      if (rest.length === 0) {
-        waitingFor.delete(waiter);
-      }
-    }
-  };
-  const definitionOf = (name) => {
-    const definition = definitions.get(name);
-    if (!definition) {
-      throw new UploadError(
-        "CONFIGURATION",
-        `Unknown upload variant '${name}'. Configured variants: ${names.join(", ") || "(none)"}.`
-      );
-    }
-    return definition;
-  };
-  const loadSource = async (sourceId) => {
-    const source = await db.upload.findUnique({
-      where: { id: sourceId },
-      omit: { data: true }
-    });
-    if (source?.status !== "completed") {
-      throw new UploadError(
-        "NOT_FOUND",
-        `Upload '${sourceId}' does not exist or is not completed, so it cannot have variants.`
-      );
-    }
-    if (source.sourceId) {
-      throw new UploadError(
-        "NOT_SUPPORTED",
-        `Upload '${sourceId}' is itself a variant. Variants are made from original uploads only; use \`acquireVariantData()\` in a generator to make one variant from another.`
-      );
-    }
-    return source;
-  };
-  const findRow = (sourceId, name) => db.upload.findFirst({
-    where: { sourceId, variant: name },
-    omit: { data: true }
-  });
-  const make = async (source, rowId, claim, name, chain) => {
-    const definition = definitionOf(name);
-    const target = definition.target;
-    const ours = { id: rowId, status: "pending", tokenId: claim };
-    const fail = async (error) => {
-      if (error !== void 0) {
-        onError(error, { sourceId: source.id, variant: name });
-      }
-      await db.upload.updateMany({ where: ours, data: { status: "failed" } });
-      return null;
-    };
-    let file;
-    try {
-      file = await definition.generate({
-        source,
-        variant: name,
-        readSource: () => readBytes(db, targets, source),
-        acquireVariantData: async (other) => {
-          const made = await obtainFor(
-            keyOf(source.id, name),
-            source.id,
-            other,
-            [...chain, name]
-          );
-          return made ? readBytes(db, targets, made) : null;
-        }
-      });
-    } catch (e) {
-      return fail(e);
-    }
-    if (!file) {
-      return fail();
-    }
-    if (!MIME_TYPE.test(file.mimeType)) {
-      return fail(
-        new UploadError(
-          "MIME_TYPE_NOT_ALLOWED",
-          `Variant '${name}' returned '${file.mimeType}', which is not a valid MIME type.`
-        )
-      );
-    }
-    const isDbTarget = target.providerType === "db";
-    if (isDbTarget && file.data.byteLength > DB_MAX_FILE_SIZE) {
-      return fail(
-        new UploadError(
-          "FILE_TOO_LARGE",
-          `Variant '${name}' is ${file.data.byteLength} bytes, which exceeds the ${DB_MAX_FILE_SIZE} byte limit for target '${target.name}'.`
-        )
-      );
-    }
-    const storageKey = isDbTarget ? null : generateStorageKey(file.mimeType);
-    if (storageKey) {
-      try {
-        await target.write(storageKey, file.data, {
-          contentType: file.mimeType
-        });
-      } catch (e) {
-        await target.delete(storageKey).catch(() => void 0);
-        await db.upload.deleteMany({ where: ours });
-        throw e;
-      }
-    }
-    const { count } = await db.upload.updateMany({
-      where: ours,
-      data: {
-        status: "completed",
-        // A retry or a takeover writes to the variant's current target,
-        // which is not necessarily the one the row was created for
-        target: target.name,
-        storageKey,
-        size: BigInt(file.data.byteLength),
-        mimeType: file.mimeType,
-        filename: file.filename ?? defaultFilename(source.filename, name, file.mimeType),
-        // Copying gives Prisma the plain-ArrayBuffer-backed array its
-        // `Bytes` input requires
-        data: isDbTarget ? new Uint8Array(file.data) : null,
-        tokenId: null
-      }
-    }).catch(async (e) => {
-      if (storageKey) {
-        await target.delete(storageKey).catch(() => void 0);
-      }
-      await db.upload.deleteMany({ where: ours }).catch(() => void 0);
-      throw e;
-    });
-    if (count !== 1) {
-      if (storageKey) {
-        await target.delete(storageKey).catch(() => void 0);
-      }
-      const current = await findRow(source.id, name);
-      return current?.status === "completed" ? current : null;
-    }
-    return db.upload.findUnique({ where: { id: rowId }, omit: { data: true } });
-  };
-  const obtainUncached = async (sourceId, name, chain) => {
-    const definition = definitionOf(name);
-    const source = await loadSource(sourceId);
-    for (; ; ) {
-      const row = await findRow(sourceId, name);
-      const now = Date.now();
-      const claim = randomUUID();
-      if (!row) {
-        let created;
-        try {
-          created = await db.upload.create({
-            data: {
-              sourceId,
-              variant: name,
-              target: definition.target.name,
-              status: "pending",
-              filename: defaultFilename(source.filename, name),
-              mimeType: "application/octet-stream",
-              size: 0n,
-              tokenId: claim,
-              userId: source.userId,
-              organizationId: source.organizationId
-            }
-          });
-        } catch (e) {
-          if (isPrismaError(e, "P2002")) {
-            continue;
-          }
-          if (isPrismaError(e, "P2003")) {
-            throw new UploadError(
-              "NOT_FOUND",
-              `Upload '${sourceId}' was deleted while its variant '${name}' was being made.`,
-              e
-            );
-          }
-          throw e;
-        }
-        return make(source, created.id, claim, name, chain);
-      }
-      if (row.status === "completed") {
-        return row;
-      }
-      if (row.status === "failed") {
-        const cutoff2 = new Date(now - retryFailedAfter);
-        if (row.updatedAt >= cutoff2) {
-          return null;
-        }
-        const { count } = await db.upload.updateMany({
-          where: { id: row.id, status: "failed", updatedAt: { lt: cutoff2 } },
-          data: { status: "pending", tokenId: claim }
-        });
-        if (count === 1) {
-          return make(source, row.id, claim, name, chain);
-        }
-        continue;
-      }
-      const cutoff = new Date(now - claimTimeout);
-      if (row.updatedAt < cutoff) {
-        const { count } = await db.upload.updateMany({
-          where: {
-            id: row.id,
-            status: "pending",
-            tokenId: row.tokenId,
-            updatedAt: { lt: cutoff }
-          },
-          data: { tokenId: claim }
-        });
-        if (count === 1) {
-          return make(source, row.id, claim, name, chain);
-        }
-        continue;
-      }
-      await sleep(pollInterval);
-    }
-  };
-  const obtain = (sourceId, name, chain) => {
-    if (chain.includes(name)) {
-      return Promise.reject(
-        new UploadError(
-          "CONFIGURATION",
-          `Variant '${name}' depends on itself: ${[...chain, name].join(" -> ")}.`
-        )
-      );
-    }
-    const key = keyOf(sourceId, name);
-    const running = inFlight.get(key);
-    if (running) {
-      return running;
-    }
-    const attempt = obtainUncached(sourceId, name, chain);
-    const forget = () => {
-      if (inFlight.get(key) === attempt) {
-        inFlight.delete(key);
-      }
-    };
-    inFlight.set(key, attempt);
-    attempt.then(forget, forget);
-    return attempt;
-  };
-  const acquireVariant = (source, name) => obtain(idOf(source), name, []);
-  const findVariant = async (source, name) => {
-    definitionOf(name);
-    return db.upload.findFirst({
-      where: { sourceId: idOf(source), variant: name, status: "completed" },
-      omit: { data: true }
-    });
-  };
-  const findVariants = async (sourceIds, name) => {
-    definitionOf(name);
-    const rows = sourceIds.length ? await db.upload.findMany({
-      where: {
-        sourceId: { in: sourceIds },
-        variant: name,
-        status: "completed"
-      },
-      omit: { data: true }
-    }) : [];
-    const bySource = /* @__PURE__ */ new Map();
-    for (const row of rows) {
-      if (row.sourceId) {
-        bySource.set(row.sourceId, row);
-      }
-    }
-    return bySource;
-  };
-  const acquireVariantData = async (source, name) => {
-    const upload = await acquireVariant(source, name);
-    if (!upload) {
-      return null;
-    }
-    return { upload, data: await readBytes(db, targets, upload) };
-  };
-  const acquireVariantUrl = async (source, name, urlOptions = {}) => {
-    const upload = await acquireVariant(source, name);
-    if (!upload) {
-      return null;
-    }
-    const target = resolveTarget(targets, upload.target);
-    if (target.providerType === "db") {
-      return toDataUri(upload.mimeType, await readBytes(db, targets, upload));
-    }
-    if (!upload.storageKey) {
-      return null;
-    }
-    return target.getSignedReadUrl(upload.storageKey, urlOptions);
-  };
-  const regenerateVariants = async (name, {
-    remake = true,
-    batchSize = 100
-  } = {}) => {
-    definitionOf(name);
-    const result = {
-      deleted: 0,
-      made: 0,
-      failed: 0,
-      errors: 0
-    };
-    const newest = await db.upload.findFirst({
-      where: { variant: name },
-      omit: { data: true },
-      orderBy: { createdAt: "desc" }
-    });
-    if (!newest) {
-      return result;
-    }
-    const cutoff = newest.createdAt;
-    const exclude = [];
-    for (; ; ) {
-      const rows = await db.upload.findMany({
-        where: {
-          variant: name,
-          createdAt: { lte: cutoff },
-          ...exclude.length > 0 && { id: { notIn: exclude } }
-        },
-        omit: { data: true },
-        orderBy: { createdAt: "asc" },
-        take: batchSize
-      });
-      if (rows.length === 0) {
-        return result;
-      }
-      for (const row of rows) {
-        const sourceId = row.sourceId ?? "";
-        try {
-          await deleteVariantRow(db, targets, row);
-          result.deleted += 1;
-        } catch (e) {
-          result.errors += 1;
-          exclude.push(row.id);
-          onError(e, { sourceId, variant: name });
-          continue;
-        }
-        if (remake) {
-          await inFlight.get(keyOf(sourceId, name))?.catch(() => null);
-          const made = await acquireVariant(sourceId, name).catch(
-            (e) => {
-              onError(e, { sourceId, variant: name });
-              return null;
-            }
-          );
-          if (made) {
-            result.made += 1;
-          } else {
-            result.failed += 1;
-          }
-          const current = await findRow(sourceId, name);
-          if (current && current.createdAt <= cutoff) {
-            exclude.push(current.id);
-          }
-        }
-      }
-    }
-  };
-  return {
-    names,
-    acquireVariant,
-    findVariant,
-    findVariants,
-    acquireVariantData,
-    acquireVariantUrl,
-    regenerateVariants,
-    deleteVariants: (source) => deleteVariants(source, { db, targets })
-  };
-}
-export {
-  defineUploadVariants,
-  deleteVariants
-};