@cedarjs/uploads 8.0.0-canary.3400 → 8.0.0-canary.3403

package/README.md

@@ -14,6 +14,11 @@ File uploads and storage for CedarJS apps.
   uploads, and confirming them.
 - **Lifecycle utilities** (`storeFile`, `deleteFile`, `cleanupStaleUploads`) for
   server-generated files and explicit cleanup.
+- **Content detection** (`detectContentType`, `acceptedContentType`) that
+  checks what an uploaded file really is from its bytes, not from its name or
+  the type the browser claimed: images, PDF, audio/video, and OpenDocument and
+  Office files. Uploads use it by default (`verifyContentType`), and store
+  each file under one canonical spelling of its type.
 - **Web hooks and components** at `@cedarjs/uploads/web`, built on Uppy.
 
 ```bash

package/dist/contentTypes.d.ts

@@ -0,0 +1,74 @@
+/**
+ * Content-type spellings and classes shared by every place uploads compare
+ * types: profile `allowedMimeTypes`, `acceptedContentType()`'s accept list,
+ * and the type a file is stored under.
+ */
+/**
+ * Other spellings of types `detectContentType()` returns, mapped to the
+ * spelling it returns (the canonical one). Comparisons treat both spellings
+ * as the same type, so an accept list naming either accepts the file, and
+ * files are stored under the canonical spelling.
+ *
+ * - `image/jpg`, `image/pjpeg`: common misspelling, and what old Internet
+ *   Explorer sent for progressive JPEGs.
+ * - `image/x-png`: what old Internet Explorer sent for PNGs.
+ * - `image/x-ms-bmp`, `image/x-bmp`: what `file`/libmagic and older systems
+ *   report for BMPs. `image/bmp` is the registered type (RFC 7903).
+ * - `application/x-pdf`: pre-registration PDF type some systems still send.
+ * - `video/x-matroska`: the type browsers and most tools report for `.mkv`.
+ *   `video/matroska` is the registered one (RFC 9559).
+ * - `audio/x-m4a`: what browsers on macOS report for `.m4a`. `audio/mp4` is
+ *   the registered type (RFC 4337).
+ * - `video/x-m4v`: what browsers on macOS report for `.m4v`, an MP4 video.
+ */
+export declare const CONTENT_TYPE_ALIASES: ReadonlyMap<string, string>;
+/**
+ * Types a browser runs script in, or renders as a document that can run
+ * script, when it is served from the app's origin. `acceptedContentType()`
+ * never accepts one of these through a wildcard: an accept list has to name
+ * it exactly. Every XML type (`text/xml`, `application/xml`, and anything
+ * with a `+xml` suffix, such as `image/svg+xml` or `application/xhtml+xml`)
+ * counts too, whether or not it is listed here.
+ */
+export declare const SCRIPTABLE_CONTENT_TYPES: ReadonlySet<string>;
+/**
+ * Office types that can carry macros. Macros run in Office on the machine of
+ * whoever downloads the file, so accepting one is a way to deliver malware
+ * to the app's other users, not an XSS risk, and a profile meant for
+ * "documents" almost never means files with macros. `acceptedContentType()`
+ * accepts these only when an accept list names them exactly, never through
+ * a wildcard such as `application/*` or `*\/*`, whether the type was
+ * detected from the bytes or claimed for bytes it doesn't recognise.
+ *
+ * The macro-enabled Office Open XML types (documents, templates, add-ins and
+ * slide shows, and the binary `.xlsb` workbook), and the legacy binary
+ * `.doc`, `.xls` and `.ppt` types, which can carry VBA too and which
+ * detection doesn't recognise. Lower case, like every canonical type.
+ */
+export declare const MACRO_CONTENT_TYPES: ReadonlySet<string>;
+/** `type/subtype`, lower-cased, without parameters such as `charset`. */
+export declare function normalizeContentType(contentType: string): string;
+/**
+ * The canonical spelling of `contentType`: normalized (lower-cased, without
+ * parameters), with aliases from `CONTENT_TYPE_ALIASES` resolved.
+ */
+export declare function canonicalContentType(contentType: string): string;
+/** Every spelling of a canonical type: itself first, then its aliases. */
+export declare function contentTypeSpellings(canonical: string): string[];
+/**
+ * Whether a browser could run script in `contentType` served inline: one of
+ * `SCRIPTABLE_CONTENT_TYPES`, or any XML type.
+ */
+export declare function isScriptableContentType(contentType: string): boolean;
+/**
+ * Whether `contentType` is only accepted when listed exactly, never through a
+ * wildcard: a type that can carry script (`isScriptableContentType()`) or
+ * macros (`MACRO_CONTENT_TYPES`).
+ */
+export declare function isExplicitOnlyContentType(contentType: string): boolean;
+/**
+ * Whether `accept` names `contentType` exactly, not through a wildcard.
+ * Aliases count as the same type.
+ */
+export declare function isContentTypeListed(accept: string[], contentType: string): boolean;
+//# sourceMappingURL=contentTypes.d.ts.map

package/dist/contentTypes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"contentTypes.d.ts","sourceRoot":"","sources":["../src/contentTypes.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,oBAAoB,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAU3D,CAAA;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,wBAAwB,EAAE,WAAW,CAAC,MAAM,CAYvD,CAAA;AAEF;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,mBAAmB,EAAE,WAAW,CAAC,MAAM,CAelD,CAAA;AAEF,yEAAyE;AACzE,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAEhE;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAIhE;AAED,0EAA0E;AAC1E,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAUhE;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAIpE;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAKtE;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,MAAM,EAAE,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAIT"}

package/dist/contentTypes.js

@@ -0,0 +1,78 @@
+const CONTENT_TYPE_ALIASES = /* @__PURE__ */ new Map([
+  ["image/jpg", "image/jpeg"],
+  ["image/pjpeg", "image/jpeg"],
+  ["image/x-png", "image/png"],
+  ["image/x-ms-bmp", "image/bmp"],
+  ["image/x-bmp", "image/bmp"],
+  ["application/x-pdf", "application/pdf"],
+  ["video/x-matroska", "video/matroska"],
+  ["audio/x-m4a", "audio/mp4"],
+  ["video/x-m4v", "video/mp4"]
+]);
+const SCRIPTABLE_CONTENT_TYPES = /* @__PURE__ */ new Set([
+  "image/svg+xml",
+  "text/html",
+  "application/xhtml+xml",
+  "text/xml",
+  "application/xml",
+  "text/xsl",
+  "text/javascript",
+  "application/javascript",
+  "application/x-javascript",
+  "text/ecmascript",
+  "application/ecmascript"
+]);
+const MACRO_CONTENT_TYPES = /* @__PURE__ */ new Set([
+  "application/vnd.ms-word.document.macroenabled.12",
+  "application/vnd.ms-word.template.macroenabled.12",
+  "application/vnd.ms-excel.sheet.macroenabled.12",
+  "application/vnd.ms-excel.template.macroenabled.12",
+  "application/vnd.ms-excel.addin.macroenabled.12",
+  "application/vnd.ms-excel.sheet.binary.macroenabled.12",
+  "application/vnd.ms-powerpoint.presentation.macroenabled.12",
+  "application/vnd.ms-powerpoint.template.macroenabled.12",
+  "application/vnd.ms-powerpoint.slideshow.macroenabled.12",
+  "application/vnd.ms-powerpoint.addin.macroenabled.12",
+  "application/vnd.ms-powerpoint.slide.macroenabled.12",
+  "application/msword",
+  "application/vnd.ms-excel",
+  "application/vnd.ms-powerpoint"
+]);
+function normalizeContentType(contentType) {
+  return contentType.split(";")[0].trim().toLowerCase();
+}
+function canonicalContentType(contentType) {
+  const normalized = normalizeContentType(contentType);
+  return CONTENT_TYPE_ALIASES.get(normalized) ?? normalized;
+}
+function contentTypeSpellings(canonical) {
+  const spellings = [canonical];
+  for (const [alias, target] of CONTENT_TYPE_ALIASES) {
+    if (target === canonical) {
+      spellings.push(alias);
+    }
+  }
+  return spellings;
+}
+function isScriptableContentType(contentType) {
+  const type = canonicalContentType(contentType);
+  return SCRIPTABLE_CONTENT_TYPES.has(type) || type.endsWith("+xml");
+}
+function isExplicitOnlyContentType(contentType) {
+  return isScriptableContentType(contentType) || MACRO_CONTENT_TYPES.has(canonicalContentType(contentType));
+}
+function isContentTypeListed(accept, contentType) {
+  const type = canonicalContentType(contentType);
+  return accept.some((entry) => canonicalContentType(entry) === type);
+}
+export {
+  CONTENT_TYPE_ALIASES,
+  MACRO_CONTENT_TYPES,
+  SCRIPTABLE_CONTENT_TYPES,
+  canonicalContentType,
+  contentTypeSpellings,
+  isContentTypeListed,
+  isExplicitOnlyContentType,
+  isScriptableContentType,
+  normalizeContentType
+};

package/dist/detectContentType.d.ts

@@ -0,0 +1,168 @@
+/**
+ * How many bytes from the start of a file `detectContentType()` needs to
+ * recognise every format it knows. Pass at least this much when reading only
+ * the head of a large file, e.g. with a ranged GET. Fewer bytes still work for
+ * most formats. An ISO-BMFF file (MP4, HEIC…) whose type depends on a
+ * compatible brand needs its whole `ftyp` box, and is reported as `null`
+ * when the box is cut off. Office files (`.docx`, `.xlsx`, `.pptx`) also
+ * need the end of the file; see `DETECT_CONTENT_TYPE_TAIL_BYTES`.
+ */
+export declare const DETECT_CONTENT_TYPE_BYTES = 4096;
+/**
+ * How many bytes from the end of a file to pass as `tail` to
+ * `detectContentType()` when `bytes` is only the head of the file. Only ZIP
+ * files (head starts with `PK\x03\x04`) need it: telling a `.docx` from a
+ * `.docm` or any other ZIP means reading the archive's central directory,
+ * which is at the end.
+ *
+ * The end-of-central-directory record is 22 bytes, plus an archive comment
+ * that office files don't have. Each directory entry is 46 bytes plus its
+ * name (and an extra field some tools add), about 100 bytes for a typical
+ * office part, so 64 KiB holds a directory of 600 or so entries: every
+ * ordinary document and workbook, and slide decks of a few hundred slides.
+ * A directory that doesn't fit is reported as `null`, never guessed; use
+ * `findZipCentralDirectory()` to read the rest of it.
+ */
+export declare const DETECT_CONTENT_TYPE_TAIL_BYTES: number;
+/**
+ * The largest ZIP central directory `detectContentType()` reads: 4 MiB, some
+ * 40,000 entries, far beyond any real office file. A ZIP with a bigger
+ * directory is reported as `null`, so a crafted archive can't make detection
+ * walk (or a caller read) an unbounded directory.
+ */
+export declare const MAX_ZIP_DIRECTORY_BYTES: number;
+export interface DetectContentTypeOptions {
+    /**
+     * The last bytes of the file, when `bytes` is only its head (for example
+     * from two ranged reads of an object in storage). Without it, the end of
+     * `bytes` is taken as the end of the file, which is right when `bytes` is
+     * the whole file. Read `DETECT_CONTENT_TYPE_TAIL_BYTES`, or the whole file
+     * when it is smaller. Only ZIP-based formats use it.
+     */
+    tail?: Uint8Array;
+}
+/**
+ * Every MIME type `detectContentType()` can return, in its canonical
+ * spelling. A claimed type in this set (or an alias of one) is one the bytes
+ * can confirm or refute, which is what `acceptedContentType()` relies on.
+ */
+export declare const DETECTABLE_CONTENT_TYPES: ReadonlySet<string>;
+/**
+ * Where a ZIP's central directory is, from the start of the file, read from
+ * the end-of-central-directory record at the very end of `tail` (the last
+ * bytes of the file). `null` when `tail` has no such record describing a
+ * non-empty directory, or the archive is one `detectContentType()` won't
+ * read (ZIP64, multi-disk, or a directory over `MAX_ZIP_DIRECTORY_BYTES`).
+ *
+ * For callers that read a large file in ranges: when `detectContentType(head,
+ * { tail })` returns `null` for a ZIP because the directory starts before the
+ * tail, read the bytes from `offset` up to where the tail starts, and pass
+ * them and the tail joined together as the new `tail`.
+ */
+export declare function findZipCentralDirectory(tail: Uint8Array): {
+    offset: number;
+    size: number;
+} | null;
+/**
+ * Tells what kind of file `bytes` is from its contents (its magic bytes), not
+ * from a filename or the type a browser claimed. This is content sniffing, in
+ * the sense of the WHATWG MIME Sniffing standard, limited to binary formats
+ * with unambiguous signatures.
+ *
+ * Use it wherever a file's type matters for safety: before handing a user
+ * upload to an image or video decoder, before serving it inline, before
+ * trusting an `accept` list. A file's claimed type and extension are chosen
+ * by whoever sent it.
+ *
+ * Returns a MIME type, or `null` when the format isn't one this function
+ * recognises. It never throws and never reads past the end of its input, so
+ * truncated or garbage input returns `null`. Pass the whole file, or its
+ * first `DETECT_CONTENT_TYPE_BYTES` bytes plus, for a ZIP, its last
+ * `DETECT_CONTENT_TYPE_TAIL_BYTES` as `tail`. Without the tail, a `.docx`,
+ * `.xlsx` or `.pptx` whose head is all there is returns `null`.
+ *
+ * Recognised: JPEG, PNG, GIF, WebP, BMP, TIFF, AVIF, HEIC/HEIF (stills and
+ * sequences), PDF, MP4 (video, and `audio/mp4` for `.m4a`), QuickTime/MOV,
+ * WebM, Matroska, OpenDocument (`.odt`, `.ods`, `.odp`) and Office Open XML
+ * (`.docx`, `.xlsx`, `.pptx`, and their macro-enabled `.docm`, `.xlsm`,
+ * `.pptm`, which are reported as their own types). MP4, MOV, HEIC and AVIF
+ * share the ISO-BMFF `ftyp` box and are told apart by its brands; WebM and
+ * Matroska share the EBML header and are told apart by its `DocType`. WebM
+ * and Matroska are always reported as video, because the header doesn't say
+ * whether there is a video track.
+ *
+ * Returned types are the canonical spellings (`image/jpeg`, `video/matroska`,
+ * `audio/mp4`); see `CONTENT_TYPE_ALIASES` for the others.
+ *
+ * Not recognised, deliberately: SVG and other text formats (an SVG is XML that
+ * can carry script, so it must never pass as a safe image, and text has no
+ * reliable signature), ZIPs that aren't one of the office formats above, and
+ * legacy binary Office files (`.doc`, `.xls`). See `DETECTABLE_CONTENT_TYPES`
+ * for the full list of possible results.
+ *
+ * @example
+ * import { detectContentType } from '@cedarjs/uploads'
+ *
+ * const type = detectContentType(bytes)
+ *
+ * if (type !== 'image/jpeg' && type !== 'image/png') {
+ *   throw new Error('Only JPEG and PNG images can be thumbnailed')
+ * }
+ *
+ * @example
+ * // A large object in storage: its head, and its tail when it is a ZIP
+ * const type = detectContentType(head, { tail })
+ */
+export declare function detectContentType(bytes: Uint8Array, options?: DetectContentTypeOptions): string | null;
+/**
+ * Whether a claimed type may be accepted before the file's bytes are seen,
+ * as when issuing a presigned upload URL: allowed by `accept`, not a
+ * wildcard, and, for a type that can carry script or macros, listed exactly.
+ * The bytes still have to be checked once they arrive.
+ */
+export declare function isClaimedContentTypeAllowed(accept: string[], claimedType: string): boolean;
+/**
+ * Decides whether a file is one of the `accept`ed types by what its bytes
+ * are, and returns the content type to record for it, or `null` when the
+ * file is not accepted. `accept` takes the same entries as an upload
+ * profile's `allowedMimeTypes`: exact types and `type/*` wildcards, with
+ * either spelling of an aliased type (`CONTENT_TYPE_ALIASES`).
+ *
+ * - When `detectContentType()` recognises the bytes, the detected type must
+ *   match `accept`, and is what's returned. The claimed type doesn't matter,
+ *   so a HEIC photo a browser sent as `application/octet-stream` is accepted
+ *   as `image/heic`. A macro-enabled Office file (`MACRO_CONTENT_TYPES`)
+ *   must be listed exactly: `['application/*']` never admits a `.docm`.
+ * - When it doesn't, the file is accepted on the strength of `claimedType`
+ *   when that matches `accept`, wildcards included, except:
+ *   - a claimed type in `DETECTABLE_CONTENT_TYPES` is refused: a real PNG
+ *     would have been recognised, so the claim is false;
+ *   - a type that can carry script (`SCRIPTABLE_CONTENT_TYPES`, and any XML
+ *     type) or macros (`MACRO_CONTENT_TYPES`, which includes the legacy
+ *     `.doc`, `.xls` and `.ppt` types) must be listed exactly. `['image/*']`
+ *     never admits an SVG, `['text/*']` never admits HTML, and `['*\/*']`
+ *     never admits a `.doc`.
+ *
+ *   No claimed type means not accepted.
+ *
+ * The returned type is canonical: lower-cased, without parameters, and with
+ * aliases resolved, so a JPEG claimed as `image/jpg` is stored as
+ * `image/jpeg`.
+ *
+ * Pass the whole file, or its head with `options.tail` (see
+ * `detectContentType()`). An office file checked by its head alone isn't
+ * recognised, and its claimed type is then refused as false.
+ *
+ * @example
+ * import { acceptedContentType, storeFile } from '@cedarjs/uploads'
+ *
+ * const mimeType = acceptedContentType(data, ['image/*'], input.mimeType)
+ *
+ * if (!mimeType) {
+ *   throw new Error('That file is not an image')
+ * }
+ *
+ * await storeFile(target, { db, filename, mimeType, data })
+ */
+export declare function acceptedContentType(bytes: Uint8Array, accept: string[], claimedType?: string, options?: DetectContentTypeOptions): string | null;
+//# sourceMappingURL=detectContentType.d.ts.map

package/dist/detectContentType.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"detectContentType.d.ts","sourceRoot":"","sources":["../src/detectContentType.ts"],"names":[],"mappings":"AAOA;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,OAAO,CAAA;AAE7C;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,8BAA8B,QAAY,CAAA;AAEvD;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,QAAkB,CAAA;AAEtD,MAAM,WAAW,wBAAwB;IACvC;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,UAAU,CAAA;CAClB;AAgBD;;;;GAIG;AACH,eAAO,MAAM,wBAAwB,EAAE,WAAW,CAAC,MAAM,CA2BvD,CAAA;AA+eF;;;;;;;;;;;GAWG;AACH,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,UAAU,GACf;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAUzC;AAoHD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,UAAU,EACjB,OAAO,GAAE,wBAA6B,GACrC,MAAM,GAAG,IAAI,CAmCf;AAED;;;;;GAKG;AACH,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,MAAM,EAAE,EAChB,WAAW,EAAE,MAAM,GAClB,OAAO,CAUT;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,MAAM,EAAE,EAChB,WAAW,CAAC,EAAE,MAAM,EACpB,OAAO,GAAE,wBAA6B,GACrC,MAAM,GAAG,IAAI,CAyBf"}