@payloadcms/plugin-cloud-storage 1.0.14 → 1.0.15

package/src/adapters/s3/index.ts

@@ -0,0 +1,38 @@
+import * as AWS from '@aws-sdk/client-s3'
+import type { Adapter, GeneratedAdapter } from '../../types'
+import { getGenerateURL } from './generateURL'
+import { getHandler } from './staticHandler'
+import { getHandleDelete } from './handleDelete'
+import { getHandleUpload } from './handleUpload'
+import { extendWebpackConfig } from './webpack'
+
+export interface Args {
+  config: AWS.S3ClientConfig
+  bucket: string
+  acl?: 'private' | 'public-read'
+}
+
+export const s3Adapter =
+  ({ config, bucket, acl }: Args): Adapter =>
+  ({ collection, prefix }): GeneratedAdapter => {
+    let storageClient: AWS.S3 | null = null
+    const getStorageClient: () => AWS.S3 = () => {
+      if (storageClient) return storageClient
+      storageClient = new AWS.S3(config)
+      return storageClient
+    }
+
+    return {
+      handleUpload: getHandleUpload({
+        collection,
+        getStorageClient,
+        bucket,
+        acl,
+        prefix,
+      }),
+      handleDelete: getHandleDelete({ getStorageClient, bucket }),
+      generateURL: getGenerateURL({ bucket, config }),
+      staticHandler: getHandler({ bucket, getStorageClient, collection }),
+      webpack: extendWebpackConfig,
+    }
+  }

package/src/adapters/s3/mock.js

@@ -0,0 +1,9 @@
+exports.S3 = () => null
+exports.Upload = () => null
+
+exports.HeadObjectCommand = () => null
+exports.PutObjectCommand = () => null
+exports.UploadPartCommand = () => null
+exports.CreateMultipartUploadCommand = () => null
+exports.CompleteMultipartUploadCommand = () => null
+exports.PutObjectTaggingCommand = () => null

package/src/adapters/s3/staticHandler.ts

@@ -0,0 +1,40 @@
+import path from 'path'
+import type { Readable } from 'stream'
+import type * as AWS from '@aws-sdk/client-s3'
+import type { CollectionConfig } from 'payload/types'
+import type { StaticHandler } from '../../types'
+import { getFilePrefix } from '../../utilities/getFilePrefix'
+
+interface Args {
+  getStorageClient: () => AWS.S3
+  bucket: string
+  collection: CollectionConfig
+}
+
+export const getHandler = ({ getStorageClient, bucket, collection }: Args): StaticHandler => {
+  return async (req, res, next) => {
+    try {
+      const prefix = await getFilePrefix({ req, collection })
+
+      const object = await getStorageClient().getObject({
+        Bucket: bucket,
+        Key: path.posix.join(prefix, req.params.filename),
+      })
+
+      res.set({
+        'Content-Length': object.ContentLength,
+        'Content-Type': object.ContentType,
+        ETag: object.ETag,
+      })
+
+      if (object?.Body) {
+        return (object.Body as Readable).pipe(res)
+      }
+
+      return next()
+    } catch (err: unknown) {
+      req.payload.logger.error(err)
+      return next()
+    }
+  }
+}

package/src/adapters/s3/webpack.ts

@@ -0,0 +1,19 @@
+import type { Configuration as WebpackConfig } from 'webpack'
+import path from 'path'
+
+export const extendWebpackConfig = (existingWebpackConfig: WebpackConfig): WebpackConfig => {
+  const newConfig: WebpackConfig = {
+    ...existingWebpackConfig,
+    resolve: {
+      ...(existingWebpackConfig.resolve || {}),
+      alias: {
+        ...(existingWebpackConfig.resolve?.alias ? existingWebpackConfig.resolve.alias : {}),
+        '@aws-sdk/client-s3': path.resolve(__dirname, './mock.js'),
+        '@aws-sdk/lib-storage': path.resolve(__dirname, './mock.js'),
+        fs: path.resolve(__dirname, './fileStub.js'),
+      },
+    },
+  }
+
+  return newConfig
+}

package/src/fields/getFields.ts

@@ -0,0 +1,155 @@
+import path from 'path'
+import type { GroupField, TextField } from 'payload/dist/fields/config/types'
+import type { CollectionConfig, Field } from 'payload/types'
+import { getAfterReadHook } from '../hooks/afterRead'
+import type { GeneratedAdapter, GenerateFileURL } from '../types'
+
+interface Args {
+  collection: CollectionConfig
+  disablePayloadAccessControl?: true
+  generateFileURL?: GenerateFileURL
+  prefix?: string
+  adapter: GeneratedAdapter
+}
+
+export const getFields = ({
+  adapter,
+  collection,
+  disablePayloadAccessControl,
+  generateFileURL,
+  prefix,
+}: Args): Field[] => {
+  const baseURLField: Field = {
+    name: 'url',
+    label: 'URL',
+    type: 'text',
+    admin: {
+      readOnly: true,
+      hidden: true,
+    },
+  }
+
+  const basePrefixField: Field = {
+    name: 'prefix',
+    type: 'text',
+    admin: {
+      readOnly: true,
+      hidden: true,
+    },
+  }
+
+  const fields = [...collection.fields]
+
+  // Inject a hook into all URL fields to generate URLs
+
+  let existingURLFieldIndex = -1
+
+  const existingURLField = fields.find((existingField, i) => {
+    if ('name' in existingField && existingField.name === 'url') {
+      existingURLFieldIndex = i
+      return true
+    }
+    return false
+  }) as TextField
+
+  if (existingURLFieldIndex > -1) {
+    fields.splice(existingURLFieldIndex, 1)
+  }
+
+  fields.push({
+    ...baseURLField,
+    ...(existingURLField || {}),
+    hooks: {
+      afterRead: [
+        getAfterReadHook({ adapter, collection, disablePayloadAccessControl, generateFileURL }),
+        ...(existingURLField?.hooks?.afterRead || []),
+      ],
+    },
+  })
+
+  if (typeof collection.upload === 'object' && collection.upload.imageSizes) {
+    let existingSizesFieldIndex = -1
+
+    const existingSizesField = fields.find((existingField, i) => {
+      if ('name' in existingField && existingField.name === 'sizes') {
+        existingSizesFieldIndex = i
+        return true
+      }
+
+      return false
+    }) as GroupField
+
+    if (existingSizesFieldIndex > -1) {
+      fields.splice(existingSizesFieldIndex, 1)
+    }
+
+    const sizesField: Field = {
+      ...(existingSizesField || {}),
+      name: 'sizes',
+      type: 'group',
+      admin: {
+        hidden: true,
+      },
+      fields: collection.upload.imageSizes.map(size => {
+        const existingSizeField = existingSizesField?.fields.find(
+          existingField => 'name' in existingField && existingField.name === size.name,
+        ) as GroupField
+
+        const existingSizeURLField = existingSizeField?.fields.find(
+          existingField => 'name' in existingField && existingField.name === 'url',
+        ) as GroupField
+
+        return {
+          ...existingSizeField,
+          name: size.name,
+          type: 'group',
+          fields: [
+            {
+              ...(existingSizeURLField || {}),
+              ...baseURLField,
+              hooks: {
+                afterRead: [
+                  getAfterReadHook({
+                    adapter,
+                    collection,
+                    size,
+                    disablePayloadAccessControl,
+                    generateFileURL,
+                  }),
+                  ...(existingSizeURLField?.hooks?.afterRead || []),
+                ],
+              },
+            },
+          ],
+        }
+      }),
+    }
+
+    fields.push(sizesField)
+  }
+
+  // If prefix is enabled, save it to db
+  if (prefix) {
+    let existingPrefixFieldIndex = -1
+
+    const existingPrefixField = fields.find((existingField, i) => {
+      if ('name' in existingField && existingField.name === 'prefix') {
+        existingPrefixFieldIndex = i
+        return true
+      }
+      return false
+    }) as TextField
+
+    if (existingPrefixFieldIndex > -1) {
+      fields.splice(existingPrefixFieldIndex, 1)
+    }
+
+    fields.push({
+      ...basePrefixField,
+      ...(existingPrefixField || {}),
+      defaultValue: path.posix.join(prefix),
+    })
+  }
+
+  return fields
+}

package/src/hooks/afterDelete.ts

@@ -0,0 +1,35 @@
+import type { TypeWithID } from 'payload/dist/globals/config/types'
+import type { FileData } from 'payload/dist/uploads/types'
+import type { CollectionAfterDeleteHook, CollectionConfig } from 'payload/types'
+import type { GeneratedAdapter, TypeWithPrefix } from '../types'
+
+interface Args {
+  collection: CollectionConfig
+  adapter: GeneratedAdapter
+}
+
+export const getAfterDeleteHook = ({
+  collection,
+  adapter,
+}: Args): CollectionAfterDeleteHook<FileData & TypeWithID & TypeWithPrefix> => {
+  return async ({ req, doc }) => {
+    try {
+      const filesToDelete: string[] = [
+        doc.filename,
+        ...Object.values(doc?.sizes || []).map(resizedFileData => resizedFileData?.filename),
+      ]
+
+      const promises = filesToDelete.map(async filename => {
+        await adapter.handleDelete({ collection, doc, req, filename })
+      })
+
+      await Promise.all(promises)
+    } catch (err: unknown) {
+      req.payload.logger.error(
+        `There was an error while deleting files corresponding to the ${collection.labels?.singular} with ID ${doc.id}:`,
+      )
+      req.payload.logger.error(err)
+    }
+    return doc
+  }
+}

package/src/hooks/afterRead.ts

@@ -0,0 +1,38 @@
+import type { ImageSize } from 'payload/dist/uploads/types'
+import type { CollectionConfig, FieldHook } from 'payload/types'
+import type { GeneratedAdapter, GenerateFileURL } from '../types'
+
+interface Args {
+  collection: CollectionConfig
+  adapter: GeneratedAdapter
+  disablePayloadAccessControl?: boolean
+  size?: ImageSize
+  generateFileURL?: GenerateFileURL
+}
+
+export const getAfterReadHook =
+  ({ collection, adapter, size, disablePayloadAccessControl, generateFileURL }: Args): FieldHook =>
+  async ({ data, value }) => {
+    const filename = size ? data?.sizes?.[size.name]?.filename : data?.filename
+    const prefix = data?.prefix
+    let url = value
+
+    if (disablePayloadAccessControl && filename) {
+      url = await adapter.generateURL({
+        collection,
+        filename,
+        prefix,
+      })
+    }
+
+    if (generateFileURL) {
+      url = await generateFileURL({
+        collection,
+        filename,
+        prefix,
+        size,
+      })
+    }
+
+    return url
+  }

package/src/hooks/beforeChange.ts

@@ -0,0 +1,30 @@
+import type { TypeWithID } from 'payload/dist/collections/config/types'
+import type { FileData } from 'payload/dist/uploads/types'
+import type { CollectionBeforeChangeHook, CollectionConfig } from 'payload/types'
+import type { GeneratedAdapter } from '../types'
+import { getIncomingFiles } from '../utilities/getIncomingFiles'
+
+interface Args {
+  collection: CollectionConfig
+  adapter: GeneratedAdapter
+}
+
+export const getBeforeChangeHook =
+  ({ collection, adapter }: Args): CollectionBeforeChangeHook<FileData & TypeWithID> =>
+  async ({ req, data }) => {
+    try {
+      const files = getIncomingFiles({ req, data })
+
+      const promises = files.map(async file => {
+        await adapter.handleUpload({ collection, data, req, file })
+      })
+
+      await Promise.all(promises)
+    } catch (err: unknown) {
+      req.payload.logger.error(
+        `There was an error while uploading files corresponding to the collection ${collection.slug} with filename ${data.filename}:`,
+      )
+      req.payload.logger.error(err)
+    }
+    return data
+  }

package/src/index.ts

@@ -0,0 +1 @@
+export { cloudStorage } from './plugin'

package/src/plugin.ts

@@ -0,0 +1,94 @@
+import type { Config } from 'payload/config'
+import { extendWebpackConfig } from './webpack'
+import type { PluginOptions } from './types'
+import { getBeforeChangeHook } from './hooks/beforeChange'
+import { getAfterDeleteHook } from './hooks/afterDelete'
+import { getFields } from './fields/getFields'
+
+// This plugin extends all targeted collections by offloading uploaded files
+// to cloud storage instead of solely storing files locally.
+
+// It is based on an adapter approach, where adapters can be written for any cloud provider.
+// Adapters are responsible for providing four actions that this plugin will use:
+// 1. handleUpload, 2. handleDelete, 3. generateURL, 4. staticHandler
+
+// Optionally, the adapter can specify any Webpack config overrides if they are necessary.
+
+export const cloudStorage =
+  (pluginOptions: PluginOptions) =>
+  (config: Config): Config => {
+    const { collections: allCollectionOptions } = pluginOptions
+
+    const webpack = extendWebpackConfig({ options: pluginOptions, config })
+
+    const initFunctions: (() => void)[] = []
+
+    return {
+      ...config,
+      admin: {
+        ...(config.admin || {}),
+        webpack,
+      },
+      collections: (config.collections || []).map(existingCollection => {
+        const options = allCollectionOptions[existingCollection.slug]
+
+        if (options?.adapter) {
+          const adapter = options.adapter({
+            collection: existingCollection,
+            prefix: options.prefix,
+          })
+
+          if (adapter.onInit) initFunctions.push(adapter.onInit)
+
+          const fields = getFields({
+            collection: existingCollection,
+            disablePayloadAccessControl: options.disablePayloadAccessControl,
+            generateFileURL: options.generateFileURL,
+            prefix: options.prefix,
+            adapter,
+          })
+
+          const handlers = [
+            ...(typeof existingCollection.upload === 'object' &&
+            Array.isArray(existingCollection.upload.handlers)
+              ? existingCollection.upload.handlers
+              : []),
+          ]
+
+          if (!options.disablePayloadAccessControl) {
+            handlers.push(adapter.staticHandler)
+          }
+
+          return {
+            ...existingCollection,
+            upload: {
+              ...(typeof existingCollection.upload === 'object' ? existingCollection.upload : {}),
+              handlers,
+              disableLocalStorage:
+                typeof options.disableLocalStorage === 'boolean'
+                  ? options.disableLocalStorage
+                  : true,
+            },
+            hooks: {
+              ...(existingCollection.hooks || {}),
+              beforeChange: [
+                ...(existingCollection.hooks?.beforeChange || []),
+                getBeforeChangeHook({ adapter, collection: existingCollection }),
+              ],
+              afterDelete: [
+                ...(existingCollection.hooks?.afterDelete || []),
+                getAfterDeleteHook({ adapter, collection: existingCollection }),
+              ],
+            },
+            fields,
+          }
+        }
+
+        return existingCollection
+      }),
+      onInit: async payload => {
+        initFunctions.forEach(fn => fn())
+        if (config.onInit) await config.onInit(payload)
+      },
+    }
+  }

package/src/types.ts

@@ -0,0 +1,73 @@
+import type { NextFunction, Response } from 'express'
+import type { TypeWithID } from 'payload/dist/collections/config/types'
+import type { FileData, ImageSize } from 'payload/dist/uploads/types'
+import type { CollectionConfig, PayloadRequest } from 'payload/types'
+import type { Configuration as WebpackConfig } from 'webpack'
+
+export interface File {
+  buffer: Buffer
+  filename: string
+  filesize: number
+  mimeType: string
+  tempFilePath?: string
+}
+
+export type HandleUpload = (args: {
+  collection: CollectionConfig
+  req: PayloadRequest
+  data: any
+  file: File
+}) => Promise<void> | void
+
+export interface TypeWithPrefix {
+  prefix?: string
+}
+
+export type HandleDelete = (args: {
+  collection: CollectionConfig
+  req: PayloadRequest
+  doc: TypeWithID & FileData & TypeWithPrefix
+  filename: string
+}) => Promise<void> | void
+
+export type GenerateURL = (args: {
+  filename: string
+  collection: CollectionConfig
+  prefix?: string
+}) => string | Promise<string>
+
+export type StaticHandler = (
+  req: PayloadRequest,
+  res: Response,
+  next: NextFunction,
+) => Promise<unknown> | unknown
+
+export interface GeneratedAdapter {
+  handleUpload: HandleUpload
+  handleDelete: HandleDelete
+  generateURL: GenerateURL
+  staticHandler: StaticHandler
+  webpack?: (config: WebpackConfig) => WebpackConfig
+  onInit?: () => void
+}
+
+export type Adapter = (args: { collection: CollectionConfig; prefix?: string }) => GeneratedAdapter
+
+export type GenerateFileURL = (args: {
+  collection: CollectionConfig
+  filename: string
+  prefix?: string
+  size?: ImageSize
+}) => Promise<string> | string
+
+export interface CollectionOptions {
+  disableLocalStorage?: boolean
+  disablePayloadAccessControl?: true
+  generateFileURL?: GenerateFileURL
+  prefix?: string
+  adapter: Adapter | null
+}
+
+export interface PluginOptions {
+  collections: Record<string, CollectionOptions>
+}

package/src/utilities/getFilePrefix.ts

@@ -0,0 +1,26 @@
+import type { CollectionConfig, PayloadRequest } from 'payload/types'
+import { IncomingUploadType } from 'payload/dist/uploads/types'
+
+export async function getFilePrefix({
+  req,
+  collection,
+}: {
+  req: PayloadRequest
+  collection: CollectionConfig
+}): Promise<string> {
+  const imageSizes = (collection?.upload as IncomingUploadType)?.imageSizes || []
+  const files = await req.payload.find({
+    collection: collection.slug,
+    where: {
+      or: [
+        {
+          filename: { equals: req.params.filename },
+        },
+        ...imageSizes.map(imageSize => ({
+          [`sizes.${imageSize.name}.filename`]: { equals: req.params.filename },
+        })),
+      ],
+    },
+  })
+  return files?.docs?.[0]?.prefix || ''
+}

package/src/utilities/getIncomingFiles.ts

@@ -0,0 +1,44 @@
+import type { FileData } from 'payload/dist/uploads/types'
+import type { PayloadRequest } from 'payload/types'
+import type { File } from '../types'
+
+export function getIncomingFiles({
+  req,
+  data,
+}: {
+  data: Partial<FileData>
+  req: PayloadRequest
+}): File[] {
+  const file = req.files?.file
+
+  let files: File[] = []
+
+  if (file && data.filename && data.mimeType) {
+    const mainFile: File = {
+      filename: data.filename,
+      mimeType: data.mimeType,
+      buffer: file.data,
+      tempFilePath: file.tempFilePath,
+      filesize: file.size,
+    }
+
+    files = [mainFile]
+
+    if (data?.sizes) {
+      Object.entries(data.sizes).forEach(([key, resizedFileData]) => {
+        if (req.payloadUploadSizes?.[key] && data.mimeType) {
+          files = files.concat([
+            {
+              filename: `${resizedFileData.filename}`,
+              mimeType: data.mimeType,
+              buffer: req.payloadUploadSizes[key],
+              filesize: req.payloadUploadSizes[key].length,
+            },
+          ])
+        }
+      })
+    }
+  }
+
+  return files
+}

package/src/utilities/getRangeFromHeader.ts

@@ -0,0 +1,27 @@
+import type { BlockBlobClient } from '@azure/storage-blob'
+import parseRange from 'range-parser'
+
+const getRangeFromHeader = async (
+  blockBlobClient: BlockBlobClient,
+  rangeHeader?: string,
+): Promise<{ start: number; end: number | undefined }> => {
+  const fullRange = { start: 0, end: undefined }
+
+  if (!rangeHeader) {
+    return fullRange
+  }
+
+  const size = await blockBlobClient.getProperties().then(props => props.contentLength)
+  if (size === undefined) {
+    return fullRange
+  }
+
+  const range = parseRange(size, rangeHeader)
+  if (range === -1 || range === -2 || range.type !== 'bytes' || range.length !== 1) {
+    return fullRange
+  }
+
+  return range[0]
+}
+
+export default getRangeFromHeader

package/src/webpack.ts

@@ -0,0 +1,46 @@
+import type { Config } from 'payload/config'
+import type { Configuration as WebpackConfig } from 'webpack'
+import type { GeneratedAdapter, PluginOptions } from './types'
+
+interface Args {
+  config: Config
+  options: PluginOptions
+}
+
+export const extendWebpackConfig =
+  ({ config, options }: Args): ((webpackConfig: WebpackConfig) => WebpackConfig) =>
+  webpackConfig => {
+    const existingWebpackConfig =
+      typeof config.admin?.webpack === 'function'
+        ? config.admin.webpack(webpackConfig)
+        : webpackConfig
+
+    const newConfig: WebpackConfig = {
+      ...existingWebpackConfig,
+      resolve: {
+        ...(existingWebpackConfig.resolve || {}),
+        alias: {
+          ...(existingWebpackConfig.resolve?.alias ? existingWebpackConfig.resolve.alias : {}),
+        },
+      },
+    }
+
+    return Object.entries(options.collections).reduce(
+      (resultingWebpackConfig, [slug, collectionOptions]) => {
+        const matchedCollection = config.collections?.find(coll => coll.slug === slug)
+
+        if (matchedCollection && typeof collectionOptions.adapter === 'function') {
+          const adapter: GeneratedAdapter = collectionOptions.adapter({
+            collection: matchedCollection,
+          })
+
+          if (adapter.webpack) {
+            return adapter.webpack(resultingWebpackConfig)
+          }
+        }
+
+        return resultingWebpackConfig
+      },
+      newConfig,
+    )
+  }