@payloadcms/plugin-cloud-storage 1.0.14 → 1.0.15-0

package/src/adapters/s3/emulator/.localstack/cache/server.test.pem.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCaPWhTRm/+I3kq
+KZkATLlXtZBGACSvPcJKs/xo1YZX1uCm9mI2/21sqXSArd89lPNJky3LzJavUpRk
+EAZunryuqB6ktxiDU5lG5AihlRXOHPdGDhMSCtyuhgqawY316H/n1LeYt/vjDJ9d
+Qq4CE0vUQMI4OApo3b74Ww2G+xxPoqjPiGCc4L1zdiUKYZm2cxk6tyy89udTbfZu
+pKGqFYQYOMVxh1EVOayg9HExHJV2woYyPNF4LIF5zrqt6VVGyiVnFTnWCMCa2Wky
+p33E76w5PgaDpUhORdwYqMkmAlMnAiFrxmtc4OKoYOZoCP7+tqKLndBvUpWdfMp4
+a4BEyMStAgMBAAECggEAWG8tCdIxFD2ykwIWv50+B5QKjJg4cGGwrLRV3KXQyTxi
+wlgyJ+Z8Dvl52u3WObDVbTH96AP9wbhPbK/jgDlzZZDvQPHaFDIyh/rnF/4q6mqh
+PIGFXarsrQQdwG2Jv5z2+S11txKGUcBC/23j5zy5FZGryiAXl9jMTgmmz+Xaba2p
+rrlwCkbf5NTRSmMrM6i5n7D6bqqE9xm+zdowzX6IxvQN+GwnqH/gebC9Bbt8ZXW8
+MmAvz8odX3PQznSmS7QkvL6YdRonj/XQts1BgoC/zb4zvhRIBxNJGKmu//7nyyFc
+4/5i1T1YXSMW8W602xRLc/A9hLek6g9XeLAF/g6i4QKBgQDNbTOprhMMEprNxnFM
+BkxByERNRVAE7HujRoMlbwPGpaBbUNpYfoc5EAAgc0QjXAeONU/++L5uyR/J12vM
+jGo05DF0HXfhzXEcwG3qw+/xYM2nO76iQtoJESCs+k3N/HKmsIIK58md35v32CEy
+iPvOSYVTeyNGCvXVtq7jN9M4VwKBgQDANjeCa9cpXlFMQt9si3HNMpuuXNt0dXQA
+XqB8c24GcbC+ab/iDBl0iV7ErftQJIa+YLCi3Me/476U0k+cPNaI5TCx6UYPg3cC
+gkM4GEGbrhqTFXxTjxPbA7KR8KNoX/ckzGnGFJ6dm2Y7izv2eCyyFVLZyv/9wH+k
+a1pohd6YmwKBgCtuix200BIdXN1kt097hWPjE43CuchQt1Mt9UxUzFeTbPPhhEhx
+d+V4nI4/0IgpTO+kFeWycZC6tR3XjdMRXOMR2+LQ2CfFZ8TManDYPEKsVM6mswdb
+Wv3PwTjRMuH6HcNgTh5AI0BdpnwgFeS5UYOrx6R+xFZqus8UcKVZiC7BAoGAe4Jb
+KPMizvJMSzStXJI+Dhz27JO+cJcIczRa651AleN4LuLE6fvcX4Xl33vsgRL3gl2f
+Iwe7UxKE5BxAC6UMNesP30KGQnG3h2UszF7Jyb+WU+z6drb5i0JU/HIkjm0b4gl3
+7lBRm/S/PIO7hCS0k7ogAtLuq5+GZaCV95QN9QECgYAjRQbu8TPcs7HXLkqNZuQU
+JEcSa3oihbkvWODbKLktWm9V+MpRpf3L60G/PSFcWe5X1ronYMxXDu26upsxa8l4
+lsSmvaZsN/Mi50bGVYFaqBtHOGM93r2LXX8n17RV8SeORIFtUW+I4HlhRoAQtdHG
+ffRxPAr0bF9ix2ehkJQW4A==
+-----END PRIVATE KEY-----

package/src/adapters/s3/emulator/docker-compose.yml

@@ -0,0 +1,15 @@
+version: '3.2'
+services:
+  localstack:
+    image: localstack/localstack:latest
+    container_name: localstack_demo
+    ports:
+      - '4563-4599:4563-4599'
+      - '8055:8080'
+    environment:
+      - SERVICES=s3
+      - DEBUG=1
+      - DATA_DIR=/tmp/localstack/data
+    volumes:
+      - './.localstack:/var/lib/localstack'
+      - '/var/run/docker.sock:/var/run/docker.sock'

package/src/adapters/s3/fileStub.js

@@ -0,0 +1 @@
+export default 'file-stub'

package/src/adapters/s3/generateURL.ts

@@ -0,0 +1,14 @@
+import path from 'path'
+import type * as AWS from '@aws-sdk/client-s3'
+import type { GenerateURL } from '../../types'
+
+interface Args {
+  config: AWS.S3ClientConfig
+  bucket: string
+}
+
+export const getGenerateURL =
+  ({ config: { endpoint }, bucket }: Args): GenerateURL =>
+  ({ filename, prefix = '' }) => {
+    return `${endpoint}/${bucket}/${path.posix.join(prefix, filename)}`
+  }

package/src/adapters/s3/handleDelete.ts

@@ -0,0 +1,17 @@
+import path from 'path'
+import type * as AWS from '@aws-sdk/client-s3'
+import type { HandleDelete } from '../../types'
+
+interface Args {
+  getStorageClient: () => AWS.S3
+  bucket: string
+}
+
+export const getHandleDelete = ({ getStorageClient, bucket }: Args): HandleDelete => {
+  return async ({ filename, doc: { prefix = '' } }) => {
+    await getStorageClient().deleteObject({
+      Bucket: bucket,
+      Key: path.posix.join(prefix, filename),
+    })
+  }
+}

package/src/adapters/s3/handleUpload.ts

@@ -0,0 +1,62 @@
+import fs from 'fs'
+import path from 'path'
+import type * as AWS from '@aws-sdk/client-s3'
+import type { CollectionConfig } from 'payload/types'
+import type stream from 'stream'
+
+import { Upload } from '@aws-sdk/lib-storage'
+import type { HandleUpload } from '../../types'
+
+interface Args {
+  collection: CollectionConfig
+  bucket: string
+  acl?: 'private' | 'public-read'
+  prefix?: string
+  getStorageClient: () => AWS.S3
+}
+
+const multipartThreshold = 1024 * 1024 * 50 // 50MB
+
+export const getHandleUpload = ({
+  getStorageClient,
+  bucket,
+  acl,
+  prefix = '',
+}: Args): HandleUpload => {
+  return async ({ data, file }) => {
+    const fileKey = path.posix.join(prefix, file.filename)
+
+    const fileBufferOrStream: Buffer | stream.Readable = file.tempFilePath
+      ? fs.createReadStream(file.tempFilePath)
+      : file.buffer
+
+    if (file.buffer.length > 0 && file.buffer.length < multipartThreshold) {
+      await getStorageClient().putObject({
+        Bucket: bucket,
+        Key: fileKey,
+        Body: fileBufferOrStream,
+        ACL: acl,
+        ContentType: file.mimeType,
+      })
+
+      return data
+    }
+
+    const parallelUploadS3 = new Upload({
+      client: getStorageClient(),
+      params: {
+        Bucket: bucket,
+        Key: fileKey,
+        Body: fileBufferOrStream,
+        ACL: acl,
+        ContentType: file.mimeType,
+      },
+      queueSize: 4,
+      partSize: multipartThreshold,
+    })
+
+    await parallelUploadS3.done()
+
+    return data
+  }
+}

package/src/adapters/s3/index.ts

@@ -0,0 +1,38 @@
+import * as AWS from '@aws-sdk/client-s3'
+import type { Adapter, GeneratedAdapter } from '../../types'
+import { getGenerateURL } from './generateURL'
+import { getHandler } from './staticHandler'
+import { getHandleDelete } from './handleDelete'
+import { getHandleUpload } from './handleUpload'
+import { extendWebpackConfig } from './webpack'
+
+export interface Args {
+  config: AWS.S3ClientConfig
+  bucket: string
+  acl?: 'private' | 'public-read'
+}
+
+export const s3Adapter =
+  ({ config, bucket, acl }: Args): Adapter =>
+  ({ collection, prefix }): GeneratedAdapter => {
+    let storageClient: AWS.S3 | null = null
+    const getStorageClient: () => AWS.S3 = () => {
+      if (storageClient) return storageClient
+      storageClient = new AWS.S3(config)
+      return storageClient
+    }
+
+    return {
+      handleUpload: getHandleUpload({
+        collection,
+        getStorageClient,
+        bucket,
+        acl,
+        prefix,
+      }),
+      handleDelete: getHandleDelete({ getStorageClient, bucket }),
+      generateURL: getGenerateURL({ bucket, config }),
+      staticHandler: getHandler({ bucket, getStorageClient, collection }),
+      webpack: extendWebpackConfig,
+    }
+  }

package/src/adapters/s3/mock.js

@@ -0,0 +1,9 @@
+exports.S3 = () => null
+exports.Upload = () => null
+
+exports.HeadObjectCommand = () => null
+exports.PutObjectCommand = () => null
+exports.UploadPartCommand = () => null
+exports.CreateMultipartUploadCommand = () => null
+exports.CompleteMultipartUploadCommand = () => null
+exports.PutObjectTaggingCommand = () => null

package/src/adapters/s3/staticHandler.ts

@@ -0,0 +1,41 @@
+import path from 'path'
+import type { Readable } from 'stream'
+import type * as AWS from '@aws-sdk/client-s3'
+import type { CollectionConfig } from 'payload/types'
+import type { StaticHandler } from '../../types'
+import { getFilePrefix } from '../../utilities/getFilePrefix'
+
+interface Args {
+  getStorageClient: () => AWS.S3
+  bucket: string
+  collection: CollectionConfig
+}
+
+export const getHandler = ({ getStorageClient, bucket, collection }: Args): StaticHandler => {
+  return async (req, res, next) => {
+    try {
+      const prefix = await getFilePrefix({ req, collection })
+
+      const object = await getStorageClient().getObject({
+        Bucket: bucket,
+        Key: path.posix.join(prefix, req.params.filename),
+      })
+
+      res.set({
+        'Accept-Ranges': object.AcceptRanges,
+        'Content-Length': object.ContentLength,
+        'Content-Type': object.ContentType,
+        ETag: object.ETag,
+      })
+
+      if (object?.Body) {
+        return (object.Body as Readable).pipe(res)
+      }
+
+      return next()
+    } catch (err: unknown) {
+      req.payload.logger.error(err)
+      return next()
+    }
+  }
+}

package/src/adapters/s3/webpack.ts

@@ -0,0 +1,23 @@
+import type { Configuration as WebpackConfig } from 'webpack'
+import path from 'path'
+
+export const extendWebpackConfig = (existingWebpackConfig: WebpackConfig): WebpackConfig => {
+  const newConfig: WebpackConfig = {
+    ...existingWebpackConfig,
+    resolve: {
+      ...(existingWebpackConfig.resolve || {}),
+      fallback: {
+        ...(existingWebpackConfig.resolve?.fallback ? existingWebpackConfig.resolve.fallback : {}),
+        stream: false,
+      },
+      alias: {
+        ...(existingWebpackConfig.resolve?.alias ? existingWebpackConfig.resolve.alias : {}),
+        '@aws-sdk/client-s3': path.resolve(__dirname, './mock.js'),
+        '@aws-sdk/lib-storage': path.resolve(__dirname, './mock.js'),
+        fs: path.resolve(__dirname, './fileStub.js'),
+      },
+    },
+  }
+
+  return newConfig
+}

package/src/fields/getFields.ts

@@ -0,0 +1,155 @@
+import path from 'path'
+import type { GroupField, TextField } from 'payload/dist/fields/config/types'
+import type { CollectionConfig, Field } from 'payload/types'
+import { getAfterReadHook } from '../hooks/afterRead'
+import type { GeneratedAdapter, GenerateFileURL } from '../types'
+
+interface Args {
+  collection: CollectionConfig
+  disablePayloadAccessControl?: true
+  generateFileURL?: GenerateFileURL
+  prefix?: string
+  adapter: GeneratedAdapter
+}
+
+export const getFields = ({
+  adapter,
+  collection,
+  disablePayloadAccessControl,
+  generateFileURL,
+  prefix,
+}: Args): Field[] => {
+  const baseURLField: Field = {
+    name: 'url',
+    label: 'URL',
+    type: 'text',
+    admin: {
+      readOnly: true,
+      hidden: true,
+    },
+  }
+
+  const basePrefixField: Field = {
+    name: 'prefix',
+    type: 'text',
+    admin: {
+      readOnly: true,
+      hidden: true,
+    },
+  }
+
+  const fields = [...collection.fields]
+
+  // Inject a hook into all URL fields to generate URLs
+
+  let existingURLFieldIndex = -1
+
+  const existingURLField = fields.find((existingField, i) => {
+    if ('name' in existingField && existingField.name === 'url') {
+      existingURLFieldIndex = i
+      return true
+    }
+    return false
+  }) as TextField
+
+  if (existingURLFieldIndex > -1) {
+    fields.splice(existingURLFieldIndex, 1)
+  }
+
+  fields.push({
+    ...baseURLField,
+    ...(existingURLField || {}),
+    hooks: {
+      afterRead: [
+        getAfterReadHook({ adapter, collection, disablePayloadAccessControl, generateFileURL }),
+        ...(existingURLField?.hooks?.afterRead || []),
+      ],
+    },
+  })
+
+  if (typeof collection.upload === 'object' && collection.upload.imageSizes) {
+    let existingSizesFieldIndex = -1
+
+    const existingSizesField = fields.find((existingField, i) => {
+      if ('name' in existingField && existingField.name === 'sizes') {
+        existingSizesFieldIndex = i
+        return true
+      }
+
+      return false
+    }) as GroupField
+
+    if (existingSizesFieldIndex > -1) {
+      fields.splice(existingSizesFieldIndex, 1)
+    }
+
+    const sizesField: Field = {
+      ...(existingSizesField || {}),
+      name: 'sizes',
+      type: 'group',
+      admin: {
+        hidden: true,
+      },
+      fields: collection.upload.imageSizes.map(size => {
+        const existingSizeField = existingSizesField?.fields.find(
+          existingField => 'name' in existingField && existingField.name === size.name,
+        ) as GroupField
+
+        const existingSizeURLField = existingSizeField?.fields.find(
+          existingField => 'name' in existingField && existingField.name === 'url',
+        ) as GroupField
+
+        return {
+          ...existingSizeField,
+          name: size.name,
+          type: 'group',
+          fields: [
+            {
+              ...(existingSizeURLField || {}),
+              ...baseURLField,
+              hooks: {
+                afterRead: [
+                  getAfterReadHook({
+                    adapter,
+                    collection,
+                    size,
+                    disablePayloadAccessControl,
+                    generateFileURL,
+                  }),
+                  ...(existingSizeURLField?.hooks?.afterRead || []),
+                ],
+              },
+            },
+          ],
+        }
+      }),
+    }
+
+    fields.push(sizesField)
+  }
+
+  // If prefix is enabled, save it to db
+  if (prefix) {
+    let existingPrefixFieldIndex = -1
+
+    const existingPrefixField = fields.find((existingField, i) => {
+      if ('name' in existingField && existingField.name === 'prefix') {
+        existingPrefixFieldIndex = i
+        return true
+      }
+      return false
+    }) as TextField
+
+    if (existingPrefixFieldIndex > -1) {
+      fields.splice(existingPrefixFieldIndex, 1)
+    }
+
+    fields.push({
+      ...basePrefixField,
+      ...(existingPrefixField || {}),
+      defaultValue: path.posix.join(prefix),
+    })
+  }
+
+  return fields
+}

package/src/hooks/afterDelete.ts

@@ -0,0 +1,35 @@
+import type { TypeWithID } from 'payload/dist/globals/config/types'
+import type { FileData } from 'payload/dist/uploads/types'
+import type { CollectionAfterDeleteHook, CollectionConfig } from 'payload/types'
+import type { GeneratedAdapter, TypeWithPrefix } from '../types'
+
+interface Args {
+  collection: CollectionConfig
+  adapter: GeneratedAdapter
+}
+
+export const getAfterDeleteHook = ({
+  collection,
+  adapter,
+}: Args): CollectionAfterDeleteHook<FileData & TypeWithID & TypeWithPrefix> => {
+  return async ({ req, doc }) => {
+    try {
+      const filesToDelete: string[] = [
+        doc.filename,
+        ...Object.values(doc?.sizes || []).map(resizedFileData => resizedFileData?.filename),
+      ]
+
+      const promises = filesToDelete.map(async filename => {
+        if (filename) await adapter.handleDelete({ collection, doc, req, filename })
+      })
+
+      await Promise.all(promises)
+    } catch (err: unknown) {
+      req.payload.logger.error(
+        `There was an error while deleting files corresponding to the ${collection.labels?.singular} with ID ${doc.id}:`,
+      )
+      req.payload.logger.error(err)
+    }
+    return doc
+  }
+}

package/src/hooks/afterRead.ts

@@ -0,0 +1,38 @@
+import type { ImageSize } from 'payload/dist/uploads/types'
+import type { CollectionConfig, FieldHook } from 'payload/types'
+import type { GeneratedAdapter, GenerateFileURL } from '../types'
+
+interface Args {
+  collection: CollectionConfig
+  adapter: GeneratedAdapter
+  disablePayloadAccessControl?: boolean
+  size?: ImageSize
+  generateFileURL?: GenerateFileURL
+}
+
+export const getAfterReadHook =
+  ({ collection, adapter, size, disablePayloadAccessControl, generateFileURL }: Args): FieldHook =>
+  async ({ data, value }) => {
+    const filename = size ? data?.sizes?.[size.name]?.filename : data?.filename
+    const prefix = data?.prefix
+    let url = value
+
+    if (disablePayloadAccessControl && filename) {
+      url = await adapter.generateURL({
+        collection,
+        filename,
+        prefix,
+      })
+    }
+
+    if (generateFileURL) {
+      url = await generateFileURL({
+        collection,
+        filename,
+        prefix,
+        size,
+      })
+    }
+
+    return url
+  }

package/src/hooks/beforeChange.ts

@@ -0,0 +1,59 @@
+import type { TypeWithID } from 'payload/dist/collections/config/types'
+import type { FileData } from 'payload/dist/uploads/types'
+import type { CollectionBeforeChangeHook, CollectionConfig } from 'payload/types'
+import type { GeneratedAdapter } from '../types'
+import { getIncomingFiles } from '../utilities/getIncomingFiles'
+
+interface Args {
+  collection: CollectionConfig
+  adapter: GeneratedAdapter
+}
+
+export const getBeforeChangeHook =
+  ({ collection, adapter }: Args): CollectionBeforeChangeHook<FileData & TypeWithID> =>
+  async ({ req, data, originalDoc }) => {
+    try {
+      const files = getIncomingFiles({ req, data })
+
+      if (files.length > 0) {
+        // If there is an original doc,
+        // And we have new files,
+        // We need to delete the old files before uploading new
+        if (originalDoc) {
+          let filesToDelete: string[] = []
+
+          if (typeof originalDoc?.filename === 'string') {
+            filesToDelete.push(originalDoc.filename)
+          }
+
+          if (typeof originalDoc.sizes === 'object') {
+            filesToDelete = filesToDelete.concat(
+              Object.values(originalDoc?.sizes || []).map(
+                resizedFileData => resizedFileData?.filename,
+              ),
+            )
+          }
+
+          const deletionPromises = filesToDelete.map(async filename => {
+            if (filename) {
+              await adapter.handleDelete({ collection, doc: originalDoc, req, filename })
+            }
+          })
+
+          await Promise.all(deletionPromises)
+        }
+
+        const promises = files.map(async file => {
+          await adapter.handleUpload({ collection, data, req, file })
+        })
+
+        await Promise.all(promises)
+      }
+    } catch (err: unknown) {
+      req.payload.logger.error(
+        `There was an error while uploading files corresponding to the collection ${collection.slug} with filename ${data.filename}:`,
+      )
+      req.payload.logger.error(err)
+    }
+    return data
+  }

package/src/index.ts

@@ -0,0 +1 @@
+export { cloudStorage } from './plugin'

package/src/plugin.ts

@@ -0,0 +1,101 @@
+import type { Config } from 'payload/config'
+import { extendWebpackConfig } from './webpack'
+import type { PluginOptions } from './types'
+import { getBeforeChangeHook } from './hooks/beforeChange'
+import { getAfterDeleteHook } from './hooks/afterDelete'
+import { getFields } from './fields/getFields'
+
+// This plugin extends all targeted collections by offloading uploaded files
+// to cloud storage instead of solely storing files locally.
+
+// It is based on an adapter approach, where adapters can be written for any cloud provider.
+// Adapters are responsible for providing four actions that this plugin will use:
+// 1. handleUpload, 2. handleDelete, 3. generateURL, 4. staticHandler
+
+// Optionally, the adapter can specify any Webpack config overrides if they are necessary.
+
+export const cloudStorage =
+  (pluginOptions: PluginOptions) =>
+  (incomingConfig: Config): Config => {
+    const { collections: allCollectionOptions, enabled } = pluginOptions
+    const config = { ...incomingConfig }
+
+    const webpack = extendWebpackConfig({ options: pluginOptions, config: incomingConfig })
+
+    config.admin = {
+      ...(config.admin || {}),
+      webpack,
+    }
+
+    // Return early if disabled. Only webpack config mods are applied.
+    if (enabled === false) {
+      return config
+    }
+
+    const initFunctions: Array<() => void> = []
+
+    return {
+      ...config,
+      collections: (config.collections || []).map(existingCollection => {
+        const options = allCollectionOptions[existingCollection.slug]
+
+        if (options?.adapter) {
+          const adapter = options.adapter({
+            collection: existingCollection,
+            prefix: options.prefix,
+          })
+
+          if (adapter.onInit) initFunctions.push(adapter.onInit)
+
+          const fields = getFields({
+            collection: existingCollection,
+            disablePayloadAccessControl: options.disablePayloadAccessControl,
+            generateFileURL: options.generateFileURL,
+            prefix: options.prefix,
+            adapter,
+          })
+
+          const handlers = [
+            ...(typeof existingCollection.upload === 'object' &&
+            Array.isArray(existingCollection.upload.handlers)
+              ? existingCollection.upload.handlers
+              : []),
+          ]
+
+          if (!options.disablePayloadAccessControl) {
+            handlers.push(adapter.staticHandler)
+          }
+
+          return {
+            ...existingCollection,
+            upload: {
+              ...(typeof existingCollection.upload === 'object' ? existingCollection.upload : {}),
+              handlers,
+              disableLocalStorage:
+                typeof options.disableLocalStorage === 'boolean'
+                  ? options.disableLocalStorage
+                  : true,
+            },
+            hooks: {
+              ...(existingCollection.hooks || {}),
+              beforeChange: [
+                ...(existingCollection.hooks?.beforeChange || []),
+                getBeforeChangeHook({ adapter, collection: existingCollection }),
+              ],
+              afterDelete: [
+                ...(existingCollection.hooks?.afterDelete || []),
+                getAfterDeleteHook({ adapter, collection: existingCollection }),
+              ],
+            },
+            fields,
+          }
+        }
+
+        return existingCollection
+      }),
+      onInit: async payload => {
+        initFunctions.forEach(fn => fn())
+        if (config.onInit) await config.onInit(payload)
+      },
+    }
+  }