@payloadcms/plugin-cloud-storage 1.0.14 → 1.0.15

package/package.json

@@ -1,64 +1,67 @@
-{
-  "name": "@payloadcms/plugin-cloud-storage",
-  "description": "The official cloud storage plugin for Payload CMS",
-  "version": "1.0.14",
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
-  "license": "SEE LICENSE IN LICENSE.MD",
-  "scripts": {
-    "build": "tsc",
-    "lint": "eslint src",
-    "lint:fix": "eslint --fix --ext .ts,.tsx src",
-    "clean": "rimraf dist && rimraf dev/yarn.lock",
-    "prepublishOnly": "yarn clean && yarn build"
-  },
-  "peerDependencies": {
-    "@aws-sdk/client-s3": "^3.142.0",
-    "@aws-sdk/lib-storage": "^3.267.0",
-    "@azure/storage-blob": "^12.11.0",
-    "@google-cloud/storage": "^6.4.1",
-    "payload": "^1.0.27"
-  },
-  "peerDependenciesMeta": {
-    "@aws-sdk/client-s3": {
-      "optional": true
-    },
-    "@aws-sdk/lib-storage": {
-      "optional": true
-    },
-    "@azure/storage-blob": {
-      "optional": true
-    },
-    "@google-cloud/storage": {
-      "optional": true
-    }
-  },
-  "files": [
-    "dist",
-    "*.js",
-    "*.d.ts",
-    "!.prettierrc.js"
-  ],
-  "devDependencies": {
-    "@aws-sdk/client-s3": "^3.142.0",
-    "@aws-sdk/lib-storage": "^3.267.0",
-    "@azure/storage-blob": "^12.11.0",
-    "@google-cloud/storage": "^6.4.1",
-    "@types/express": "^4.17.9",
-    "@typescript-eslint/eslint-plugin": "5.12.1",
-    "@typescript-eslint/parser": "5.12.1",
-    "cross-env": "^7.0.3",
-    "dotenv": "^8.2.0",
-    "eslint": "^8.19.0",
-    "eslint-config-airbnb-base": "^14.2.1",
-    "eslint-config-prettier": "^8.5.0",
-    "eslint-plugin-import": "2.25.4",
-    "eslint-plugin-prettier": "^4.0.0",
-    "nodemon": "^2.0.6",
-    "payload": "^1.6.6",
-    "prettier": "^2.7.1",
-    "rimraf": "^4.1.2",
-    "ts-node": "^9.1.1",
-    "typescript": "^4.1.3"
-  }
-}
+{
+  "name": "@payloadcms/plugin-cloud-storage",
+  "description": "The official cloud storage plugin for Payload CMS",
+  "version": "1.0.15",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "license": "MIT",
+  "scripts": {
+    "build": "tsc",
+    "lint": "eslint src",
+    "lint:fix": "eslint --fix --ext .ts,.tsx src",
+    "clean": "rimraf dist && rimraf dev/yarn.lock",
+    "prepublishOnly": "yarn clean && yarn build"
+  },
+  "peerDependencies": {
+    "@aws-sdk/client-s3": "^3.142.0",
+    "@aws-sdk/lib-storage": "^3.267.0",
+    "@azure/storage-blob": "^12.11.0",
+    "@google-cloud/storage": "^6.4.1",
+    "payload": "^1.0.27"
+  },
+  "peerDependenciesMeta": {
+    "@aws-sdk/client-s3": {
+      "optional": true
+    },
+    "@aws-sdk/lib-storage": {
+      "optional": true
+    },
+    "@azure/storage-blob": {
+      "optional": true
+    },
+    "@google-cloud/storage": {
+      "optional": true
+    }
+  },
+  "files": [
+    "dist",
+    "*.js",
+    "*.d.ts",
+    "!.prettierrc.js"
+  ],
+  "devDependencies": {
+    "@aws-sdk/client-s3": "^3.142.0",
+    "@aws-sdk/lib-storage": "^3.267.0",
+    "@azure/storage-blob": "^12.11.0",
+    "@google-cloud/storage": "^6.4.1",
+    "@types/express": "^4.17.9",
+    "@typescript-eslint/eslint-plugin": "5.12.1",
+    "@typescript-eslint/parser": "5.12.1",
+    "cross-env": "^7.0.3",
+    "dotenv": "^8.2.0",
+    "eslint": "^8.19.0",
+    "eslint-config-airbnb-base": "^14.2.1",
+    "eslint-config-prettier": "^8.5.0",
+    "eslint-plugin-import": "2.25.4",
+    "eslint-plugin-prettier": "^4.0.0",
+    "nodemon": "^2.0.6",
+    "payload": "^1.6.6",
+    "prettier": "^2.7.1",
+    "rimraf": "^4.1.2",
+    "ts-node": "^9.1.1",
+    "typescript": "^4.1.3"
+  },
+  "dependencies": {
+    "range-parser": "^1.2.1"
+  }
+}

package/s3.d.ts

@@ -1 +1 @@
-export * from './dist/adapters/s3'
+export * from './dist/adapters/s3'

package/s3.js

@@ -1 +1 @@
-exports.s3Adapter = require('./dist/adapters/s3').s3Adapter
+exports.s3Adapter = require('./dist/adapters/s3').s3Adapter

package/src/adapters/azure/emulator/docker-compose.yml

@@ -0,0 +1,16 @@
+version: "3"
+
+services:
+  azure-storage:
+    image: mcr.microsoft.com/azure-storage/azurite:3.18.0
+    restart: always
+    command: "azurite --loose --blobHost 0.0.0.0 --tableHost 0.0.0.0 --queueHost 0.0.0.0"
+    ports:
+      - "10000:10000"
+      - "10001:10001"
+      - "10002:10002"
+    volumes:
+      - ./azurestoragedata:/data"
+
+volumes:
+  azurestoragedata:

package/src/adapters/azure/fileStub.js

@@ -0,0 +1 @@
+export default 'file-stub'

package/src/adapters/azure/generateURL.ts

@@ -0,0 +1,13 @@
+import path from 'path'
+import type { GenerateURL } from '../../types'
+
+interface Args {
+  containerName: string
+  baseURL: string
+}
+
+export const getGenerateURL =
+  ({ containerName, baseURL }: Args): GenerateURL =>
+  ({ filename, prefix = '' }) => {
+    return `${baseURL}/${containerName}/${path.posix.join(prefix, filename)}`
+  }

package/src/adapters/azure/handleDelete.ts

@@ -0,0 +1,16 @@
+import path from 'path'
+import type { CollectionConfig } from 'payload/types'
+import type { ContainerClient } from '@azure/storage-blob'
+import type { HandleDelete } from '../../types'
+
+interface Args {
+  collection: CollectionConfig
+  getStorageClient: () => ContainerClient
+}
+
+export const getHandleDelete = ({ getStorageClient }: Args): HandleDelete => {
+  return async ({ filename, doc: { prefix = '' } }) => {
+    const blockBlobClient = getStorageClient().getBlockBlobClient(path.posix.join(prefix, filename))
+    await blockBlobClient.deleteIfExists()
+  }
+}

package/src/adapters/azure/handleUpload.ts

@@ -0,0 +1,41 @@
+import path from 'path'
+import fs from 'fs'
+import { Readable } from 'stream'
+import type { ContainerClient } from '@azure/storage-blob'
+import { AbortController } from '@azure/abort-controller'
+import type { CollectionConfig } from 'payload/types'
+import type { HandleUpload } from '../../types'
+
+interface Args {
+  collection: CollectionConfig
+  getStorageClient: () => ContainerClient
+  prefix?: string
+}
+
+const multipartThreshold = 1024 * 1024 * 50 // 50MB
+export const getHandleUpload = ({ getStorageClient, prefix = '' }: Args): HandleUpload => {
+  return async ({ data, file }) => {
+    const blockBlobClient = getStorageClient().getBlockBlobClient(
+      path.posix.join(prefix, file.filename),
+    )
+
+    // when there are no temp files, or the upload is less than the threshold size, do not stream files
+    if (!file.tempFilePath && file.buffer.length > 0 && file.buffer.length < multipartThreshold) {
+      await blockBlobClient.upload(file.buffer, file.buffer.byteLength, {
+        blobHTTPHeaders: { blobContentType: file.mimeType },
+      })
+
+      return data
+    }
+
+    const fileBufferOrStream: Readable = file.tempFilePath
+      ? fs.createReadStream(file.tempFilePath)
+      : Readable.from(file.buffer)
+
+    await blockBlobClient.uploadStream(fileBufferOrStream, 4 * 1024 * 1024, 4, {
+      abortSignal: AbortController.timeout(30 * 60 * 1000),
+    })
+
+    return data
+  }
+}

package/src/adapters/azure/index.ts

@@ -0,0 +1,47 @@
+import { BlobServiceClient, ContainerClient } from '@azure/storage-blob'
+import type { Adapter, GeneratedAdapter } from '../../types'
+import { getHandler } from './staticHandler'
+import { getGenerateURL } from './generateURL'
+import { getHandleDelete } from './handleDelete'
+import { getHandleUpload } from './handleUpload'
+import { extendWebpackConfig } from './webpack'
+
+export interface Args {
+  connectionString: string
+  containerName: string
+  baseURL: string
+  allowContainerCreate: boolean
+}
+
+export const azureBlobStorageAdapter = ({
+  connectionString,
+  allowContainerCreate,
+  containerName,
+  baseURL,
+}: Args): Adapter => {
+  let storageClient: ContainerClient | null = null
+  const getStorageClient = () => {
+    if (storageClient) return storageClient
+    const blobServiceClient = BlobServiceClient.fromConnectionString(connectionString)
+    return (storageClient = blobServiceClient.getContainerClient(containerName))
+  }
+
+  const createContainerIfNotExists = () => {
+    getStorageClient().createIfNotExists({ access: 'blob' })
+  }
+
+  return ({ collection, prefix }): GeneratedAdapter => {
+    return {
+      handleUpload: getHandleUpload({
+        collection,
+        getStorageClient,
+        prefix,
+      }),
+      handleDelete: getHandleDelete({ collection, getStorageClient }),
+      generateURL: getGenerateURL({ containerName, baseURL }),
+      staticHandler: getHandler({ getStorageClient, collection }),
+      webpack: extendWebpackConfig,
+      ...(allowContainerCreate && { onInit: createContainerIfNotExists }),
+    }
+  }
+}

package/src/adapters/azure/mock.js

@@ -0,0 +1,13 @@
+exports.BlobServiceClient = {
+  fromConnectionString: () => ({
+    getContainerClient: () => ({
+      createIfNotExists: () => null,
+    }),
+  }),
+}
+
+exports.AbortController = {
+  timeout: () => null,
+}
+
+exports.Readable = { from: () => null }

package/src/adapters/azure/staticHandler.ts

@@ -0,0 +1,38 @@
+import type { ContainerClient } from '@azure/storage-blob'
+import path from 'path'
+import type { CollectionConfig } from 'payload/types'
+import type { StaticHandler } from '../../types'
+import { getFilePrefix } from '../../utilities/getFilePrefix'
+import getRangeFromHeader from '../../utilities/getRangeFromHeader'
+
+interface Args {
+  getStorageClient: () => ContainerClient
+  collection: CollectionConfig
+}
+
+export const getHandler = ({ getStorageClient, collection }: Args): StaticHandler => {
+  return async (req, res, next) => {
+    try {
+      const prefix = await getFilePrefix({ req, collection })
+      const blockBlobClient = getStorageClient().getBlockBlobClient(
+        path.posix.join(prefix, req.params.filename),
+      )
+
+      const { start, end } = await getRangeFromHeader(blockBlobClient, req.headers.range)
+
+      const blob = await blockBlobClient.download(start, end)
+      // eslint-disable-next-line no-underscore-dangle
+      const response = blob._response
+      res.header(response.headers.rawHeaders())
+      res.status(response.status)
+
+      if (blob?.readableStreamBody) {
+        return blob.readableStreamBody.pipe(res)
+      }
+
+      return next()
+    } catch (err: unknown) {
+      return next()
+    }
+  }
+}

package/src/adapters/azure/webpack.ts

@@ -0,0 +1,20 @@
+import type { Configuration as WebpackConfig } from 'webpack'
+import path from 'path'
+
+export const extendWebpackConfig = (existingWebpackConfig: WebpackConfig): WebpackConfig => {
+  const newConfig: WebpackConfig = {
+    ...existingWebpackConfig,
+    resolve: {
+      ...(existingWebpackConfig.resolve || {}),
+      alias: {
+        ...(existingWebpackConfig.resolve?.alias ? existingWebpackConfig.resolve.alias : {}),
+        stream: path.resolve(__dirname, './mock.js'),
+        '@azure/storage-blob': path.resolve(__dirname, './mock.js'),
+        '@azure/abort-controller': path.resolve(__dirname, './mock.js'),
+        fs: path.resolve(__dirname, './fileStub.js'),
+      },
+    },
+  }
+
+  return newConfig
+}

package/src/adapters/gcs/emulator/docker-compose.yml

@@ -0,0 +1,15 @@
+version: "3"
+
+services:
+  google-cloud-storage:
+    image: fsouza/fake-gcs-server
+    restart: always
+    command: ["-scheme", "http", "-port", "4443", "-public-host", "http://localhost:4443", "-external-url", "http://localhost:4443", "-backend", "memory"]
+    ports:
+      - "4443:4443"
+    volumes:
+      - ./google-cloud-storage/payload-bucket:/data/payload-bucket
+
+volumes:
+  google-cloud-storage:
+

package/src/adapters/gcs/generateURL.ts

@@ -0,0 +1,16 @@
+import path from 'path'
+import { Storage } from '@google-cloud/storage'
+import type { GenerateURL } from '../../types'
+
+interface Args {
+  getStorageClient: () => Storage
+  bucket: string
+}
+
+export const getGenerateURL =
+  ({ getStorageClient, bucket }: Args): GenerateURL =>
+  ({ filename, prefix = '' }) => {
+    return decodeURIComponent(
+      getStorageClient().bucket(bucket).file(path.posix.join(prefix, filename)).publicUrl(),
+    )
+  }

package/src/adapters/gcs/handleDelete.ts

@@ -0,0 +1,16 @@
+import path from 'path'
+import { Storage } from '@google-cloud/storage'
+import type { HandleDelete } from '../../types'
+
+interface Args {
+  getStorageClient: () => Storage
+  bucket: string
+}
+
+export const getHandleDelete = ({ getStorageClient, bucket }: Args): HandleDelete => {
+  return async ({ filename, doc: { prefix = '' } }) => {
+    await getStorageClient().bucket(bucket).file(path.posix.join(prefix, filename)).delete({
+      ignoreNotFound: true,
+    })
+  }
+}

package/src/adapters/gcs/handleUpload.ts

@@ -0,0 +1,34 @@
+import path from 'path'
+import type { Storage } from '@google-cloud/storage'
+import type { CollectionConfig } from 'payload/types'
+import type { HandleUpload } from '../../types'
+
+interface Args {
+  collection: CollectionConfig
+  bucket: string
+  acl?: 'Private' | 'Public'
+  prefix?: string
+  getStorageClient: () => Storage
+}
+
+export const getHandleUpload = ({
+  getStorageClient,
+  bucket,
+  acl,
+  prefix = '',
+}: Args): HandleUpload => {
+  return async ({ data, file }) => {
+    const gcsFile = getStorageClient().bucket(bucket).file(path.posix.join(prefix, file.filename))
+    await gcsFile.save(file.buffer, {
+      metadata: {
+        contentType: file.mimeType,
+      },
+    })
+
+    if (acl) {
+      await gcsFile[`make${acl}`]()
+    }
+
+    return data
+  }
+}

package/src/adapters/gcs/index.ts

@@ -0,0 +1,37 @@
+import { Storage, StorageOptions } from '@google-cloud/storage'
+import type { Adapter, GeneratedAdapter } from '../../types'
+import { getGenerateURL } from './generateURL'
+import { getHandler } from './staticHandler'
+import { getHandleDelete } from './handleDelete'
+import { getHandleUpload } from './handleUpload'
+import { extendWebpackConfig } from './webpack'
+
+export interface Args {
+  options: StorageOptions
+  bucket: string
+  acl?: 'Private' | 'Public'
+}
+
+export const gcsAdapter =
+  ({ options, bucket, acl }: Args): Adapter =>
+  ({ collection, prefix }): GeneratedAdapter => {
+    let storageClient: Storage | null = null
+    const getStorageClient = () => {
+      if (storageClient) return storageClient
+      return (storageClient = new Storage(options))
+    }
+
+    return {
+      handleUpload: getHandleUpload({
+        collection,
+        getStorageClient,
+        bucket,
+        acl,
+        prefix,
+      }),
+      handleDelete: getHandleDelete({ getStorageClient, bucket }),
+      generateURL: getGenerateURL({ getStorageClient, bucket }),
+      staticHandler: getHandler({ getStorageClient, bucket, collection }),
+      webpack: extendWebpackConfig,
+    }
+  }

package/src/adapters/gcs/mock.js

@@ -0,0 +1,3 @@
+exports.Storage = function () {
+  return null
+}

package/src/adapters/gcs/staticHandler.ts

@@ -0,0 +1,34 @@
+import path from 'path'
+import type { Storage } from '@google-cloud/storage'
+import type { CollectionConfig } from 'payload/types'
+import type { StaticHandler } from '../../types'
+import { getFilePrefix } from '../../utilities/getFilePrefix'
+
+interface Args {
+  getStorageClient: () => Storage
+  bucket: string
+  collection: CollectionConfig
+}
+
+export const getHandler = ({ getStorageClient, bucket, collection }: Args): StaticHandler => {
+  return async (req, res, next) => {
+    try {
+      const prefix = await getFilePrefix({ req, collection })
+      const file = getStorageClient()
+        .bucket(bucket)
+        .file(path.posix.join(prefix, req.params.filename))
+
+      const [metadata] = await file.getMetadata()
+
+      res.set({
+        'Content-Length': metadata.size,
+        'Content-Type': metadata.contentType,
+        ETag: metadata.etag,
+      })
+
+      return file.createReadStream().pipe(res)
+    } catch (err: unknown) {
+      return next()
+    }
+  }
+}

package/src/adapters/gcs/webpack.ts

@@ -0,0 +1,17 @@
+import type { Configuration as WebpackConfig } from 'webpack'
+import path from 'path'
+
+export const extendWebpackConfig = (existingWebpackConfig: WebpackConfig): WebpackConfig => {
+  const newConfig: WebpackConfig = {
+    ...existingWebpackConfig,
+    resolve: {
+      ...(existingWebpackConfig.resolve || {}),
+      alias: {
+        ...(existingWebpackConfig.resolve?.alias ? existingWebpackConfig.resolve.alias : {}),
+        '@google-cloud/storage': path.resolve(__dirname, './mock.js'),
+      },
+    },
+  }
+
+  return newConfig
+}

package/src/adapters/s3/emulator/docker-compose.yml

@@ -0,0 +1,15 @@
+version: '3.2'
+services:
+  localstack:
+    image: localstack/localstack:latest
+    container_name: localstack_demo
+    ports:
+      - '4563-4599:4563-4599'
+      - '8055:8080'
+    environment:
+      - SERVICES=s3
+      - DEBUG=1
+      - DATA_DIR=/tmp/localstack/data
+    volumes:
+      - './.localstack:/var/lib/localstack'
+      - '/var/run/docker.sock:/var/run/docker.sock'

package/src/adapters/s3/fileStub.js

@@ -0,0 +1 @@
+export default 'file-stub'

package/src/adapters/s3/generateURL.ts

@@ -0,0 +1,14 @@
+import path from 'path'
+import type * as AWS from '@aws-sdk/client-s3'
+import type { GenerateURL } from '../../types'
+
+interface Args {
+  config: AWS.S3ClientConfig
+  bucket: string
+}
+
+export const getGenerateURL =
+  ({ config: { endpoint }, bucket }: Args): GenerateURL =>
+  ({ filename, prefix = '' }) => {
+    return `${endpoint}/${bucket}/${path.posix.join(prefix, filename)}`
+  }

package/src/adapters/s3/handleDelete.ts

@@ -0,0 +1,17 @@
+import path from 'path'
+import type * as AWS from '@aws-sdk/client-s3'
+import type { HandleDelete } from '../../types'
+
+interface Args {
+  getStorageClient: () => AWS.S3
+  bucket: string
+}
+
+export const getHandleDelete = ({ getStorageClient, bucket }: Args): HandleDelete => {
+  return async ({ filename, doc: { prefix = '' } }) => {
+    await getStorageClient().deleteObject({
+      Bucket: bucket,
+      Key: path.posix.join(prefix, filename),
+    })
+  }
+}

package/src/adapters/s3/handleUpload.ts

@@ -0,0 +1,62 @@
+import fs from 'fs'
+import path from 'path'
+import type * as AWS from '@aws-sdk/client-s3'
+import type { CollectionConfig } from 'payload/types'
+import type stream from 'stream'
+
+import { Upload } from '@aws-sdk/lib-storage'
+import type { HandleUpload } from '../../types'
+
+interface Args {
+  collection: CollectionConfig
+  bucket: string
+  acl?: 'private' | 'public-read'
+  prefix?: string
+  getStorageClient: () => AWS.S3
+}
+
+const multipartThreshold = 1024 * 1024 * 50 // 50MB
+
+export const getHandleUpload = ({
+  getStorageClient,
+  bucket,
+  acl,
+  prefix = '',
+}: Args): HandleUpload => {
+  return async ({ data, file }) => {
+    const fileKey = path.posix.join(prefix, file.filename)
+
+    const fileBufferOrStream: Buffer | stream.Readable = file.tempFilePath
+      ? fs.createReadStream(file.tempFilePath)
+      : file.buffer
+
+    if (file.buffer.length > 0 && file.buffer.length < multipartThreshold) {
+      await getStorageClient().putObject({
+        Bucket: bucket,
+        Key: fileKey,
+        Body: fileBufferOrStream,
+        ACL: acl,
+        ContentType: file.mimeType,
+      })
+
+      return data
+    }
+
+    const parallelUploadS3 = new Upload({
+      client: getStorageClient(),
+      params: {
+        Bucket: bucket,
+        Key: fileKey,
+        Body: fileBufferOrStream,
+        ACL: acl,
+        ContentType: file.mimeType,
+      },
+      queueSize: 4,
+      partSize: multipartThreshold,
+    })
+
+    await parallelUploadS3.done()
+
+    return data
+  }
+}