adapt-authoring-middleware 0.0.1 → 1.0.1

package/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,55 +1,55 @@
-name: Bug Report
-description: File a bug report
-labels: ["bug"]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Thanks for taking the time to fill out this bug report!
-  - type: textarea
-    id: description
-    attributes:
-      label: What happened?
-      description: Please describe the issue
-    validations:
-      required: true
-  - type: textarea
-    id: expected
-    attributes:
-      label: Expected behaviour
-      description: Tell us what should have happened
-  - type: textarea
-    id: repro-steps
-    attributes:
-      label: Steps to reproduce
-      description: Tell us how to reproduce the issue
-    validations:
-      required: true
-  - type: input
-    id: aat-version
-    attributes:
-      label: Authoring tool version
-      description: What version of the Adapt authoring tool are you running?
-    validations:
-      required: true
-  - type: input
-    id: fw-version
-    attributes:
-      label: Framework version
-      description: What version of the Adapt framework are you running?
-  - type: dropdown
-    id: browsers
-    attributes:
-      label: What browsers are you seeing the problem on?
-      multiple: true
-      options:
-        - Firefox
-        - Chrome
-        - Safari
-        - Microsoft Edge
-  - type: textarea
-    id: logs
-    attributes:
-      label: Relevant log output
-      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
-      render: sh
+name: Bug Report
+description: File a bug report
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: textarea
+    id: description
+    attributes:
+      label: What happened?
+      description: Please describe the issue
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behaviour
+      description: Tell us what should have happened
+  - type: textarea
+    id: repro-steps
+    attributes:
+      label: Steps to reproduce
+      description: Tell us how to reproduce the issue
+    validations:
+      required: true
+  - type: input
+    id: aat-version
+    attributes:
+      label: Authoring tool version
+      description: What version of the Adapt authoring tool are you running?
+    validations:
+      required: true
+  - type: input
+    id: fw-version
+    attributes:
+      label: Framework version
+      description: What version of the Adapt framework are you running?
+  - type: dropdown
+    id: browsers
+    attributes:
+      label: What browsers are you seeing the problem on?
+      multiple: true
+      options:
+        - Firefox
+        - Chrome
+        - Safari
+        - Microsoft Edge
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+      render: sh

package/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,22 +1,22 @@
-name: Feature request
-description: Request a new feature
-labels: ["enhancement"]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Thanks for taking the time to request a new feature in the Adapt authoring tool! The Adapt team will consider all new feature requests, but unfortunately cannot commit to every one.
-  - type: textarea
-    id: description
-    attributes:
-      label: Feature description
-      description: Please describe your feature request
-    validations:
-      required: true
-  - type: checkboxes
-    id: contribute
-    attributes:
-      label: Can you work on this feature?
-      description: If you are able to commit your own time to work on this feature, it will greatly increase the liklihood of it being implemented by the core dev team. Otherwise, it will be triaged and prioritised alongside the core team's current priorities.
-      options:
-        - label: I can contribute
+name: Feature request
+description: Request a new feature
+labels: ["enhancement"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to request a new feature in the Adapt authoring tool! The Adapt team will consider all new feature requests, but unfortunately cannot commit to every one.
+  - type: textarea
+    id: description
+    attributes:
+      label: Feature description
+      description: Please describe your feature request
+    validations:
+      required: true
+  - type: checkboxes
+    id: contribute
+    attributes:
+      label: Can you work on this feature?
+      description: If you are able to commit your own time to work on this feature, it will greatly increase the liklihood of it being implemented by the core dev team. Otherwise, it will be triaged and prioritised alongside the core team's current priorities.
+      options:
+        - label: I can contribute

package/.github/dependabot.yml

@@ -1,11 +1,11 @@
-# To get started with Dependabot version updates, you'll need to specify which
-# package ecosystems to update and where the package manifests are located.
-# Please see the documentation for all configuration options:
-# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
-
-version: 2
-updates:
-  - package-ecosystem: "npm" # See documentation for possible values
-    directory: "/" # Location of package manifests
-    schedule:
-      interval: "weekly"
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "npm" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

package/.github/pull_request_template.md

@@ -1,25 +1,25 @@
-[//]: # (Please title your PR according to eslint commit conventions)
-[//]: # (See https://github.com/conventional-changelog/conventional-changelog/tree/master/packages/conventional-changelog-eslint#eslint-convention for details)
-
-[//]: # (Add a link to the original issue)
-
-[//]: # (Delete as appropriate)
-### Fix
-* A sentence describing each fix
-
-### Update
-* A sentence describing each udpate
-
-### New
-* A sentence describing each new feature
-
-### Breaking
-* A sentence describing each breaking change
-
-[//]: # (List appropriate steps for testing if needed)
-### Testing
-1. Steps for testing
-
-[//]: # (Mention any other dependencies)
-
-
+[//]: # (Please title your PR according to eslint commit conventions)
+[//]: # (See https://github.com/conventional-changelog/conventional-changelog/tree/master/packages/conventional-changelog-eslint#eslint-convention for details)
+
+[//]: # (Add a link to the original issue)
+
+[//]: # (Delete as appropriate)
+### Fix
+* A sentence describing each fix
+
+### Update
+* A sentence describing each udpate
+
+### New
+* A sentence describing each new feature
+
+### Breaking
+* A sentence describing each breaking change
+
+[//]: # (List appropriate steps for testing if needed)
+### Testing
+1. Steps for testing
+
+[//]: # (Mention any other dependencies)
+
+

package/.github/workflows/add-to-project-label.yml

@@ -1,18 +1,18 @@
-name: Label PRs to allow add-to-project to run
-
-permissions:
-  pull-requests: write
-
-on:
-  schedule:
-    - cron: '0 6 * * *'
-
-jobs:
-  add-to-project-label:
-    name: Add label after a day
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/stale@v9
-        with:
-          days-before-stale: 1
-          stale-pr-label: 'sorted'
+name: Label PRs to allow add-to-project to run
+
+permissions:
+  pull-requests: write
+
+on:
+  schedule:
+    - cron: '0 6 * * *'
+
+jobs:
+  add-to-project-label:
+    name: Add label after a day
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v9
+        with:
+          days-before-stale: 1
+          stale-pr-label: 'sorted'

package/.github/workflows/new.yml

@@ -1,19 +1,15 @@
-name: Add to main project
-
-on:
-  issues:
-    types:
-      - opened
-  pull_request:
-    types:
-      - opened
-
-jobs:
-  add-to-project:
-    name: Add to main project
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/add-to-project@v0.1.0
-        with:
-          project-url: https://github.com/orgs/adapt-security/projects/5
-          github-token: ${{ secrets.PROJECTS_SECRET }}
+# Calls the org-level reusable workflow to add PRs to the TODO Board
+
+name: Add PR to Project
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+
+jobs:
+  add-to-project:
+    uses: adapt-security/.github/.github/workflows/new.yml@main
+    secrets:
+      PROJECTS_SECRET: ${{ secrets.PROJECTS_SECRET }}

package/.github/workflows/releases.yml

@@ -0,0 +1,32 @@
+name: Release
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # to be able to publish a GitHub release
+      issues: write # to be able to comment on released issues
+      pull-requests: write # to be able to comment on released pull requests
+      id-token: write # to enable use of OIDC for trusted publishing and npm provenance
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 'lts/*'
+      - name: Update npm
+        run: npm install -g npm@latest
+      - name: Install dependencies
+        run: npm ci
+      - name: Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: npx semantic-release

package/.github/workflows/standardjs.yml

@@ -0,0 +1,13 @@
+name: Standard.js formatting check
+on: push
+jobs:
+  default:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - uses: actions/setup-node@master
+        with:
+          node-version: 'lts/*'
+          cache: 'npm'
+      - run: npm ci
+      - run: npx standard

package/adapt-authoring.json

@@ -1,5 +1,5 @@
-{
-  "documentation": {
-    "enable": true
-  }
-}
+{
+  "documentation": {
+    "enable": true
+  }
+}

package/conf/config.schema.json

@@ -1,38 +1,38 @@
-{
-  "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "type": "object",
-  "properties": {
-    "acceptedTypes": {
-      "description": "Content types the API accepts (may use MIME types or extension names)",
-      "type": "array",
-      "items": { "type": "string" },
-      "default": ["application/json"]
-    },
-    "apiRequestLimit": {
-      "description": "The number of API requests allowed by a single IP within the specified time limit",
-      "type": "number",
-      "default": 50
-    },
-    "apiRequestLimitDuration": {
-      "description": "Amount of time before the request count is reset",
-      "type": "string",
-      "isTimeMs": true,
-      "default": "1s"
-    },
-    "fileUploadMaxFileSize": {
-      "description": "Default file size limit for uploaded files. Note that other modules may specify their own limits, please check full config documentation for details.",
-      "type": "string",
-      "isBytes": true,
-      "default": "50mb",
-      "_adapt": {
-        "isPublic": true
-      }
-    },
-    "uploadTempDir": {
-      "description": "Temporary directory for file uploads",
-      "type": "string",
-      "isDirectory": true,
-      "default": "$TEMP/file-uploads"
-    }
-  }
-}
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "acceptedTypes": {
+      "description": "Content types the API accepts (may use MIME types or extension names)",
+      "type": "array",
+      "items": { "type": "string" },
+      "default": ["application/json"]
+    },
+    "apiRequestLimit": {
+      "description": "The number of API requests allowed by a single IP within the specified time limit",
+      "type": "number",
+      "default": 250
+    },
+    "apiRequestLimitDuration": {
+      "description": "Amount of time before the request count is reset",
+      "type": "string",
+      "isTimeMs": true,
+      "default": "1s"
+    },
+    "fileUploadMaxFileSize": {
+      "description": "Default file size limit for uploaded files. Note that other modules may specify their own limits, please check full config documentation for details.",
+      "type": "string",
+      "isBytes": true,
+      "default": "50mb",
+      "_adapt": {
+        "isPublic": true
+      }
+    },
+    "uploadTempDir": {
+      "description": "Temporary directory for file uploads",
+      "type": "string",
+      "isDirectory": true,
+      "default": "$TEMP/file-uploads"
+    }
+  }
+}

package/errors/errors.json

@@ -1,29 +1,29 @@
-{
-  "BODY_PARSE_FAILED": {
-    "data": {
-      "error": "The error message"
-    },
-    "description": "Failed to parse request body data",
-    "statusCode": 400
-  },
-  "FILE_EXCEEDS_MAX_SIZE": {
-    "data": {
-      "maxSize": "The maximum file size",
-      "size": "Size of file"
-    },
-    "description": "Uploaded file exceeds the size limit",
-    "statusCode": 413
-  },
-  "RATE_LIMIT_EXCEEDED": {
-    "description": "API rate limit has been exceeded",
-    "statusCode": 429
-  },
-  "UNEXPECTED_FILE_TYPES": {
-    "data": {
-      "expectedFileTypes": "The list of expected file types",
-      "invalidFiles": "The list of invalid files"
-    },
-    "description": "Recieved unexpected file types",
-    "statusCode": 400
-  }
+{
+  "BODY_PARSE_FAILED": {
+    "data": {
+      "error": "The error message"
+    },
+    "description": "Failed to parse request body data",
+    "statusCode": 400
+  },
+  "FILE_EXCEEDS_MAX_SIZE": {
+    "data": {
+      "maxSize": "The maximum file size",
+      "size": "Size of file"
+    },
+    "description": "Uploaded file exceeds the size limit",
+    "statusCode": 413
+  },
+  "RATE_LIMIT_EXCEEDED": {
+    "description": "API rate limit has been exceeded",
+    "statusCode": 429
+  },
+  "UNEXPECTED_FILE_TYPES": {
+    "data": {
+      "expectedFileTypes": "The list of expected file types",
+      "invalidFiles": "The list of invalid files"
+    },
+    "description": "Recieved unexpected file types",
+    "statusCode": 400
+  }
 }

package/index.js

@@ -1,5 +1,5 @@
-/**
- * Reusable Express.js middleware
- * @namespace middleware
- */
-export { default } from './lib/MiddlewareModule.js'
+/**
+ * Reusable Express.js middleware
+ * @namespace middleware
+ */
+export { default } from './lib/MiddlewareModule.js'

package/lib/MiddlewareModule.js

@@ -1,278 +1,282 @@
-import { AbstractModule, App } from 'adapt-authoring-core'
-import axios from 'axios'
-import bodyParser from 'body-parser'
-import bytes from 'bytes'
-import compression from 'compression'
-import { createWriteStream } from 'fs'
-import { fileTypeFromFile } from 'file-type'
-import formidable from 'formidable'
-import fs from 'fs/promises'
-import path from 'path'
-import helmet from 'helmet'
-import { RateLimiterMongo } from 'rate-limiter-flexible'
-import { unzip } from 'zipper'
-/**
- * Adds useful Express middleware to the server stack
- * @memberof middleware
- * @extends {AbstractModule}
- */
-class MiddlewareModule extends AbstractModule {
-  get zipTypes () {
-    return [
-      'application/zip',
-      'application/x-zip-compressed'
-    ]
-  }
-
-  isZip (mimeType) {
-    return this.zipTypes.includes(mimeType)
-  }
-
-  /** @override */
-  async init () {
-    const server = await this.app.waitForModule('server')
-    const helmetFunc = helmet({ 
-      xFrameOptions: false 
-    })
-    // add custom middleware
-    // server.root.addMiddleware(helmetFunc)
-    server.api.addMiddleware(
-      helmetFunc,
-      this.rateLimiter(),
-      this.bodyParserJson(),
-      this.bodyParserUrlEncoded(),
-      compression()
-    )
-  }
-
-  /**
-   * Limits how many requests indivual IPs can make 
-   * @return {Function} Express middleware function
-   */
-  async rateLimiter () {
-    const [mongodb, server] = await this.app.waitForModule('mongodb', 'server')
-    const { db } = await mongodb.getStats()
-    const rateLimiter = new RateLimiterMongo({
-      storeClient: mongodb.client,
-      dbName: db,
-      keyPrefix: 'ratelimiter',
-      points: this.getConfig('apiRequestLimit'),
-      duration: this.getConfig('apiRequestLimitDuration') / 1000
-    })
-    return async (req, res, next) => {
-      try {
-        const data = await rateLimiter.consume(req.ip)
-        const resetAt = new Date(Date.now() + data.msBeforeNext)
-        res.set({
-          'Retry-After': data.msBeforeNext / 1000,
-          'X-RateLimit-Limit': this.getConfig('apiRequestLimit'),
-          'X-RateLimit-Remaining': data.remainingPoints,
-          'X-RateLimit-Reset': resetAt
-        })
-        next()
-      } catch (e) {
-        res.sendError(this.app.errors.RATE_LIMIT_EXCEEDED.setData({ url: req.url, resetAt }))
-      }
-    }
-  }
-
-  /**
-   * Parses incoming JSON data to req.body
-   * @see https://github.com/expressjs/body-parser#bodyparserjsonoptions
-   * @return {Function} Express middleware function
-   */
-  bodyParserJson () {
-    return (req, res, next) => {
-      bodyParser.json()(req, res, (error, ...args) => {
-        if (error) return next(this.app.errors.BODY_PARSE_FAILED.setData({ error: error.message }))
-        next(null, ...args)
-      })
-    }
-  }
-
-  /**
-   * Parses incoming URL-encoded data to req.body
-   * @see https://github.com/expressjs/body-parser#bodyparserurlencodedoptions
-   * @return {Function} Express middleware function
-   */
-  bodyParserUrlEncoded () {
-    return (req, res, next) => {
-      bodyParser.urlencoded({ extended: true })(req, res, (error, ...args) => {
-        if (error) return next(this.app.errors.BODY_PARSE_FAILED.setData({ error: error.message }))
-        next(null, ...args)
-      })
-    }
-  }
-
-  /**
-   * Sets default file upload options
-   * @param {object} options The initial options object
-   * @returns {FileUploadOptions}
-   */
-  setDefaultFileOptions (options = {}) {
-    Object.entries({
-      maxFileSize: this.getConfig('fileUploadMaxFileSize'),
-      multiples: true,
-      uploadDir: this.getConfig('uploadTempDir'),
-      promisify: false,
-      unzip: false,
-      removeZipSource: true
-    }).forEach(([k, v]) => {
-      if (k === 'expectedFileTypes' && !Array.isArray(v)) v = [v]
-      if (!Object.prototype.hasOwnProperty.call(options, k)) options[k] = v
-    })
-  }
-
-  /**
-   * Handles incoming file uploads
-   * @param {Array<String>} expectedFileTypes List of file types to accept
-   * @param {FileUploadOptions} options
-   * @return {Function} The Express handler
-   */
-  fileUploadParser (expectedFileTypes, options = {}) {
-    options.expectedFileTypes = expectedFileTypes
-    return (req, res, next) => {
-      return new Promise(async (resolve, reject) => {
-        const middleware = await App.instance.waitForModule('middleware')
-        middleware.setDefaultFileOptions(options)
-
-        if (options.promisify) {
-          next = e => e ? reject(e) : resolve()
-        }
-        if (!req.headers['content-type']?.startsWith('multipart/form-data')) {
-          return next()
-        }
-        try {
-          await fs.mkdir(options.uploadDir, { recursive: true })
-        } catch (e) {
-          if (e.code !== 'EEXIST') return next(e)
-        }
-        formidable(options).parse(req, async (error, fields, files) => {
-          if (error) {
-            if (error.code === 1009) {
-              const [maxSize, size] = error.message.match(/(\d+) bytes/g).map(s => bytes(Number(s.replace(' bytes', ''))))
-              error = App.instance.errors.FILE_EXCEEDS_MAX_SIZE.setData({ maxSize, size })
-            }
-            return next(error)
-          }
-          // covert fields back from arrays and add to body
-          Object.keys(fields).forEach(k => {
-            let val = fields[k][0]
-            try { val = JSON.parse(val) } catch (e) {}
-            req.body[k] = val
-          })
-          if (Object.keys(files).length === 0) { // no files uploaded
-            return next()
-          }
-          try {
-            await validateUploadedFiles(req, files, options)
-          } catch (e) {
-            return next(e)
-          }
-          if (options.unzip) {
-            await Promise.all(Object.entries(files).map(async ([k, [f]]) => {
-              if (!middleware.isZip(f.mimetype)) {
-                return Promise.resolve()
-              }
-              f.mimetype = 'application/zip' // always set to the same value for easier checking elsewhere
-              f.filepath = await unzip(f.filepath, `${f.filepath}_unzip`, { removeSource: options.removeZipSource || true })
-            }))
-          }
-          Object.assign(req, { fileUpload: { files } })
-          next()
-        })
-      })
-    }
-  }
-
-  /**
-   * Handles incoming file uploads via URL
-   * @param {Array<String>} expectedFileTypes List of file types to accept
-   * @param {FileUploadOptions} options
-   * @return {Function} The Express handler
-   */
-  urlUploadParser (expectedFileTypes, options) {
-    options.expectedFileTypes = expectedFileTypes
-    return (req, res, next) => {
-      return new Promise(async (resolve, reject) => {
-        const middleware = await App.instance.waitForModule('middleware')
-        middleware.setDefaultFileOptions(options)
-
-        if (options.promisify) {
-          next = e => e ? reject(e) : resolve()
-        }
-        if (!req.body.url) {
-          return next()
-        }
-        let responseData
-        try {
-          responseData = (await axios.get(req.body.url, { responseType: 'stream' })).data
-        } catch (e) {
-          if (e.code === 'ERR_INVALID_URL' || e.response.status === 404) {
-            return next(this.app.errors.INVALID_ASSET_URL.setData({ url: req.body.url }))
-          }
-          return next(e)
-        }
-        const contentType = responseData.headers['content-type']
-        const subtype = contentType.split('/')[1]
-        const fileName = `${new Date().getTime()}.${subtype}`
-        const uploadPath = path.resolve(options.uploadDir, fileName)
-        // set up file data to mimic formidable
-        const fileData = {
-          fields: req.apiData.data,
-          files: {
-            file: [{
-              filepath: uploadPath,
-              originalFilename: fileName,
-              newFilename: fileName,
-              mimetype: contentType,
-              size: Number(responseData.headers['content-length'])
-            }]
-          }
-        }
-        let fileStream
-        try {
-          validateUploadedFiles(req, fileData.files, options)
-          await fs.mkdir(options.uploadDir, { recursive: true })
-          fileStream = createWriteStream(uploadPath)
-        } catch (e) {
-          if (e.code !== 'EEXIST') return next(e)
-        }
-        responseData.pipe(fileStream).on('close', async () => {
-          req.fileUpload = fileData
-          if (subtype === 'zip' && options.unzip) {
-            req.fileUpload.files.course.filepath = await unzip(uploadPath, `${uploadPath}_unzip`, { removeSource: options.removeSource })
-          }
-          next()
-        }).on('error', next)
-      })
-    }
-  }
-}
-/** @ignore */
-async function validateUploadedFiles (req, filesObj, options) {
-  const errors = App.instance.errors
-  const assetErrors = []
-  const filesArr = Object.values(filesObj).reduce((memo, f) => memo.concat(f), []) // flatten nested arrays
-  await Promise.all(filesArr.map(async f => {
-    if (!options.expectedFileTypes.includes(f.mimetype)) {
-      // formidable mimetype isn't allowed, try inspecting the file
-      f.mimetype = (await fileTypeFromFile(f.filepath))?.mime
-      if(!f.mimetype && path.extname(f.originalFilename ) === '.srt') {
-        f.mimetype = 'application/x-subrip'
-      }
-      if (!options.expectedFileTypes.includes(f.mimetype)) {
-        assetErrors.push(errors.UNEXPECTED_FILE_TYPES.setData({ expectedFileTypes: options.expectedFileTypes, invalidFiles: [f.originalFilename], mimetypes: [f.mimetype] }))
-      }
-    }
-    if (!f.size > options.maxFileSize) {
-      assetErrors.push(errors.FILE_EXCEEDS_MAX_SIZE.setData({ size: bytes(f.size), maxSize: bytes(options.maxFileSize) }))
-    }
-  }))
-  if (assetErrors.length) {
-    throw errors.VALIDATION_FAILED
-      .setData({ schemaName: 'fileupload', errors: assetErrors.map(req.translate).join(', ') })
-  }
-}
-
-export default MiddlewareModule
+import { AbstractModule, App } from 'adapt-authoring-core'
+import axios from 'axios'
+import bodyParser from 'body-parser'
+import bytes from 'bytes'
+import compression from 'compression'
+import { createWriteStream } from 'fs'
+import { fileTypeFromFile } from 'file-type'
+import formidable from 'formidable'
+import fs from 'fs/promises'
+import path from 'path'
+import helmet from 'helmet'
+import { RateLimiterMongo } from 'rate-limiter-flexible'
+import { unzip } from 'zipper'
+/**
+ * Adds useful Express middleware to the server stack
+ * @memberof middleware
+ * @extends {AbstractModule}
+ */
+class MiddlewareModule extends AbstractModule {
+  get zipTypes () {
+    return [
+      'application/zip',
+      'application/x-zip-compressed'
+    ]
+  }
+
+  isZip (mimeType) {
+    return this.zipTypes.includes(mimeType)
+  }
+
+  /** @override */
+  async init () {
+    const server = await this.app.waitForModule('server')
+    const helmetFunc = helmet({
+      xFrameOptions: false
+    })
+    // add custom middleware
+    // server.root.addMiddleware(helmetFunc)
+    server.api.addMiddleware(
+      helmetFunc,
+      this.rateLimiter(),
+      this.bodyParserJson(),
+      this.bodyParserUrlEncoded(),
+      compression()
+    )
+  }
+
+  /**
+   * Limits how many requests indivual IPs can make
+   * @return {Function} Express middleware function
+   */
+  async rateLimiter () {
+    const mongodb = await this.app.waitForModule('mongodb')
+    const { db } = await mongodb.getStats()
+    const rateLimiter = new RateLimiterMongo({
+      storeClient: mongodb.client,
+      dbName: db,
+      keyPrefix: 'ratelimiter',
+      points: this.getConfig('apiRequestLimit'),
+      duration: this.getConfig('apiRequestLimitDuration') / 1000
+    })
+    return async (req, res, next) => {
+      let resetAt
+      try {
+        const data = await rateLimiter.consume(req.ip)
+        resetAt = new Date(Date.now() + data.msBeforeNext)
+        res.set({
+          'Retry-After': data.msBeforeNext / 1000,
+          'X-RateLimit-Limit': this.getConfig('apiRequestLimit'),
+          'X-RateLimit-Remaining': data.remainingPoints,
+          'X-RateLimit-Reset': resetAt
+        })
+        next()
+      } catch (e) {
+        res.sendError(this.app.errors.RATE_LIMIT_EXCEEDED.setData({ url: req.url, resetAt }))
+      }
+    }
+  }
+
+  /**
+   * Parses incoming JSON data to req.body
+   * @see https://github.com/expressjs/body-parser#bodyparserjsonoptions
+   * @return {Function} Express middleware function
+   */
+  bodyParserJson () {
+    return (req, res, next) => {
+      bodyParser.json()(req, res, (error, ...args) => {
+        if (error) return next(this.app.errors.BODY_PARSE_FAILED.setData({ error: error.message }))
+        if (req.body === undefined) req.body = {}
+        next(null, ...args)
+      })
+    }
+  }
+
+  /**
+   * Parses incoming URL-encoded data to req.body
+   * @see https://github.com/expressjs/body-parser#bodyparserurlencodedoptions
+   * @return {Function} Express middleware function
+   */
+  bodyParserUrlEncoded () {
+    return (req, res, next) => {
+      bodyParser.urlencoded({ extended: true })(req, res, (error, ...args) => {
+        if (error) return next(this.app.errors.BODY_PARSE_FAILED.setData({ error: error.message }))
+        next(null, ...args)
+      })
+    }
+  }
+
+  /**
+   * Sets default file upload options
+   * @param {object} options The initial options object
+   * @returns {FileUploadOptions}
+   */
+  setDefaultFileOptions (options = {}) {
+    Object.entries({
+      maxFileSize: this.getConfig('fileUploadMaxFileSize'),
+      multiples: true,
+      uploadDir: this.getConfig('uploadTempDir'),
+      promisify: false,
+      unzip: false,
+      removeZipSource: true
+    }).forEach(([k, v]) => {
+      if (k === 'expectedFileTypes' && !Array.isArray(v)) v = [v]
+      if (!Object.prototype.hasOwnProperty.call(options, k)) options[k] = v
+    })
+  }
+
+  /**
+   * Handles incoming file uploads
+   * @param {Array<String>} expectedFileTypes List of file types to accept
+   * @param {FileUploadOptions} options
+   * @return {Function} The Express handler
+   */
+  fileUploadParser (expectedFileTypes, options = {}) {
+    options.expectedFileTypes = expectedFileTypes
+    return (req, res, next) => {
+      // Below is wrapped in a promise so other code can use the Promise interface rather than a standard callback
+      return new Promise(async (resolve, reject) => { // eslint-disable-line no-async-promise-executor
+        const middleware = await App.instance.waitForModule('middleware')
+        middleware.setDefaultFileOptions(options)
+
+        if (options.promisify) {
+          next = e => e ? reject(e) : resolve()
+        }
+        if (!req.headers['content-type']?.startsWith('multipart/form-data')) {
+          return next()
+        }
+        try {
+          await fs.mkdir(options.uploadDir, { recursive: true })
+        } catch (e) {
+          if (e.code !== 'EEXIST') return next(e)
+        }
+        formidable(options).parse(req, async (error, fields, files) => {
+          if (error) {
+            if (error.code === 1009) {
+              const [maxSize, size] = error.message.match(/(\d+) bytes/g).map(s => bytes(Number(s.replace(' bytes', ''))))
+              error = App.instance.errors.FILE_EXCEEDS_MAX_SIZE.setData({ maxSize, size })
+            }
+            return next(error)
+          }
+          // covert fields back from arrays and add to body
+          Object.keys(fields).forEach(k => {
+            let val = fields[k][0]
+            try { val = JSON.parse(val) } catch (e) {}
+            req.body[k] = val
+          })
+          if (Object.keys(files).length === 0) { // no files uploaded
+            return next()
+          }
+          try {
+            await validateUploadedFiles(req, files, options)
+          } catch (e) {
+            return next(e)
+          }
+          if (options.unzip) {
+            await Promise.all(Object.entries(files).map(async ([k, [f]]) => {
+              if (!middleware.isZip(f.mimetype)) {
+                return Promise.resolve()
+              }
+              f.mimetype = 'application/zip' // always set to the same value for easier checking elsewhere
+              f.filepath = await unzip(f.filepath, `${f.filepath}_unzip`, { removeSource: options.removeZipSource || true })
+            }))
+          }
+          Object.assign(req, { fileUpload: { files } })
+          next()
+        })
+      })
+    }
+  }
+
+  /**
+   * Handles incoming file uploads via URL
+   * @param {Array<String>} expectedFileTypes List of file types to accept
+   * @param {FileUploadOptions} options
+   * @return {Function} The Express handler
+   */
+  urlUploadParser (expectedFileTypes, options) {
+    options.expectedFileTypes = expectedFileTypes
+    return (req, res, next) => {
+      // Below is wrapped in a promise so other code can use the Promise interface rather than a standard callback
+      return new Promise(async (resolve, reject) => { // eslint-disable-line no-async-promise-executor
+        const middleware = await App.instance.waitForModule('middleware')
+        middleware.setDefaultFileOptions(options)
+
+        if (options.promisify) {
+          next = e => e ? reject(e) : resolve()
+        }
+        if (!req.body.url) {
+          return next()
+        }
+        let responseData
+        try {
+          responseData = (await axios.get(req.body.url, { responseType: 'stream' })).data
+        } catch (e) {
+          if (e.code === 'ERR_INVALID_URL' || e.response.status === 404) {
+            return next(this.app.errors.INVALID_ASSET_URL.setData({ url: req.body.url }))
+          }
+          return next(e)
+        }
+        const contentType = responseData.headers['content-type']
+        const subtype = contentType.split('/')[1]
+        const fileName = `${new Date().getTime()}.${subtype}`
+        const uploadPath = path.resolve(options.uploadDir, fileName)
+        // set up file data to mimic formidable
+        const fileData = {
+          fields: req.apiData.data,
+          files: {
+            file: [{
+              filepath: uploadPath,
+              originalFilename: fileName,
+              newFilename: fileName,
+              mimetype: contentType,
+              size: Number(responseData.headers['content-length'])
+            }]
+          }
+        }
+        let fileStream
+        try {
+          validateUploadedFiles(req, fileData.files, options)
+          await fs.mkdir(options.uploadDir, { recursive: true })
+          fileStream = createWriteStream(uploadPath)
+        } catch (e) {
+          if (e.code !== 'EEXIST') return next(e)
+        }
+        responseData.pipe(fileStream).on('close', async () => {
+          req.fileUpload = fileData
+          if (subtype === 'zip' && options.unzip) {
+            req.fileUpload.files.course.filepath = await unzip(uploadPath, `${uploadPath}_unzip`, { removeSource: options.removeSource })
+          }
+          next()
+        }).on('error', next)
+      })
+    }
+  }
+}
+/** @ignore */
+async function validateUploadedFiles (req, filesObj, options) {
+  const errors = App.instance.errors
+  const assetErrors = []
+  const filesArr = Object.values(filesObj).reduce((memo, f) => memo.concat(f), []) // flatten nested arrays
+  await Promise.all(filesArr.map(async f => {
+    if (!options.expectedFileTypes.includes(f.mimetype)) {
+      // formidable mimetype isn't allowed, try inspecting the file
+      f.mimetype = (await fileTypeFromFile(f.filepath))?.mime
+      if (!f.mimetype && path.extname(f.originalFilename) === '.srt') {
+        f.mimetype = 'application/x-subrip'
+      }
+      if (!options.expectedFileTypes.includes(f.mimetype)) {
+        assetErrors.push(errors.UNEXPECTED_FILE_TYPES.setData({ expectedFileTypes: options.expectedFileTypes, invalidFiles: [f.originalFilename], mimetypes: [f.mimetype] }))
+      }
+    }
+    if (!f.size > options.maxFileSize) {
+      assetErrors.push(errors.FILE_EXCEEDS_MAX_SIZE.setData({ size: bytes(f.size), maxSize: bytes(options.maxFileSize) }))
+    }
+  }))
+  if (assetErrors.length) {
+    throw errors.VALIDATION_FAILED
+      .setData({ schemaName: 'fileupload', errors: assetErrors.map(req.translate).join(', ') })
+  }
+}
+
+export default MiddlewareModule

package/lib/typedefs.js

@@ -1,13 +1,13 @@
-/**
- * This file exists to define the below types for documentation purposes.
- */
-/**
- * Options which can be passed to file upload middleware
- * @memberof middleware
- * @typedef {Object} FileUploadOptions
- * @property {number} maxFileSize Maximum file size allowed by upload
- * @property {string} uploadDir Directory file upload should be stored
- * @property {Boolean} promisify If true, middleware will return a promise rather than use the standard callback. Useful when calling middleware outside of an Express middleware stack
- * @property {Boolean} removeZipSource To be used in conjunction with the unzip option. Whether the original zip file should be removed after unzipping (true by default)
- * @property {Boolean} unzip Whether any zip files should be unzipped by the handler
- */
+/**
+ * This file exists to define the below types for documentation purposes.
+ */
+/**
+ * Options which can be passed to file upload middleware
+ * @memberof middleware
+ * @typedef {Object} FileUploadOptions
+ * @property {number} maxFileSize Maximum file size allowed by upload
+ * @property {string} uploadDir Directory file upload should be stored
+ * @property {Boolean} promisify If true, middleware will return a promise rather than use the standard callback. Useful when calling middleware outside of an Express middleware stack
+ * @property {Boolean} removeZipSource To be used in conjunction with the unzip option. Whether the original zip file should be removed after unzipping (true by default)
+ * @property {Boolean} unzip Whether any zip files should be unzipped by the handler
+ */

package/package.json

@@ -1,29 +1,63 @@
-{
-  "name": "adapt-authoring-middleware",
-  "version": "0.0.1",
-  "description": "Express middleware to be added to the server",
-  "homepage": "https://github.com/adapt-security/adapt-authoring-middleware",
-  "license": "GPL-3.0",
-  "type": "module",
-  "main": "index.js",
-  "repository": "github:adapt-security/adapt-authoring-middleware",
-  "dependencies": {
-    "axios": "^1.6.7",
-    "body-parser": "^1.20.2",
-    "bytes": "^3.1.2",
-    "compression": "^1.7.4",
-    "file-type": "^20.5.0",
-    "formidable": "^3.5.1",
-    "helmet": "^8.0.0",
-    "lodash": "^4.17.21",
-    "rate-limiter-flexible": "^5.0.3",
-    "zipper": "github:adapt-security/zipper"
-  },
-  "peerDependencies": {
-    "adapt-authoring-core": "github:adapt-security/adapt-authoring-core"
-  },
-  "devDependencies": {
-    "eslint": "^9.14.0",
-    "standard": "^17.1.0"
-  }
-}
+{
+  "name": "adapt-authoring-middleware",
+  "version": "1.0.1",
+  "description": "Express middleware to be added to the server",
+  "homepage": "https://github.com/adapt-security/adapt-authoring-middleware",
+  "license": "GPL-3.0",
+  "type": "module",
+  "main": "index.js",
+  "repository": "github:adapt-security/adapt-authoring-middleware",
+  "dependencies": {
+    "axios": "^1.6.7",
+    "body-parser": "^2.2.2",
+    "bytes": "^3.1.2",
+    "compression": "^1.7.4",
+    "file-type": "^20.5.0",
+    "formidable": "^3.5.1",
+    "helmet": "^8.0.0",
+    "lodash": "^4.17.21",
+    "rate-limiter-flexible": "^8.2.0",
+    "zipper": "github:adapt-security/zipper"
+  },
+  "peerDependencies": {
+    "adapt-authoring-core": "^1.0.0"
+  },
+  "devDependencies": {
+    "@semantic-release/commit-analyzer": "^13.0.1",
+    "@semantic-release/git": "^10.0.1",
+    "@semantic-release/github": "^12.0.2",
+    "@semantic-release/npm": "^13.1.2",
+    "@semantic-release/release-notes-generator": "^14.1.0",
+    "conventional-changelog-eslint": "^6.0.0",
+    "semantic-release": "^25.0.2",
+    "semantic-release-replace-plugin": "^1.2.7",
+    "standard": "^17.1.0"
+  },
+  "release": {
+    "plugins": [
+      [
+        "@semantic-release/commit-analyzer",
+        {
+          "preset": "eslint"
+        }
+      ],
+      [
+        "@semantic-release/release-notes-generator",
+        {
+          "preset": "eslint"
+        }
+      ],
+      "@semantic-release/npm",
+      "@semantic-release/github",
+      [
+        "@semantic-release/git",
+        {
+          "assets": [
+            "package.json"
+          ],
+          "message": "Chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
+        }
+      ]
+    ]
+  }
+}

package/.eslintignore

@@ -1 +0,0 @@
-node_modules

package/.eslintrc

@@ -1,14 +0,0 @@
-{
-  "env": {
-      "browser": false,
-      "node": true,
-      "commonjs": false,
-      "es2020": true
-  },
-  "extends": [
-      "standard"
-  ],
-  "parserOptions": {
-      "ecmaVersion": 2020
-  }
-}

package/.github/workflows/labelled_prs.yml

@@ -1,16 +0,0 @@
-name: Add labelled PRs to project
-
-on:
-  pull_request:
-    types: [ labeled ]
-
-jobs:
-  add-to-project:
-    if: ${{ github.event.label.name == 'dependencies' }}
-    name: Add to main project
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/add-to-project@v0.1.0
-        with:
-          project-url: https://github.com/orgs/adapt-security/projects/5
-          github-token: ${{ secrets.PROJECTS_SECRET }}