jskos-server 2.4.0

package/utils/middleware.js

@@ -0,0 +1,636 @@
+import config from "../config/index.js"
+import _ from "lodash"
+import jskos from "jskos-tools"
+import { DuplicateEntityError, EntityNotFoundError, CreatorDoesNotMatchError, DatabaseInconsistencyError, InvalidBodyError } from "../errors/index.js"
+
+import { Transform, Readable } from "node:stream"
+import JSONStream from "JSONStream"
+import * as anystream from "json-anystream"
+import express from "express"
+
+import { cleanJSON } from "./utils.js"
+import { getUrisOfUser } from "./users.js"
+import { createServices } from "../services/index.js"
+import { createAdjuster } from "./adjust.js"
+
+const services = createServices(config)
+const adjust = createAdjuster(config, services)
+
+/**
+ * Wraps an async middleware function that returns data in the Promise.
+ * The result of the Promise will be written into req.data for access by following middlewaren.
+ * A rejected Promise will be caught and relayed to the Express error handling.
+ *
+ * adjusted from: https://thecodebarbarian.com/80-20-guide-to-express-error-handling
+ */
+const wrapAsync = (fn) => {
+  return (req, res, next) => {
+    fn(req, res, next).then(data => {
+      // On success, save the result of the Promise in req.data.
+      req.data = data
+      next()
+    }).catch(error => {
+      // Catch and change certain errors
+      if (error.code === 11000) {
+        const _id = _.get(error, "keyValue._id") || _.get(error, "writeErrors[0].err.op._id")
+        error = new DuplicateEntityError(null, `${_id} (${req.type})`)
+      }
+      // Pass error to the next error middleware.
+      next(error)
+    })
+  }
+}
+
+// Middleware wrapper that calls the middleware depending on req.query.download
+const wrapDownload = (fn, isDownload = true) => {
+  return (req, res, next) => {
+    if (!!req.query.download === isDownload) {
+      fn(req, res, next)
+    } else {
+      next()
+    }
+  }
+}
+
+const buildUrlForLinkHeader = ({ query, rel, req }) => {
+  let url = config.baseUrl.substring(0, config.baseUrl.length - 1) + req.path
+  if (!query && req) {
+    query = req.query
+  }
+  let index = 0
+  _.forOwn(_.omit(query, ["bulk"]), (value, key) => {
+    url += `${(index == 0 ? "?" : "&")}${key}=${encodeURIComponent(value)}`
+    index += 1
+  })
+  return `<${url}>; rel="${rel}"`
+}
+
+/**
+ * Returns `true` if the creator of `object` matches `user`, `false` if not.
+ * `object.creator` can be
+ * - an array of objects
+ * - an object
+ * - a string
+ * The object for a creator will be checked for properties `uri` (e.g. JSKOS mapping) and `id` (e.g. annotations).
+ *
+ * If config.auth.allowCrossUserEditing is enabled, this returns true as long as a user and object are given.
+ *
+ * @param {object} options.req the request object (that includes req.user, req.crossUser, and req.auth)
+ * @param {object} options.object any object that has the property `creator`
+ * @param {boolean} options.withContributors allow contributors to be matched (for object with superordinated object)
+ */
+const matchesCreator = ({ req = {}, object, withContributors = false }) => {
+  const { user, crossUser, auth } = req
+  if (!auth) {
+    return true
+  }
+  if (!object || !user) {
+    return false
+  }
+  const userUris = getUrisOfUser(user)
+  if (crossUser === true || _.intersection(crossUser || [], userUris).length) {
+    return true
+  }
+  // Support arrays, objects, and strings as creators
+  let creators = Array.isArray(object.creator) ? object.creator : (_.isObject(object.creator) ? [object.creator] : [{ uri: object.creator }])
+  // Also check contributors if requested
+  let contributors = withContributors ? (object.contributor || []) : []
+  for (let creator of creators.concat(contributors)) {
+    if (userUris.includes(creator.uri) || userUris.includes(creator.id)) {
+      return true
+    }
+  }
+  return false
+}
+
+/**
+ * Middleware that adds default headers.
+ */
+const addDefaultHeaders = (req, res, next) => {
+  if (req.headers.origin) {
+    // Allow all origins by returning the request origin in the header
+    res.setHeader("Access-Control-Allow-Origin", req.headers.origin)
+  } else {
+    // Fallback to * if there is no origin in header
+    res.setHeader("Access-Control-Allow-Origin", "*")
+  }
+  res.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Authorization")
+  res.setHeader("Access-Control-Allow-Methods", "GET,PUT,POST,PATCH,DELETE")
+  res.setHeader("Access-Control-Expose-Headers", "X-Total-Count, Link")
+  res.setHeader("Content-Type", "application/json; charset=utf-8")
+  // Deprecation headers for /narrower, /ancestors, /search, and /suggest
+  // TODO for 3.0: Remove these headers
+  if (["/narrower", "/ancestors", "/search", "/suggest"].includes(req.path)) {
+    res.setHeader("Deprecation", true)
+    const links = []
+    links.push(buildUrlForLinkHeader({ req, rel: "alternate" }))
+    links[0] = links[0].replace(req.path, `/concepts${req.path}`)
+    links.push("<https://github.com/gbv/jskos-server/releases/tag/v2.0.0>; rel=\"deprecation\"")
+    res.set("Link", links.join(","))
+  }
+  next()
+}
+
+/**
+ * Middleware that adds default properties:
+ *
+ * - If req.query exists, make sure req.query.limit and req.query.offset are set as numbers and make req.bulk a Boolean.
+ * - If possible, set req.type depending on the endpoint (one of concepts, schemes, mappings, annotations, suggest).
+ */
+const addMiddlewareProperties = (req, res, next) => {
+
+  if (req.query) {
+    const query = { ...req.query }
+
+    // Limit for pagination
+    const defaultLimit = 100
+    query.limit = parseInt(req.query.limit)
+    if (isNaN(query.limit) || req.query.limit <= 0) {
+      query.limit = defaultLimit
+    }
+    // Offset for pagination
+    const defaultOffset = 0
+    query.offset = parseInt(req.query.offset)
+    if (isNaN(query.offset) || req.query.offset < 0) {
+      query.offset = defaultOffset
+    }
+    // Bulk option for POST endpoints
+    query.bulk = query.bulk === "true" || query.bulk === "1"
+
+    // req.query is read-only since Express 5, so this is a hack.
+    // better create a custom query parser instead
+    // See <https://stackoverflow.com/questions/79597051/is-there-any-way-to-modify-req-query-in-express-v5>
+    Object.defineProperty(
+      req,
+      "query",
+      {
+        ...Object.getOwnPropertyDescriptor(req, "query"),
+        writable: false,
+        value: query,
+      })
+  }
+
+  // req.path -> req.type
+  let type = req.path.substring(1)
+  type = type.substring(0, type.indexOf("/") == -1 ? type.length : type.indexOf("/"))
+  if (type == "voc") {
+    if (req.path.includes("/top") || (req.path.includes("/concepts") && req.method !== "DELETE")) {
+      type = "concepts"
+    } else {
+      type = "schemes"
+    }
+  }
+  if (type == "mappings") {
+    if (req.path.includes("/suggest")) {
+      type = "suggest"
+    } else if (req.path.includes("/voc")) {
+      type = "schemes"
+    }
+  }
+  if (["concepts", "narrower", "ancestors", "search"].includes(type)) {
+    if (req.path.includes("/suggest")) {
+      type = "suggest"
+    } else {
+      type = "concepts"
+    }
+  }
+  if (type == "suggest" && _.get(req, "query.format", "").toLowerCase() == "jskos") {
+    type = "concepts"
+  }
+  req.type = type
+
+  // Add req.action
+  const action = {
+    GET: "read",
+    POST: "create",
+    PUT: "update",
+    PATCH: "update",
+    DELETE: "delete",
+  }[req.method]
+  req.action = action
+  // Add req.anonymous, req.crossUser, and req.auth if necessary
+  if (config[type] && config[type].anonymous) {
+    req.anonymous = true
+  }
+  if (["PUT", "PATCH", "DELETE"].includes(req.method)) {
+    if (config[type] && config[type][action] && config[type][action].crossUser) {
+      req.crossUser = config[type][action].crossUser
+    }
+  }
+  if (config[type] && config[type][action] && config[type][action].auth) {
+    req.auth = true
+  }
+  next()
+}
+
+/**
+ * Middleware that receives a list of supported download formats and overrides req.query.download if the requested format is not supported.
+ *
+ * @param {Array} formats
+ */
+const supportDownloadFormats = (formats) => (req, res, next) => {
+  if (req.query.download && !formats.includes(req.query.download)) {
+    req.query.download = null
+  }
+  next()
+}
+
+/**
+ * Sets pagination headers (X-Total-Count, Link) for a response.
+ * See also: https://developer.github.com/v3/#pagination
+ * For Link header rels:
+ * - first and last are always set
+ * - prev will be set if previous page exists (i.e. if offset > 0)
+ * - next will be set if next page exists (i.e. if offset + limit < total)
+ *
+ * Requires req.data to be set.
+ */
+const addPaginationHeaders = (req, res, next) => {
+  const limit = req.query.limit
+  const offset = req.query.offset
+  const total = req.data?.totalCount ?? req.data?.length ?? null
+
+  if (req == null || res == null || limit == null || offset == null) {
+    next()
+    return
+  }
+  // Set X-Total-Count header
+  if (total === null) {
+    // ! This is a workaround! We don't know the total number, so we just return an unreasonably high number here. See #176.
+    res.set("X-Total-Count", 9999999)
+  } else {
+    res.set("X-Total-Count", total)
+  }
+  let links = []
+  let query = _.cloneDeep(req.query)
+  query.limit = limit
+  // rel: first
+  query.offset = 0
+  links.push(buildUrlForLinkHeader({ req, query, rel: "first" }))
+  // rel: prev
+  if (offset > 0) {
+    query.offset = Math.max(offset - limit, 0)
+    links.push(buildUrlForLinkHeader({ req, query, rel: "prev" }))
+  }
+  // rel: next
+  if (total && limit + offset < total || req.data && req.data.length === limit) {
+    query.offset = offset + limit
+    links.push(buildUrlForLinkHeader({ req, query, rel: "next" }))
+  }
+  // rel: last
+  if (total !== null) {
+    let current = 0
+    while (current + limit < total) {
+      current += limit
+    }
+    query.offset = current
+    links.push(buildUrlForLinkHeader({ req, query, rel: "last" }))
+  } else if (req.data.length < limit) {
+    // Current page is last
+    links.push(buildUrlForLinkHeader({ req, query, rel: "last" }))
+  }
+  // Push existing Link header to the back
+  links.push(res.get("Link"))
+  // Set Link header
+  res.set("Link", links.join(","))
+  next()
+}
+
+/**
+ * Middleware that returns JSON given in req.data.
+ */
+const returnJSON = (req, res) => {
+  // Convert Mongoose documents into plain objects
+  let data
+  if (Array.isArray(req.data)) {
+    data = req.data.map(doc => doc?.toObject ? doc.toObject() : doc)
+    // Preserve totalCount
+    data.totalCount = req.data.totalCount
+  } else {
+    data = req.data?.toObject ? req.data?.toObject() : req.data
+  }
+  cleanJSON(data, 0, config.closedWorldAssumption)
+  let statusCode = 200
+  if (req.method == "POST") {
+    statusCode = 201
+  }
+  res.status(statusCode).json(data)
+}
+
+/**
+ * Middleware that handles download streaming.
+ * Requires a database cursor in req.data.
+ *
+ * @param {String} filename - resulting filename without extension
+ */
+const handleDownload = (filename) => (req, res) => {
+  let results = req.data, single = false
+  // Convert to stream if necessary
+  if (!(results instanceof Readable)) {
+    if (!Array.isArray(results)) {
+      single = true
+    }
+    results = new Readable({ objectMode: true })
+    results.push(req.data)
+    results.push(null)
+  }
+  /**
+   * Transformation object to remove _id parameter from objects in a stream.
+   */
+  const removeIdTransform = new Transform({
+    objectMode: true,
+    transform(chunk, encoding, callback) {
+      cleanJSON(chunk)
+      this.push(chunk)
+      callback()
+    },
+  })
+  // Default transformation: JSON
+  let transform = JSONStream.stringify(single ? "" : "[\n\t", ",\n\t", single ? "\n" : "\n]\n")
+  let fileEnding = "json"
+  let first = true, delimiter = ","
+  let csv
+  switch (req.query.download) {
+    case "ndjson":
+      fileEnding = "ndjson"
+      res.set("Content-Type", "application/x-ndjson; charset=utf-8")
+      transform = new Transform({
+        objectMode: true,
+        transform(chunk, encoding, callback) {
+          this.push(JSON.stringify(chunk) + "\n")
+          callback()
+        },
+      })
+      break
+    case "csv":
+    case "tsv":
+      fileEnding = req.query.download
+      if (req.query.download == "csv") {
+        delimiter = ","
+        res.set("Content-Type", "text/csv; charset=utf-8")
+      } else {
+        delimiter = "\t"
+        res.set("Content-Type", "text/tab-separated-values; charset=utf-8")
+      }
+      csv = jskos.mappingCSV({
+        lineTerminator: "\r\n",
+        creator: true,
+        schemes: true,
+        delimiter,
+      })
+      transform = new Transform({
+        objectMode: true,
+        transform(chunk, encoding, callback) {
+          // Small workaround to prepend a line to CSV
+          if (first) {
+            this.push(`"fromScheme"${delimiter}"fromNotation"${delimiter}"toScheme"${delimiter}"toNotation"${delimiter}"toNotation2"${delimiter}"toNotation3"${delimiter}"toNotation4"${delimiter}"toNotation5"${delimiter}"type"${delimiter}"creator"\n`)
+            first = false
+          }
+          this.push(csv.fromMapping(chunk, { fromCount: 1, toCount: 5 }))
+          callback()
+        },
+      })
+      break
+  }
+  // Add file header
+  res.set("Content-disposition", `attachment; filename=${filename}.${fileEnding}`)
+  // results is a database cursor
+  results
+    .pipe(removeIdTransform)
+    .pipe(transform)
+    .pipe(res)
+}
+
+/**
+ * Extracts a creator objects from a request.
+ *
+ * @param {*} req request object
+ */
+const getCreator = (req) => {
+  let creator = {}
+  const creatorUriPath = req.type === "annotations" ? "id" : "uri"
+  const creatorNamePath = req.type === "annotations" ? "name" : "prefLabel.en"
+  const userUris = getUrisOfUser(req.user)
+  if (req.user && !userUris.includes(req.query.identity)) {
+    _.set(creator, creatorUriPath, req.user.uri)
+  } else if (req.query.identity) {
+    _.set(creator, creatorUriPath, req.query.identity)
+  }
+  if (req.query.identityName) {
+    _.set(creator, creatorNamePath, req.query.identityName)
+  } else if (req.query.identityName !== "") {
+    const name = _.get(Object.values(_.get(req, "user.identities", [])).find(i => i.uri === _.get(creator, creatorUriPath)) || req.user, "name")
+    if (name) {
+      _.set(creator, creatorNamePath, name)
+    }
+  }
+  if (!_.get(creator, creatorUriPath) && !_.get(creator, creatorNamePath)) {
+    creator = null
+  }
+  return creator
+}
+
+/**
+ * See https://github.com/gbv/jskos-server/issues/153#issuecomment-997847433
+ *
+ * @param {Object} options.object JSKOS object
+ * @param {Object} [options.existing] existing object from database for PUT/PATCH
+ * @param {Object} [options.creator] creator object, usually extracted via `getCreator` above
+ * @param {Object} options.req request object (necessary for `type`, `user`, `method`, `anonymous`, and `auth`)
+ */
+const handleCreatorForObject = ({ object, existing, creator, req }) => {
+  if (!object) {
+    return object
+  }
+
+  if (req.type === "annotations") {
+    // No "contributor" for annotations
+    delete object.contributor
+  } else if (creator) {
+    // JSKOS creator has to be an array
+    creator = [creator]
+  }
+
+  const userUris = getUrisOfUser(req.user)
+  const anonymous = req.anonymous
+  const auth = req.auth
+
+  if (req.method === "POST") {
+    if (anonymous) {
+      delete object.creator
+      delete object.contributor
+    } else if (auth) {
+      if (creator) {
+        object.creator = creator
+      } else {
+        delete object.creator
+      }
+    }
+  } else if (req.method === "PUT" || req.method === "PATCH") {
+    if (anonymous) {
+      if (req.method === "PUT") {
+        object.creator = existing && existing.creator
+        object.contributor = existing && existing.contributor
+      } else {
+        delete object.creator
+        delete object.contributor
+      }
+    } else if (auth) {
+      if (existing && existing.creator) {
+        // Don't allow overriding existing creator
+        if (req.method === "PUT") {
+          object.creator = existing.creator
+        } else {
+          delete object.creator
+        }
+      } else if (object.creator && creator) {
+        // If creator is overridden, it can only be the user
+        object.creator = creator
+      }
+      // Update creator and/or add to contributor
+      if (creator) {
+        if (req.type === "annotations") {
+          // Only update creator if it's the user
+          if (userUris.includes((object.creator || existing && existing.creator || {}).id)) {
+            object.creator = creator
+          }
+        } else {
+          const findUserPredicate = c => jskos.compare(c, { identifier: userUris })
+          const objectCreatorIndex = (object.creator || []).findIndex(findUserPredicate)
+          const existingCreatorIndex = (existing && existing.creator || []).findIndex(findUserPredicate)
+          const objectContributorIndex = (object.contributor || []).findIndex(findUserPredicate)
+          const existingContributorIndex = (existing && existing.contributor || []).findIndex(findUserPredicate)
+          if (objectCreatorIndex !== -1) {
+            object.creator[objectCreatorIndex] = creator[0]
+          } else if (objectContributorIndex !== -1) {
+            object.contributor.splice(objectContributorIndex, 1)
+            object.contributor.push(creator[0])
+          } else if (existingCreatorIndex !== -1 && !object.creator) {
+            object.creator = existing.creator
+            object.creator[existingCreatorIndex] = creator[0]
+          } else if (existingContributorIndex !== -1 && !object.contributor) {
+            object.contributor = existing.contributor
+            object.contributor.splice(existingContributorIndex, 1)
+            object.contributor.push(creator[0])
+          } else {
+            object.contributor = (object.contributor || existing.contributor || []).concat(creator)
+          }
+        }
+      }
+    }
+  }
+  return object
+}
+
+/**
+ * Custom body parser middleware.
+ * - For POSTs, adds body stream via json-anystream and adjusts objects via handleCreatorForObject.
+ * - For PUT/PATCH/DELETE, parses JSON body, queries the existing entity which is saved in req.existing, checks creator, and adjusts object via handleCreatorForObject.
+ *
+ * @param {*} req
+ * @param {*} res
+ * @param {*} next
+ */
+const bodyParser = (req, res, next) => {
+
+  // Assemble creator once
+  const creator = getCreator(req)
+
+  // Wrap handleCreatorForObject method
+  const adjust = (object, existing) => {
+    return handleCreatorForObject({
+      object,
+      existing,
+      creator,
+      req,
+    })
+  }
+
+  if (req.method == "POST") {
+    // For POST requests, parse body with json-anystream middleware
+    anystream.addStream(adjust)(req, res, next)
+  } else {
+    // For all other requests, parse as JSON
+    express.json()(req, res, async (...params) => {
+      // Get existing
+      const uri = req.params._id || (req.body || {}).uri || req.query.uri
+      let existing
+      try {
+        existing = await services[req.type].getItem(uri)
+      } catch (error) {
+        // Ignore
+      }
+      if (!existing) {
+        next(new EntityNotFoundError(null, uri))
+      } else {
+        // Override certain properties with entities from database
+        // Note: For POST request, this needs to be done individually in the services/{entity}.js file.
+        if (["mappings", "annotations"].includes(req.type)) {
+          await services.schemes.replaceSchemeProperties(req.body, ["fromScheme", "toScheme"])
+          await services.schemes.replaceSchemeProperties(existing, ["fromScheme", "toScheme"])
+        }
+        let superordinated = {
+          existing: null,
+          payload: null,
+        }
+        // Check for superordinated object for existing (currently only `partOf`)
+        if (req.type === "mappings" && existing.partOf && existing.partOf[0]) {
+          // Get concordance via service
+          try {
+            const concordance = await services.concordances.getItem(existing.partOf[0].uri)
+            superordinated.existing = concordance
+          } catch (error) {
+            const message = `Existing concordance with URI ${existing.partOf[0].uri} could not be found in database.`
+            config.error(message)
+            next(new DatabaseInconsistencyError(message))
+          }
+        }
+        // Check superordinated object for payload
+        if (req.type === "mappings" && req.body && req.body.partOf && req.body.partOf[0]) {
+          // Get concordance via service
+          try {
+            const concordance = await services.concordances.getItem(req.body.partOf[0].uri)
+            superordinated.payload = concordance
+          } catch (error) {
+            next(new InvalidBodyError(`Concordance with URI ${req.body.partOf[0].uri} could not be found.`))
+          }
+        }
+        let creatorMatches = true
+        if (superordinated.existing) {
+          // creator or contributor must match for existing superordinated object
+          creatorMatches = creatorMatches && matchesCreator({ req, object: superordinated.existing, withContributors: true })
+        } else {
+          // creator needs to match for object that is updated
+          creatorMatches = creatorMatches && matchesCreator({ req, object: existing })
+        }
+        if (superordinated.payload) {
+          // creator or contributor must also match for the payload's superordinated object
+          creatorMatches = creatorMatches && matchesCreator({ req, object: superordinated.payload, withContributors: true })
+        }
+        if (!creatorMatches) {
+          next(new CreatorDoesNotMatchError())
+        } else {
+          req.existing = existing
+          req.body = adjust(req.body, existing)
+          next(...params)
+        }
+      }
+    })
+  }
+}
+
+export {
+  wrapAsync,
+  wrapDownload,
+  adjust,
+  matchesCreator,
+  addDefaultHeaders,
+  supportDownloadFormats,
+  addMiddlewareProperties,
+  addPaginationHeaders,
+  returnJSON,
+  handleDownload,
+  bodyParser,
+  getCreator,
+  handleCreatorForObject,
+}