jskos-server 2.4.0

package/test/validate.js

@@ -0,0 +1,226 @@
+// Tests for /validate endpoints
+
+import chai from "./chai.js"
+import * as server from "../server.js"
+import { globSync } from "glob"
+import fs from "node:fs"
+import assert from "node:assert"
+import { assertMongoDB, dropDatabaseBeforeAndAfter, setupInMemoryMongo, createCollectionsAndIndexes, teardownInMemoryMongo } from "./test-utils.js"
+
+let types = ["resource", "item", "concept", "scheme", "mapping", "concordance", "registry", "distributions", "occurrence", "bundle", "annotation"]
+let examples = {}
+
+// Import local examples
+for (let type of types) {
+  examples[type] = []
+  for (let expected of [true, false]) {
+    let files = globSync(`./node_modules/jskos-validate/examples/${type}/${expected ? "pass" : "fail"}/*.json`)
+    for (let file of files) {
+      try {
+        let object = JSON.parse(fs.readFileSync(file))
+        examples[type].push({
+          object,
+          expected,
+          file,
+        })
+      } catch(error) {
+        console.log("Unable to parse file", file)
+      }
+    }
+  }
+}
+
+describe("Validation endpoint testing", () => {
+  before(async () => {
+    const mongoUri = await setupInMemoryMongo({ replSet: false })
+    process.env.MONGO_URI = mongoUri
+    await createCollectionsAndIndexes()
+  })
+
+  after(async () => {
+    // close server if you started one
+    await teardownInMemoryMongo()
+  })
+
+  // 🗑 Drop DB before *and* after every single `it()` in this file
+  dropDatabaseBeforeAndAfter()
+
+  // 🔌 Sanity‐check that mongoose really is connected
+  assertMongoDB()
+
+
+  describe("Validation endpoint: jskos-validate tests", () => {
+
+    // Validate difference object types
+    for (let type of types) {
+      let typePlural =
+        type.endsWith("y") && !/[aeiou]y$/i.test(type)
+          ? type.slice(0, -1) + "ies"
+          : type + "s"
+      describe(typePlural, () => {
+        for (let { object, expected, file } of examples[type]) {
+          it(`should validate ${typePlural} (${file})`, done => {
+          // Support for arrays of objects
+            let objects = [object]
+            if (Array.isArray(object)) {
+              objects = object
+            }
+            for (let object of objects) {
+              chai.request.execute(server.app)
+                .post("/validate")
+                .query({
+                  type,
+                })
+                .send(object)
+                .end((error, res) => {
+                  assert.equal(error, null)
+                  res.should.have.status(201)
+                  res.body.should.be.an("array")
+                  if (expected) {
+                    assert.strictEqual(res.body[0], true)
+                  } else {
+                    res.body[0].should.be.an("array")
+                    assert.notEqual(res.body[0].length, 0)
+                  }
+                  done()
+                })
+            }
+          })
+        }
+      })
+    }
+  })
+
+  describe("Validation endpoint: parameters", () => {
+
+    it("should validate empty object without type parameter", done => {
+      chai.request.execute(server.app)
+        .post("/validate")
+        .send({})
+        .end((error, res) => {
+          assert.equal(error, null)
+          res.should.have.status(201)
+          res.body.should.be.an("array")
+          assert.equal(res.body[0], true)
+          done()
+        })
+    })
+
+    it("should fail validation for object with unknown parameter, but pass when `unknownFields` is set", done => {
+      const object = { abcdef: 1 }
+      chai.request.execute(server.app)
+        .post("/validate?type=concept")
+        .send(object)
+        .end((error, res) => {
+          assert.equal(error, null)
+          res.should.have.status(201)
+          res.body.should.be.an("array")
+          res.body[0].should.be.an("array")
+          assert.notEqual(res.body[0].length, 0)
+
+          // Set parameter
+          chai.request.execute(server.app)
+            .post("/validate")
+            .query({
+              unknownFields: true,
+            })
+            .send(object)
+            .end((error, res) => {
+              assert.equal(error, null)
+              res.should.have.status(201)
+              res.body.should.be.an("array")
+              assert.equal(res.body[0], true)
+              done()
+            })
+        })
+    })
+
+    it("should remember validated schemes if no type is set", done => {
+      const objects = [
+        {
+          type: ["http://www.w3.org/2004/02/skos/core#ConceptScheme"],
+          uri: "http://example.org/voc",
+          notationPattern: "[a-z]+",
+        },
+        {
+          type: ["http://www.w3.org/2004/02/skos/core#Concept"],
+          uri: "http://example.org/1",
+          notation: ["abc"],
+          inScheme: [{uri: "http://example.org/voc"}],
+        },
+        {
+          type: ["http://www.w3.org/2004/02/skos/core#Concept"],
+          uri: "http://example.org/2",
+          notation: ["123"],
+          inScheme: [{uri: "http://example.org/voc"}],
+        },
+      ]
+      chai.request.execute(server.app)
+        .post("/validate")
+        .send(objects)
+        .end((error, res) => {
+          assert.equal(error, null)
+          res.should.have.status(201)
+          res.body.should.be.an("array")
+          assert.equal(res.body.length, objects.length)
+          assert.equal(res.body[0], true)
+          assert.equal(res.body[1], true)
+          // Last concept should fail because notation does not match scheme's notationPattern
+          res.body[2].should.be.an("array")
+          assert.equal(res.body[2].length, 1)
+          done()
+        })
+    })
+
+    it("should POST a scheme, then use that scheme's notationPattern to validate objects when `knownSchemes` is set", done => {
+      const scheme = {
+        type: ["http://www.w3.org/2004/02/skos/core#ConceptScheme"],
+        uri: "http://example.org/voc",
+        notationPattern: "[a-z]+",
+      }
+      const objects = [
+        {
+          uri: "http://example.org/1",
+          notation: ["abc"],
+          inScheme: [{ uri: scheme.uri }],
+        },
+        {
+          uri: "http://example.org/2",
+          notation: ["123"],
+          inScheme: [{ uri: scheme.uri }],
+        },
+      ]
+      // 1. POST scheme
+      chai.request.execute(server.app)
+        .post("/voc")
+        .send(scheme)
+        .end((error, res) => {
+          assert.equal(error, null)
+          res.should.have.status(201)
+          res.body.should.be.an("object")
+          assert.equal(res.body.uri, scheme.uri)
+          // 2. Validate objects
+          chai.request.execute(server.app)
+            .post("/validate")
+            .query({
+              knownSchemes: true,
+            // type: concept is implied
+            })
+            .send(objects)
+            .end((error, res) => {
+              assert.equal(error, null)
+              res.should.have.status(201)
+              res.body.should.be.an("array")
+              // First concept should pass
+              assert.equal(res.body[0], true)
+              // Second concept should fail
+              res.body[1].should.be.an("array")
+              assert.notEqual(res.body[1].length, 0)
+              done()
+            })
+        })
+    })
+
+  })
+
+})

package/utils/adjust.js

@@ -0,0 +1,206 @@
+import _ from "lodash"
+import { AnnotationService } from "../services/annotations.js"
+import { ConceptService } from "../services/concepts.js"
+
+function createAdjuster(config) {
+  const services = {
+    annotations: new AnnotationService(config),
+    concepts: new ConceptService(config),
+  }
+
+  // Adjust data in req.data based on req.type (which is set by `addMiddlewareProperties`)
+  const adjust = async (req, res, next) => {
+  /**
+   * Skip adjustments if either:
+   * - there is no data
+   * - there is no data type (i.e. we don't know which adjustment method to use)
+   * - the request was a bulk operation
+   */
+    if (!req.data || !req.type || req.query.bulk) {
+      next()
+    }
+    req.data = await adjust.data({ req })
+    next()
+  }
+
+  // Wrapper around adjustments; `req` only has required property path `query.properties` if `data` and `type` are given.
+  adjust.data = async ({ req, data, type }) => {
+    data = data ?? req.data
+    type = type ?? req.type
+    // If data is still a mongoose object, convert it to plain object
+    if (Array.isArray(data) && data[0]?.toObject) {
+      data = data.map(item => item.toObject ? item.toObject() : item)
+    } else if (!Array.isArray(data) && data.toObject) {
+      data = data.toObject()
+    }
+    // Remove "s" from the end of type if it's not an array
+    if (!Array.isArray(data)) {
+      type = type.substring(0, type.length - 1)
+    }
+    if (adjust[type]) {
+      let addProperties = [], removeProperties = [], mode = 0 // mode 0 = add, mode 1 = remove
+      for (let prop of _.get(req, "query.properties", "").split(",")) {
+        if (prop.startsWith("*")) {
+          addProperties.push("narrower")
+          addProperties.push("ancestors")
+          addProperties.push("annotations")
+          continue
+        }
+        if (prop.startsWith("-")) {
+          mode = 1
+          prop = prop.slice(1)
+        } else if (prop.startsWith("+")) {
+          mode = 0
+          prop = prop.slice(1)
+        }
+        if (mode === 1) {
+          removeProperties.push(prop)
+        } else {
+          addProperties.push(prop)
+          // If a property is explicitly added after it was removed, it should not be removed anymore
+          removeProperties = removeProperties.filter(p => p !== prop)
+        }
+      }
+      addProperties = addProperties.filter(Boolean)
+      removeProperties = removeProperties.filter(Boolean)
+      // Adjust data with properties
+      data = await adjust[type](data, addProperties)
+      // Remove properties if necessary
+      const dataToAdjust = Array.isArray(data) ? data : [data]
+      removeProperties.forEach(property => {
+        dataToAdjust.filter(Boolean).forEach(entity => {
+          delete entity[property]
+        })
+      })
+    }
+    return data
+  }
+
+  // Add @context and type to annotations.
+  adjust.annotation = (annotation) => {
+    if (annotation) {
+      annotation["@context"] = "http://www.w3.org/ns/anno.jsonld"
+      annotation.type = "Annotation"
+    }
+    return annotation
+  }
+  adjust.annotations = annotations => {
+    return annotations.map(annotation => adjust.annotation(annotation))
+  }
+
+  // Add @context and type to concepts. Also load properties narrower, ancestors, and annotations if necessary.
+  adjust.concept = async (concept, properties = []) => {
+    if (concept) {
+      concept["@context"] = "https://gbv.github.io/jskos/context.json"
+      concept.type = concept.type || ["http://www.w3.org/2004/02/skos/core#Concept"]
+      // Add properties (narrower, ancestors)
+      for (let property of ["narrower", "ancestors"].filter(p => properties.includes(p))) {
+        concept[property] = await Promise.all((await services.concepts[`get${property.charAt(0).toUpperCase() + property.slice(1)}`]({ uri: concept.uri })).map(concept => adjust.concept(concept)))
+      }
+      // Add properties (annotations)
+      if (config.annotations && properties.includes("annotations") && concept.uri) {
+        concept.annotations = (await services.annotations.queryItems({ target: concept.uri })).map(annotation => adjust.annotation(annotation))
+      }
+    }
+    return concept
+  }
+  adjust.concepts = async (concepts, properties) => {
+    return await Promise.all(concepts.map(concept => adjust.concept(concept, properties)))
+  }
+
+  // Add @context to concordances.
+  adjust.concordance = (concordance) => {
+    if (concordance) {
+      concordance["@context"] = "https://gbv.github.io/jskos/context.json"
+      // Remove existing "distributions" array (except for external URLs)
+      concordance.distributions = (concordance.distributions || []).filter(dist => !dist.download || !dist.download.startsWith(config.baseUrl))
+      // Add distributions for JSKOS and CSV
+      concordance.distributions = [
+        {
+          download: `${config.baseUrl}mappings?partOf=${encodeURIComponent(concordance.uri)}&download=ndjson`,
+          format: "http://format.gbv.de/jskos",
+          mimetype: "application/x-ndjson; charset=utf-8",
+        },
+        {
+          download: `${config.baseUrl}mappings?partOf=${encodeURIComponent(concordance.uri)}&download=csv`,
+          mimetype: "text/csv; charset=utf-8",
+        },
+      ].concat(concordance.distributions)
+    }
+    return concordance
+  }
+  adjust.concordances = (concordances) => {
+    return concordances.map(concordance => adjust.concordance(concordance))
+  }
+
+  // Add @context to mappings. Also load annotations if necessary.
+  adjust.mapping = async (mapping, properties = []) => {
+    if (mapping) {
+      mapping["@context"] = "https://gbv.github.io/jskos/context.json"
+      // Add properties (annotations)
+      if (config.annotations && properties.includes("annotations") && mapping.uri) {
+        mapping.annotations = (await services.annotations.queryItems({ target: mapping.uri })).map(annotation => adjust.annotation(annotation))
+      }
+    }
+    return mapping
+  }
+  adjust.mappings = async (mappings, properties) => {
+    return await Promise.all(mappings.map(mapping => adjust.mapping(mapping, properties)))
+  }
+
+  // Add @context and type to schemes.
+  adjust.scheme = (scheme) => {
+    if (scheme) {
+      scheme["@context"] = "https://gbv.github.io/jskos/context.json"
+      scheme.type = scheme.type || ["http://www.w3.org/2004/02/skos/core#ConceptScheme"]
+      // Remove existing "distributions" array (except for external URLs)
+      scheme.distributions = (scheme.distributions || []).filter(dist => !dist.download || !dist.download.startsWith(config.baseUrl))
+      if (scheme.concepts && scheme.concepts.length) {
+      // If this instance contains concepts for this scheme, add distribution for it
+        scheme.distributions = [
+          {
+            download: `${config.baseUrl}voc/concepts?uri=${encodeURIComponent(scheme.uri)}&download=ndjson`,
+            format: "http://format.gbv.de/jskos",
+            mimetype: "application/x-ndjson; charset=utf-8",
+          },
+          {
+            download: `${config.baseUrl}voc/concepts?uri=${encodeURIComponent(scheme.uri)}&download=json`,
+            mimetype: "application/json; charset=utf-8",
+          },
+        ].concat(scheme.distributions)
+        // Also add `API` field if it does not exist
+        if (!scheme.API) {
+          scheme.API = [
+            {
+              type: "http://bartoc.org/api-type/jskos",
+              url: config.baseUrl,
+            },
+          ]
+        }
+      }
+      // Add distributions based on API field
+      (scheme.API || []).filter(api => api.type === "http://bartoc.org/api-type/jskos" && api.url !== config.baseUrl).forEach(api => {
+        scheme.distributions.push({
+          download: `${api.url}voc/concepts?uri=${encodeURIComponent(scheme.uri)}&download=ndjson`,
+          format: "http://format.gbv.de/jskos",
+          mimetype: "application/x-ndjson; charset=utf-8",
+        })
+        scheme.distributions.push({
+          download: `${api.url}voc/concepts?uri=${encodeURIComponent(scheme.uri)}&download=json`,
+          mimetype: "application/json; charset=utf-8",
+        })
+      })
+      if (!scheme.distributions.length) {
+        delete scheme.distributions
+      }
+    }
+    return scheme
+  }
+  adjust.schemes = (schemes) => {
+    return schemes.map(scheme => adjust.scheme(scheme))
+  }
+
+  return adjust
+}
+
+export { createAdjuster }

package/utils/auth.js

@@ -0,0 +1,154 @@
+/**
+ * Module that prepares authentication middleware via Passport.
+ *
+ * Exports a function that return default or optional authentication.
+ * Optional authentication should be used if `auth` is set to `false` for a particular endpoint.
+ * For example: app.get("/mappings", useAuth(config.mappings.read.auth), (req, res) => { ... })
+ * req.user will cointain the user if authorized, otherwise stays undefined.
+ */
+
+import _ from "lodash"
+import { ForbiddenAccessError } from "../errors/index.js"
+
+import config from "../config/index.js"
+
+import passport from "passport"
+import { Strategy as JwtStrategy, ExtractJwt } from "passport-jwt"
+import { Strategy as AnonymousStrategy } from "passport-anonymous"
+
+passport.use(new AnonymousStrategy())
+
+const actions = {
+  POST: "create",
+  PUT: "update",
+  PATCH: "update",
+  DELETE: "delete",
+}
+
+// Add some properties and methods related to authentication
+// This middleware is added to both auth.main and auth.optional
+const authPreparation = (req, res, next) => {
+  // Add action
+  req.action = actions[req.method] || "read"
+
+  // Add user URIs and providers
+  req.uris = [req.user?.uri].concat(Object.values(req.user?.identities || {}).map(id => id.uri)).filter(Boolean)
+  req.userProviders = Object.keys(req.user?.identities || {})
+
+  // Add isAuthorizedFor method
+  req.isAuthorizedFor = function ({ type, action, whitelist, providers, throwError = false } = {}) {
+    type = type ?? this.type
+    action = action ?? this.action
+
+    if (!config[type]?.[action]?.auth && type !== "checkAuth") {
+      // If action does not require auth at all, the request is authorized
+      return true
+    } else if (!this.user) {
+      // If action requires auth, but user isn't logged in, the request is not authorized
+      // For routes using the `auth.main` middleware, this is called early, but not for routes with `auth.optional`.
+      if (throwError) {
+        throw new ForbiddenAccessError("Access forbidden. Could not authenticate via JWT.")
+      }
+      return false
+    }
+
+    if (whitelist === undefined) {
+      whitelist = expandWhiteList(config[type]?.[action]?.identities, config.identityGroups)
+    }
+    providers = providers ?? config[type]?.[action]?.identityProviders
+
+    if (whitelist && _.intersection(whitelist, this.uris).length == 0) {
+      if (throwError) {
+        throw new ForbiddenAccessError("Access forbidden. A whitelist is in place, but authenticated user is not on the whitelist.")
+      }
+      return false
+    }
+
+    if (providers && _.intersection(providers, this.userProviders).length == 0) {
+      if (throwError) {
+        throw new ForbiddenAccessError("Access forbidden, missing identity provider. One of the following providers is necessary: " + providers.join(", "))
+      }
+      return false
+    }
+
+    return true
+  }
+
+  next()
+}
+
+function expandWhiteList(whitelist, identityGroups) {
+  if (whitelist && identityGroups) {
+    return whitelist.map(uri => uri in identityGroups ? identityGroups[uri].identities : uri).flat()
+  }
+  return whitelist
+}
+
+let optional = []
+let auth = (req, res, next) => {
+  next(new ForbiddenAccessError("Access forbidden. No authentication configured."))
+}
+
+// Prepare authorization via JWT
+if (config.auth.algorithm && config.auth.key) {
+  const opts = {
+    jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(),
+    secretOrKey: config.auth.key,
+    algorithms: [config.auth.algorithm],
+  }
+  try {
+    const strategy = new JwtStrategy(opts, (jwt_payload, done) => done(null, jwt_payload.user))
+    const strategyName = "jwt" // TODO: derive strategyName from opts
+
+    passport.use(strategyName, strategy)
+    optional.push(strategyName)
+
+    // Use like this: app.get("/secureEndpoint", auth, (req, res) => { ... })
+    // res.user will contain the current authorized user.
+    auth = (req, res, next) => {
+      passport.authenticate(strategyName, { session: false }, (error, user) => {
+        if (error || !user) {
+          return next(new ForbiddenAccessError("Access forbidden. Could not authenticate via JWT."))
+        }
+        req.user = user
+        return next()
+      })(req, res, next)
+    }
+  } catch(error) {
+    config.error("Error setting up JWT authentication")
+  }
+} else {
+  config.warn("Note: To provide authentication via JWT, please add `auth.algorithm` and `auth.key` to the configuration file!")
+}
+
+// Configure identities whitelists and identity providers
+const authAuthorize = (req, res, next) => {
+  let whitelist, providers, type, action
+
+  if (req.type == "checkAuth") {
+    ({ type, action } = req.query || {})
+    if (type && action && config[type][action]) {
+      whitelist = config[type][action].identities
+      providers = config[type][action].identityProviders
+    } else {
+      whitelist = config.identities
+      providers = config.identityProviders
+    }
+  }
+
+  whitelist = expandWhiteList(whitelist, config.identityGroups)
+
+  try {
+    req.isAuthorizedFor({ type, action, whitelist, providers, throwError: true })
+    next()
+  } catch (error) {
+    next(error)
+  }
+}
+
+// Also use anonymous strategy for endpoints that can be used authenticated or not authenticated
+optional.push("anonymous")
+
+export const useAuth = required => required
+  ? [auth, authPreparation, authAuthorize]
+  : [passport.authenticate(optional, { session: false }), authPreparation]

package/utils/changes.js

@@ -0,0 +1,88 @@
+// utils/changes.js
+import * as jskos from "jskos-tools"
+import { connection, waitForReplicaSet } from "./db.js"
+import { ConfigurationError } from "../errors/index.js"
+
+export const collections = {
+  voc:          "terminologies",
+  concepts:     "concepts",
+  mappings:     "mappings",
+  concordances: "concordances",
+  annotations:  "annotations",
+  registries:   "registries",
+}
+
+export const operationTypeMap = { insert: "create", update: "update", delete: "delete" }
+export let isChangesApiAvailable = false
+
+export default function registerChangesRoutes(app) {
+  for (const [route, collName] of Object.entries(collections)) {
+    app.ws(`/${route}/changes`, (ws) => {
+      const stream = connection.db
+        .collection(collName)
+        .watch([], { fullDocument: "updateLookup" })
+
+      stream.on("change", change => {
+        const { operationType, documentKey, fullDocument } = change
+        const evt = {
+          objectType: jskos.guessObjectType(fullDocument),
+          type:       operationTypeMap[operationType],
+          id:         documentKey._id,
+          timestamp:  clusterTimeToISOString(change.clusterTime),
+          ...(operationType !== "delete" && { document: fullDocument }),
+        }
+        ws.send(JSON.stringify(evt))
+      })
+
+      stream.on("error", (err) => {
+        if (err?.name === "MongoClientClosedError") {
+          return
+        }
+        console.error(
+          `[changes] ChangeStream error on "${route}" (${collName}):`,
+          err,
+        )
+      })
+
+      ws.on("close", () => stream.close())
+    })
+  }
+}
+
+// After DB connection, conditionally enable change-stream routes
+export async function setupChangesApi(app, config) {
+  if (!config.changes) {
+    console.log("Changes API is disabled by configuration.")
+    return
+  }
+
+  if (!await waitForReplicaSet(config.changes)) {
+    throw new ConfigurationError(
+      "Changes API enabled, but MongoDB replica set did not initialize in time.",
+    )
+  }
+
+  // Register WebSocket change-stream endpoints
+  registerChangesRoutes(app)
+  isChangesApiAvailable = true
+  console.log("Changes API enabled: replica set confirmed, endpoints are registered.")
+}
+
+/**
+ * Converts a MongoDB cluster time object to an ISO 8601 string.
+ *
+ * @param {{ getHighBits: () => number } | null} clusterTime - The cluster time object.
+ * @returns {string | null} ISO timestamp derived from the cluster time, or null if input is invalid.
+ */
+function clusterTimeToISOString(clusterTime) {
+  if ((!clusterTime) || (typeof clusterTime.getHighBits !== "function")) {
+    return null
+  }
+
+  const seconds = clusterTime.getHighBits()
+  if (typeof seconds !== "number" || !Number.isFinite(seconds)) {
+    return null
+  }
+
+  return new Date(seconds * 1000).toISOString()
+}