jskos-server 2.4.0

package/config/config.test.json

@@ -0,0 +1,107 @@
+{
+  "title": "Testing",
+  "auth": {
+    "algorithm": "HS256",
+    "key": "test"
+  },
+  "schemes": {
+    "read": {
+      "auth": false
+    },
+    "create": {
+      "auth": false
+    },
+    "update": {
+      "auth": false,
+      "crossUser": true
+    },
+    "delete": {
+      "auth": false,
+      "crossUser": true
+    }
+  },
+  "concepts": {
+    "read": {
+      "auth": false
+    },
+    "create": {
+      "auth": false
+    },
+    "update": {
+      "auth": false,
+      "crossUser": true
+    },
+    "delete": {
+      "auth": true,
+      "identities": ["a:group"]
+    }
+  },
+  "concordances": {
+    "read": {
+      "auth": false
+    },
+    "create": {
+      "auth": true
+    },
+    "update": {
+      "auth": true,
+      "crossUser": false
+    },
+    "delete": {
+      "auth": true,
+      "crossUser": false
+    }
+  },
+  "mappings": {
+    "read": {
+      "auth": false
+    },
+    "create": {
+      "auth": true
+    },
+    "update": {
+      "auth": true,
+      "crossUser": false
+    },
+    "delete": {
+      "auth": true,
+      "crossUser": false
+    },
+    "fromSchemeWhitelist": null,
+    "toSchemeWhitelist": null,
+    "anonymous": false
+  },
+  "annotations": {
+    "read": {
+      "auth": false
+    },
+    "create": {
+      "auth": true
+    },
+    "update": {
+      "auth": true,
+      "crossUser": false
+    },
+    "delete": {
+      "auth": true,
+      "crossUser": false,
+      "identityProviders": null
+    },
+    "moderatingIdentities": ["http://test-moderating.user"],
+    "mismatchTagVocabulary": {
+      "uri": "https://uri.gbv.de/terminology/mismatch/"
+    }
+  },
+  "identities": ["http://test.user", "http://test-moderating.user"],
+  "identityProviders": ["test"],
+  "identityGroups": {
+    "a:group": {
+      "identities": [
+        "http://in-group.user"
+      ],
+      "claims": [
+      ]
+    }
+  },
+  "namespace": "f06ab008-a909-40a2-b8e7-643e955b522d"
+}

package/config/index.js

@@ -0,0 +1,77 @@
+import _ from "lodash"
+import { validateConfig, setupConfig } from "../config/setup.js"
+import fs from "node:fs"
+import path from "node:path"
+
+import { v4 as uuid } from "uuid"
+
+// Prepare environment
+import * as dotenv from "dotenv"
+dotenv.config()
+const env = process.env.NODE_ENV
+const configFile = process.env.CONFIG_FILE || "./config.json"
+
+const __dirname = import.meta.dirname
+
+// Adjust path if it's relative
+let configFilePath
+if (configFile.startsWith("/")) {
+  configFilePath = configFile
+} else {
+  configFilePath = path.resolve(__dirname, configFile)
+}
+
+// If file doesn't exist, create it with an empty array
+if (env !== "test" && !fs.existsSync(configFilePath)) {
+  fs.writeFileSync(configFilePath, "{}")
+}
+
+// Load environment config
+let configEnv = {}
+let configEnvFile = path.resolve(__dirname, `./config.${env}.json`)
+if (fs.existsSync(configEnvFile)) {
+  configEnv = JSON.parse(fs.readFileSync(configEnvFile))
+  console.log(`Read configuration from ${configEnvFile}`)
+}
+// Load user config
+let configUser = {}
+if (env !== "test") {
+  configUser = JSON.parse(fs.readFileSync(configFilePath))
+  console.log(`Read configuration from ${configFilePath}`)
+}
+
+// Validate
+Object.entries({ environment: configEnv, user: configUser }).forEach(([name, config]) => {
+  try {
+    validateConfig(config)
+  } catch(error) {
+    console.error(`Could not validate ${name} configuration: ${error}`)
+    process.exit(1)
+  }
+})
+
+// Validate
+try {
+  validateConfig(configEnv)
+} catch(error) {
+  console.error(`Could not validate environemnt configuration: ${error}`)
+  process.exit(1)
+}
+try {
+  validateConfig(configUser)
+} catch(error) {
+  console.error(`Could not validate user configuration: ${error}`)
+  process.exit(1)
+}
+
+// Before merging, check whether `namespace` exists in the user config and if not, generate a namespace and save it to user config
+if (!configUser.namespace && env != "test") {
+  configUser.namespace = uuid()
+  fs.writeFileSync(configFilePath, JSON.stringify(configUser, null, 2))
+  console.log(`Info/Config: Created a namespace and wrote it to ${configFilePath}.`)
+}
+
+const config = _.defaultsDeep({ env }, configEnv, configUser)
+setupConfig(config)
+
+export default config

package/config/setup.js

@@ -0,0 +1,212 @@
+import _ from "lodash"
+
+import AJV from "ajv"
+import addAjvFormats from "ajv-formats"
+
+import configSchema from "./config.schema.json" with { type: "json" }
+import statusSchema from "../status.schema.json" with { type: "json" }
+import configDefault from "./config.default.json" with { type: "json" }
+import info from "../package.json" with { type: "json" }
+
+const ajv = new AJV({ allErrors: true })
+addAjvFormats(ajv)
+ajv.addSchema(configSchema)
+ajv.addSchema(statusSchema)
+
+function ajvErrorsToString(errors) {
+  let message = ""
+  for (let error of errors || []) {
+    switch (error.keyword) {
+      case "additionalProperties":
+        message += `${error.dataPath} ${error.message} (${error.params.additionalProperty})`
+        break
+      default:
+        message += `${error.dataPath} ${error.message} (${error.schemaPath})`
+    }
+    message += "\n      "
+  }
+  return message.trimEnd()
+}
+
+export function validateConfig(data) {
+  if (!ajv.validate(configSchema, data)) {
+    throw new Error(ajvErrorsToString(ajv.errors))
+  }
+  for (let type in (data.registries?.types || {})) {
+    if (data.registries.types[type].mustExist && data.registries.types[type].uriRequired === false) {
+      throw new Error(`Registry member type ${type} mustExist so uriRequired must not be false`)
+    }
+  }
+  return data
+}
+
+export function validateStatus(data) {
+  if (!ajv.validate(statusSchema, data)) {
+    throw new Error(ajvErrorsToString(ajv.errors))
+  }
+  return data
+}
+
+export function setupConfig(config) {
+  config.env = config.env ?? "development"
+  const testing = config.env === "test"
+
+  // Merge in default values
+  config = _.defaultsDeep(config, configDefault)
+  const defaultChangesConfig = { retries: 20, interval: 5000 }
+  if (config.changes === true) {
+    config.changes = defaultChangesConfig
+  } else if (config.changes) {
+    config.changes = { ...defaultChangesConfig, ...config.changes }
+  }
+
+  // Add versions from package
+  config.version = info.apiVersion
+  config.serverVersion = info.version
+
+  // Logging functions
+  config.log = (...args) => {
+    if (!testing && (config.verbosity === true || config.verbosity === "log")) {
+      console.log(new Date(), ...args)
+    }
+  }
+  config.warn = (...args) => {
+    if (!testing && (config.verbosity === true || config.verbosity === "log" || config.verbosity === "warn")) {
+      console.warn(new Date(), ...args)
+    }
+  }
+  config.error = (...args) => {
+    if (!testing && config.verbosity !== false) {
+      console.error(new Date(), ...args)
+    }
+  }
+
+  // Set composed config variables
+  config.mongo.auth = config.mongo.user ? `${config.mongo.user}:${config.mongo.pass}@` : ""
+  config.mongo.url = `mongodb://${config.mongo.auth}${config.mongo.host}:${config.mongo.port}`
+  // Adjust database name during tests
+  if (testing) {
+    config.mongo.db += "-test-" + config.namespace
+  }
+
+  // Set baseUrl to localhost if not set
+  if (!config.baseUrl) {
+    Object.defineProperty(config, "baseUrl", {
+      get: function () {
+        return `http://localhost:${this.port}/`
+      },
+    })
+  }
+  if (!config.baseUrl.endsWith("/")) {
+    config.baseUrl += "/"
+  }
+
+  // Further expansion of config
+  const defaultActions = {
+    read: {
+      auth: false,
+    },
+    create: {
+      auth: true,
+    },
+    update: {
+      auth: true,
+      crossUser: false,
+    },
+    delete: {
+      auth: true,
+      crossUser: false,
+    },
+  }
+  const allTypes = ["schemes", "concepts", "mappings", "concordances", "annotations", "registries"]
+  for (let type of allTypes) {
+    if (config[type] === true) {
+    // Default is read-only without authentication
+      config[type] = {
+        read: {
+          auth: false,
+        },
+      }
+    }
+    if (config[type]) {
+      for (let action of ["read", "create", "update", "delete"]) {
+        if (config[type][action] === true) {
+          config[type][action] = defaultActions[action]
+        }
+        // Fill identities, identityProviders, and ips if necessary (not for read)
+        if (config[type][action] && action != "read") {
+          for (let prop of ["identities", "identityProviders", "ips"]) {
+            if (config[type][action][prop] === undefined) {
+              const value = config[type][prop] || config[prop]
+              if (value) {
+                config[type][action][prop] = value
+              }
+            }
+          }
+        }
+      }
+      // Fill in origin of URIs
+      if (config[type].create) {
+        const { uriBase, uriOrigin } = config[type].create
+        if (!uriOrigin) {
+          config[type].create.uriOrigin = "external"
+        } else if (uriOrigin !== "external" && uriBase === false) {
+          throw new Error(`uriBase of ${type}.create must not be false if uriOrigin is not external`)
+        }
+
+        //if (type
+        // TODO: hard-coded settings for mappings, concordances, and annotations
+      }
+    }
+  }
+
+  // mappings: if anonymous is given, assume crossUser for update and delete
+  if (config.mappings?.anonymous) {
+    if (config.mappings.update) {
+      config.mappings.update.crossUser = true
+    }
+    if (config.mappings.delete) {
+      config.mappings.delete.crossUser = true
+    }
+  }
+
+  // registries: assume types as enabled above
+  if (config.registries) {
+    // default types
+    if (!config.registries.types) {
+      config.registries.types = Object.fromEntries(allTypes.map(type => [type, !!config[type]]))
+    }
+    const { types } = config.registries
+    for (let type in types) {
+      // default values
+      if (types[type] === true) {
+        types[type] = {
+          mustExist: false,
+          skipInvalid: false,
+        }
+      }
+      if (types[type] && !("uriRequired" in types[type])) {
+        types[type].uriRequired = true
+      }
+      if (types[type]?.mustExist && !config[type]?.read) {
+        config.warn(`registry with member type ${type} require URI but config.${type}.read is not enabled!`)
+      }
+    }
+  }
+
+  // Adjust annotations.mismatchTagVocabulary to have the correct "API" field so that clients using cocoda-sdk can natively query it
+  if (config.annotations?.mismatchTagVocabulary) {
+    if (!config.annotations?.mismatchTagVocabulary?.API?.[0]) {
+      config.annotations.mismatchTagVocabulary.API = [
+        {
+          type: "http://bartoc.org/api-type/jskos",
+          url: config.baseUrl,
+        },
+      ]
+    } else {
+      config.warn("annotations.mismatchTagVocabulary currently does not support loading concepts from an external API. It will be attempted to load concepts from this instance instead.")
+    }
+  }
+
+  return config
+}

package/docker/.env

@@ -0,0 +1 @@
+CONFIG_FILE=/config/config.json

package/docker/Dockerfile

@@ -0,0 +1,20 @@
+FROM node:22-alpine
+
+WORKDIR /usr/src/app
+
+# Copy and install dependencies
+COPY package*.json ./
+RUN npm ci
+
+# Bundle app source
+COPY . .
+
+EXPOSE 3000
+
+RUN mkdir /config
+COPY docker/.env .env
+
+# Use pm2 to run app
+RUN npm i -g pm2
+
+CMD ["/usr/src/app/docker/docker-entrypoint.sh"]

package/docker/README.md

@@ -0,0 +1,175 @@
+# [JSKOS Server](https://github.com/gbv/jskos-server)
+
+JSKOS Server implements the JSKOS API web service and storage for [JSKOS](https://gbv.github.io/jskos/jskos.html) data such as controlled vocabularies, concepts, and concept mappings. It is part of a larger infrastructure of [Project coli-conc](https://coli-conc.gbv.de).
+
+- See [GitHub](https://github.com/gbv/jskos-server) for more information about the tool.
+
+**Note:** The old Docker Hub image (`coliconc/jskos-server`) is deprecated as of March 2023 and will not be updated anymore. We are moving all our Docker images to GitHub's Container Registry. From now on, **all new Docker images** will be available under `ghcr.io/gbv/jskos-server` (https://github.com/gbv/jskos-server/pkgs/container/jskos-server). Old images will still be available through Docker Hub for the foreseeable future.
+
+## Supported Architectures
+Currently, only `x86-64` is supported, but we are planning to add more architectures soon.
+
+## Available Tags
+- The current release version is available under `latest`. However, new major versions might break compatibility of the previously used config file, therefore it is recommended to use a version tag instead.
+- We follow SemVer for versioning the application. Therefore, `x` offers the latest image for the major version x, `x.y` offers the latest image for the minor version x.y, and `x.y.z` offers the image for a specific patch version x.y.z.
+- Additionally, the latest development version is available under `dev`.
+
+
+
+## Usage
+
+It is recommended to run the image using [Docker Compose](https://docs.docker.com/compose/) together with the required MongoDB database. Note that depending on your system, it might be necessary to use `sudo docker compose`. For older Docker versions, use `docker-compose` instead of `docker compose`.
+
+JSKOS Server can run with **or without** MongoDB configured as a **replica set**, depending on whether you want to use the **Change Streams WebSocket API** (`/changes`).
+
+If you **do not need** real-time change notifications via WebSocket, you can run MongoDB in standalone mode. This is the simplest and most common setup.
+
+
+1. Create `docker-compose.yml`
+
+##### Option 1: Without Replica Set (simpler setup)
+
+In this setup, the Change API must remain **disabled** (default). The server will run normally without real-time WebSocket updates.
+
+```yaml
+version: "3"
+
+services:
+  jskos-server:
+    image: ghcr.io/gbv/jskos-server
+    # replace this with your UID/GID if necessary (id -u; id -g); remove on macOS/Windows
+    user: 1000:1000
+    depends_on:
+      - mongo
+    volumes:
+      - ./data/config:/config
+    # environment:
+    #   - NODE_ENV=production # note that this requires the server to be run behind a HTTPS proxy
+    ports:
+      - 3000:3000
+    restart: unless-stopped
+
+  mongo:
+    image: mongo:7
+    # replace this with your UID/GID if necessary (id -u; id -g); remove on macOS/Windows
+    user: 1000:1000
+    volumes:
+      - ./data/db:/data/db
+    restart: unless-stopped
+```
+
+##### Option 2: With Replica Set (for Change Streams support)
+
+If you want to use the WebSocket API for Change Streams, MongoDB must be configured as a **replica set**. You can do this with an additional setup container. In this setup, you **can enable** `changes` in your `config.json` to use the `/changes` WebSocket endpoint.
+
+```yaml
+version: "3"
+
+services:
+  jskos-server:
+    image: ghcr.io/gbv/jskos-server
+    # replace this with your UID/GID if necessary (id -u; id -g); remove on macOS/Windows
+    user: 1000:1000
+    depends_on:
+      - mongo
+    volumes:
+      - ./data/config:/config
+    # environment:
+    #   - NODE_ENV=production # note that this requires the server to be run behind a HTTPS proxy
+    ports:
+      - 3000:3000
+    restart: unless-stopped
+
+  mongo:
+    image: mongo:7
+    # replace this with your UID/GID if necessary (id -u; id -g); remove on macOS/Windows
+    user: 1000:1000
+    entrypoint: [ "/usr/bin/mongod",  # The MongoDB server binary
+    "--bind_ip_all",                  # Permit connections from any network interface
+    "--replSet", "rs0"                # Start in replica‐set mode, naming the set “rs0”
+    ]
+    volumes:
+      - ./data/db:/data/db
+    restart: unless-stopped
+
+  mongo-setup-replica-set:
+    image: mongo:7
+    depends_on:
+      - mongo
+    volumes:
+      - ./mongo-initdb.d:/docker-entrypoint-initdb.d
+    restart: "no"
+```
+
+You'll also need to add a `mongo_setup.sh` script in `./mongo-initdb.d/` to initialize the replica set.
+
+
+2. Create data folders:
+
+```bash
+mkdir -p ./data/{config,db}
+```
+
+3. Start the application:
+
+```bash
+docker compose up -d
+```
+
+This will create and start a jskos-server container running under host port 3000 with data persistence under `./data`:
+
+- `./data/config`: configuration files required for jskos-server (`config.json`), see below
+- `./data/db`: data of the MongoDB container (note: make sure MongoDB data persistence works with your system, see section "Where to Store Data" [here](https://hub.docker.com/_/mongo))
+
+Note that the `user` directives in the compose file make sure that the generated files are owned by your host user account. Use `id -u`/`id -g` to find out your UID/GID respectively, or remove the directives if you are using Docker on macOS/Windows.
+
+You can now access the application under `http://localhost:3000`.
+
+## Application Setup
+Note: After adjusting any configurations, it is required to restart or recreate the container:
+- After changing configuration files, restart the container: `docker compose restart jskos-server`
+- After changing `docker-compose.yml` (e.g. adjusting environment variables), recreate the container: `docker compose up -d`
+
+
+### Replica Set Initialization Script
+
+Place the following `mongo_setup.sh` (or similar) in `./mongo-initdb.d/` on the host:
+
+```bash
+#!/bin/bash
+
+echo "🕒 Sleeping for 10 seconds to allow MongoDB to start..."
+sleep 10
+
+echo "⚙️ Initiating Replica Set at $(date)"
+
+mongosh --host mongo:27017 <<EOF
+rs.initiate({
+  _id: "rs0",
+  members: [ { _id: 0, host: "mongo:27017", priority: 1 } ]
+});
+EOF
+```
+
+* **Sleep**: ensures the `mongo` service is accepting connections.
+* **`mongosh`**: connects to the `mongo` container by its Docker network alias.
+* **`rs.initiate()`**: configures the single-node replica set named `rs0`.
+
+With this setup, **jskos-server** can safely use MongoDB change streams (`db.collection.watch()`) for real-time WebSocket notifications.
+
+### Configuration
+The folder `/config` (mounted as `./data/config` if configured as above) contains the configuration file `config.json` where jskos-server is configured. Please refer to the [documentation on GitHub](https://github.com/gbv/jskos-server#configuration) to see how to configure jskos-server. Note that this image creates a configuration file if none is found on startup. By default, the MongoDB `mongo` will be used (as configured above).
+
+### Environment Variables
+
+| Environment Variable | Description                                                                                   | Example Value       |
+|----------------------|-----------------------------------------------------------------------------------------------|---------------------|
+| `NODE_ENV`           | Should be set to `production` if used in production.*                                         | production          |
+| `CONFIG_FILE`        | Path to the configuration file **inside the container**. Usually does not need to be changed. | /config/config.json |
+
+*If `NODE_ENV` is set to production (which is recommended for productive use), jskos-server expects to be run behind a proxy using HTTPS. This also means that the config option `baseUrl` is required (see [documentation on GitHub](https://github.com/gbv/jskos-server#configuration)).
+
+### Data Import
+To get your data into jskos-server, please refer to [this section](https://github.com/gbv/jskos-server#data-import) in the documentation. For those commands to work inside the Docker container, skip the linking part and replace `./bin/import.js` with `docker compose exec jskos-server /usr/src/app/bin/import.js` for each command.
+
+**Note:** If files are imported, these have to be mounted into the container first, and the path inside the container has to be given. For example, you could mount the host folder `./data/imports` to `/imports` inside the container and then use the path `/imports/myfile.ndjson` with the import command. **We are planning to improve the import process with the 1.2.0 release, so those commands will change in the near future.** It will also be possible to upload files via a web interface so that no commands will be necessary anymore.

package/docker/docker-compose.yml

@@ -0,0 +1,29 @@
+version: "3"
+
+services:
+  jskos-server:
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile
+    depends_on:
+      - mongo
+    volumes:
+      - ./data/config:/config
+    ports:
+      - 3000:3000
+    restart: unless-stopped
+
+  mongo:
+    image: mongo:7
+    entrypoint: [ "/usr/bin/mongod", "--bind_ip_all", "--replSet", "rs0" ]
+    volumes:
+      - ./data/db:/data/db
+    restart: unless-stopped
+
+  mongo-setup-replica-set:
+    image: mongo:7
+    depends_on:
+      - mongo
+    volumes: 
+      - ./mongo-initdb.d:/docker-entrypoint-initdb.d
+    restart: "no"

package/docker/docker-entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# Add configuration file if necessary
+if [ ! -f /config/config.json ]; then
+  echo '{"mongo":{"host":"mongo"}}' > /config/config.json
+fi
+
+pm2-runtime server.js

package/docker/mongo-initdb.d/mongo_setup.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+echo "🕒 Sleeping for 10 seconds to allow MongoDB to start..."
+sleep 10
+
+echo "📅 mongo_setup.sh time now: $(date +"%T")"
+
+echo "⚙️ Initiating Replica Set..."
+
+mongosh --host mongo:27017 <<EOF
+rs.initiate({
+  _id: "rs0",
+  version: 1,
+  members: [
+    {
+      _id: 0,
+      host: "mongo:27017",
+      priority: 1
+    }
+  ]
+});
+EOF

package/ecosystem.example.json

@@ -0,0 +1,7 @@
+{
+  "name": "jskos-server",
+  "script": "./server.js",
+  "env": {
+    "NODE_ENV": "production"
+  }
+}