aiiinotate 0.2.0

package/migrations/migrationScripts/20251002141951-manifest2-schema.js

@@ -0,0 +1,43 @@
+/**
+ * define the validation schema for collection `manifests2`.
+ */
+
+import build from "#src/app.js"
+
+/**
+ * @param db {import('mongodb').Db}
+ * @param client {import('mongodb').MongoClient}
+ * @returns {Promise<void>}
+ */
+export const up = async (db, client) => {
+  const
+    fastify = await build(),
+    fastifySchema = fastify.schemasPresentation2.getSchema("manifestMongo"),
+    schema = fastify.schemasResolver(fastifySchema),
+    commandDoc = {
+      collMod: "manifests2",
+      validator: { $jsonSchema: schema },
+    },
+    r = await db.command(commandDoc);
+
+
+  if ( r.ok !== 1 ) {
+    throw new Error(`command failed with error: ${r}`);
+  }
+};
+
+/**
+ * @param db {import('mongodb').Db}
+ * @param client {import('mongodb').MongoClient}
+ * @returns {Promise<void>}
+ */
+export const down = async (db, client) => {
+  const r = await db.command({
+    collMod: "manifests2",
+    validator: {}
+  });
+  if ( r.ok !== 1 ) {
+    throw new Error(`command failed with error: ${r}`);
+  }
+};
+

package/migrations/migrationScripts/20251006212110-manifest-unique-index.js

@@ -0,0 +1,29 @@
+/** create a unique index on `manifests2.@id` */
+
+const
+  colName = "manifests2",
+  indexSpec = { "@id": 1 },
+  indexOptions = { name: "manifestIdIndex", unique: true };
+
+/**
+ * @param {import('mongodb').Db} db
+ * @param {import('mongodb').MongoClient} client
+ * @returns {Promise<void>}
+ */
+export const up = async (db, client) => {
+  const collection = db.collection(colName);
+  const result = await collection.createIndex(indexSpec, indexOptions);
+  console.log("created index:", result);
+};
+
+/**
+ * @param {import('mongodb').Db} db
+ * @param {import('mongodb').MongoClient} client
+ * @returns {Promise<void>}
+ */
+export const down = async (db, client) => {
+  const collection = db.collection(colName);
+  const result = await collection.dropIndex(indexOptions.name);
+  console.log("dropped index:", result);
+};
+

package/migrations/migrationScripts/20251028115614-annotations2-id-index.js

@@ -0,0 +1,27 @@
+// template migration file generated by migrate-mongo
+
+import {createIndex, removeIndex} from "../manageIndex.js";
+
+const
+  colName = "annotations2",
+  indexSpec = { "@id": 1 },
+  indexOptions = { name: "annotationIdIndex" };
+
+/**
+ * @param {import('mongodb').Db} db
+ * @param {import('mongodb').MongoClient} client
+ * @returns {Promise<void>}
+ */
+export const up = async (db, client) => {
+  await createIndex(db, colName, indexSpec, indexOptions);
+};
+
+/**
+ * @param {import('mongodb').Db} db
+ * @param {import('mongodb').MongoClient} client
+ * @returns {Promise<void>}
+ */
+export const down = async (db, client) => {
+  await removeIndex(db, colName, indexOptions);
+};
+

package/migrations/migrationTemplate.js

@@ -0,0 +1,25 @@
+// template migration file generated by migrate-mongo
+
+/**
+ * @param {import('mongodb').Db} db
+ * @param {import('mongodb').MongoClient} client
+ * @returns {Promise<void>}
+ */
+export const up = async (db, client) => {
+  // TODO write your migration here.
+  // See https://github.com/seppevs/migrate-mongo/#creating-a-new-migration-script
+  // Example:
+  // await db.collection('albums').updateOne({artist: 'The Beatles'}, {$set: {blacklisted: true}});
+};
+
+/**
+ * @param {import('mongodb').Db} db
+ * @param {import('mongodb').MongoClient} client
+ * @returns {Promise<void>}
+ */
+export const down = async (db, client) => {
+  // TODO write the statements to rollback your migration (if possible)
+  // Example:
+  // await db.collection('albums').updateOne({artist: 'The Beatles'}, {$set: {blacklisted: false}});
+};
+

package/package.json

@@ -0,0 +1,78 @@
+{
+  "name": "aiiinotate",
+  "version": "0.2.0",
+  "description": "a fast IIIF-compliant annotation server",
+  "main": "index.js",
+  "type": "module",
+  "directories": {
+    "doc": "docs"
+  },
+  "scripts": {
+    "setup": "bash run.sh -s",
+    "start": "bash run.sh -d",
+    "prod": "bash run.sh -p",
+    "test": "bash run.sh -t",
+    "cli": "dotenvx run -f ./src/config/.env -- node ./cli/index.js",
+    "lint": "npx eslint --fix",
+    "migrate-make": "npm run cli -- migrate make",
+    "migrate-apply": "npm run cli migrate apply",
+    "migrate-revert": "npm run cli migrate revert",
+    "migrate-revert-all": "npm run cli migrate revert-all"
+  },
+  "pre-commit": [
+    "lint"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Aikon-platform/aiiinotate.git"
+  },
+  "keywords": [
+    "annotation-server",
+    "iiif",
+    "annotation",
+    "international-image-interoperability-framework"
+  ],
+  "author": "Aikon",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/Aikon-platform/aiiinotate/issues"
+  },
+  "homepage": "https://github.com/Aikon-platform/aiiinotate#readme",
+  "imports": {
+    "#types": "./src/types.js",
+    "#cli/*.js": "./cli/*.js",
+    "#src/*.js": "./src/*.js",
+    "#migrations/*.js": "./migrations/*.js",
+    "#db/*.js": "./src/db/*.js",
+    "#schemas/*.js": "./src/schemas/*.js",
+    "#config/*.js": "./src/config/*.js",
+    "#data/*.js": "./src/data/*.js",
+    "#utils/*.js": "./src/data/utils/*.js",
+    "#fileServer/*.js": "./src/fileServer/*.js",
+    "#fileServer/*.json": "./src/fileServer/*.json",
+    "#manifests/*.js": "./src/data/manifests/*.js",
+    "#annotations/*.js": "./src/data/annotations/*.js"
+  },
+  "dependencies": {
+    "@dotenvx/dotenvx": "^1.49.0",
+    "@fastify/ajv-compiler": "^4.0.2",
+    "@fastify/cors": "^11.1.0",
+    "@fastify/mongodb": "^9.0.2",
+    "@fastify/swagger": "^9.5.2",
+    "commander": "^14.0.0",
+    "fastify": "^5.5.0",
+    "migrate-mongo": "^12.1.3",
+    "mongodb": "^6.18.0",
+    "swagger-markdown": "^3.0.0",
+    "uuid": "^11.1.0"
+  },
+  "devDependencies": {
+    "@eslint/css": "^0.10.0",
+    "@eslint/js": "^9.33.0",
+    "@eslint/json": "^0.13.1",
+    "@stylistic/eslint-plugin": "^5.2.3",
+    "eslint": "^9.33.0",
+    "globals": "^16.3.0",
+    "pre-commit": "^1.2.2"
+  }
+}

package/run.sh

@@ -0,0 +1,70 @@
+#!/bin/env bash
+
+source "./scripts/utils.sh";
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+ENV_FILE="$SCRIPT_DIR/src/config/.env";
+
+print_usage() {
+    cat<<EOF
+
+    USAGE bash run.sh [-s, -d, -p, -t, -c, -h]
+
+    (use from the scripts defined in 'package.json': 'npm start')
+
+    -s: setup the app
+    -t: test the app
+    -d: run the app in dev mode
+    -p: run the app in prod mode
+    -c: run the command line interface
+    -h: print help and exit
+
+EOF
+}
+
+start () {
+    local mode="$1"
+
+    if [ ! -f "$ENV_FILE" ];
+    then echo -e "\nERROR: .env file not found at '$ENV_FILE'. exiting..." && exit 1;
+    fi;
+
+    start_mongod
+
+    if [ "$mode" = "setup" ]; then
+        dotenvx run -f "$ENV_FILE" -- \
+        node "$SCRIPT_DIR/scripts/setup.js";
+    elif [ "$mode" = "dev" ]; then
+        dotenvx run -f "$ENV_FILE" -- \
+        node --watch "$SCRIPT_DIR/src/server.js";
+    elif [ "$mode" = "test" ]; then
+        dotenvx run -f "$ENV_FILE" -- \
+        node --test --test-isolation=none;
+    elif [ "$mode" = "cli" ] ; then
+        dotenvx run -f "$ENV_FILE" -- \
+        node "$SCRIPT_DIR/cli/index.js";
+    else echo -e "\nERROR: mode not implemented: '$mode'\n"; print_usage; exit 1;
+    fi;
+}
+
+while getopts 'hdptcs' mode_flag; do
+    case "${mode_flag}" in
+        s) MODE="setup"
+            break;;
+        d) MODE="dev"
+            break;;
+        p) MODE="prod"
+            break;;
+        t) MODE="test"
+            break;;
+        c) MODE="cli"
+            break;;
+        h) print_usage
+           exit 0;;
+        *) print_usage
+           exit 1;;
+    esac
+done
+
+start "$MODE";
+

package/scripts/_migrations.sh

@@ -0,0 +1,79 @@
+#!/bin/env bash
+
+# use through package.json scripts.
+
+# these sripts are used to manage all migrations in parallel
+# on both the main database and the test database.
+#NOTE migrate-init is not implemented.
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+source "$SCRIPT_DIR/utils.sh" || exit 1;
+
+MIGRATIONS_CONFIG_MAIN="$MIGRATIONS_DIR/migrate-mongo-config-main.js";
+MIGRATIONS_CONFIG_TEST="$MIGRATIONS_DIR/migrate-mongo-config-test.js";
+MIGRATIONS_CONFIGS=("$MIGRATIONS_CONFIG_MAIN" "$MIGRATIONS_CONFIG_TEST");
+
+##################################################
+# functions
+
+# create a migration
+# NOTE `migrate-mongo create` generates a timestamped file and, since this script is run twice
+# (once per config file), 2 migration files are created. so what we do is mimic migrate-make by
+# copying a blank template file to the `migrationScripts` directory.
+migrate_make() {
+    config_fp=$1;  # path to config file
+    migration_name="$2";  # name of migration to create
+
+    if [ -z "$migration_name" ];
+    then echo "ERROR. a migration name must be given."; exit 1;
+    fi;
+
+    cp "$MIGRATIONS_DIR/migrationTemplate.js" "$MIGRATIONS_DIR/migrationScripts/$(date +'%Y%m%d%H%M%S')-$migration_name.js";
+}
+
+# apply migrations
+migrate_apply() {
+    config_fp=$1;  # path to config file
+    dotenvx run -f "$ENV_FILE" -- npx migrate-mongo up -f "$config_fp";
+}
+
+# rvert the last migration
+migrate_revert() {
+    config_fp=$1;  # path to config file
+    dotenvx run -f "$ENV_FILE" -- npx migrate-mongo down -f "$config_fp";
+}
+
+# undo all migrations
+migrate_revert_all() {
+    config_fp=$1;  # path to config file
+
+    for _ in "$MIGRATIONS_DIR"/migrationScripts/*;
+    do dotenvx run -f "$ENV_FILE" -- npx migrate-mongo down -f "$config_fp";
+    done;
+}
+
+
+##################################################
+# cli
+
+OP=$1
+MIGRATION_NAME=$2  # only used by `migrate_make`
+
+case "$OP" in
+    make) FUNC=migrate_make;;
+    apply) FUNC=migrate_apply;;
+    revert) FUNC=migrate_revert;;
+    revertAll) FUNC=migrate_revert_all;;
+esac;
+
+if [ -z "$FUNC" ];
+then echo "ERROR: unrecognized keyword '$OP'. please use one of 'make', 'apply', 'revert', 'revertAll'. exiting..."; exit 1;
+fi;
+
+start_mongod
+
+for config_fp in "${MIGRATIONS_CONFIGS[@]}"; do
+    $FUNC "$config_fp" "$MIGRATION_NAME";
+done
+

package/scripts/_setup.js

@@ -0,0 +1,31 @@
+#!usr/bin/env node
+import path from "node:path";
+import { execSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+
+import { MongoClient } from "mongodb";
+
+const
+  // path to dirctory of curent file
+  dirScripts = path.dirname(fileURLToPath(import.meta.url)),
+  dirRoot = path.resolve(dirScripts, ".."),
+  dirMigrations = path.resolve(dirRoot, "migrations");
+
+const connString = process.env.MONGODB_CONNSTRING;
+
+(async () => {
+  let client;
+  try {
+    client = new MongoClient(connString);
+    await client.connect();
+    console.log("CONNECTED !")
+    console.log(dirScripts);
+    console.log(dirRoot);
+    console.log(dirMigrations);
+  } finally {
+    if ( client != null ) {
+      client.close()
+    }
+  }
+}
+)();

package/scripts/setup_mongodb.sh

@@ -0,0 +1,61 @@
+#!/bin/env bash
+
+# mongo install guide: https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-ubuntu/#std-label-install-mdb-community-ubuntu
+
+#NOTE only the mongodb installation is done here. for database creation, see `setup_mongodb_populate.sh`
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "$SCRIPT_DIR/utils.sh";
+
+install_mongodb_ubuntu () {
+
+    sudo apt-get install gnupg curl
+
+    curl -fsSL https://www.mongodb.org/static/pgp/server-8.0.asc | \
+    sudo gpg -o /usr/share/keyrings/mongodb-server-8.0.gpg \
+    --dearmor
+
+    # assert we have an 86 64 architecture
+    if [ "$(arch)" != "x86_64" ];
+    then echo "MongoDB only supports x86_64 architectures (yours is $(arch)). exiting..."; exit 1
+    fi;
+
+    # fetch the release name. Mongo only supports LTS versions, so if the user's Ubuntu version is not LTS, we get the name of the last LTS released before the user's version.
+    source "/etc/lsb-release"
+    if float_comparison "$DISTRIB_RELEASE >= 24.04";
+    then DISTRIB="noble";
+    elif float_comparison  "$DISTRIB_RELEASE >= 22.04";
+    then DISTRIB="jammy";
+    elif float_comparison "$DISTRIB_RELEASE >= 20.04";
+    then DISTRIB="focal";
+    else echo "Your Ubuntu version ($DISTRIB_RELEASE) is not supported by MongoDB 8.0"; exit 1;
+    fi;
+
+    # create list file
+    echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-8.0.gpg ] https://repo.mongodb.org/apt/ubuntu $DISTRIB/mongodb-org/8.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-8.0.list
+
+    sudo apt-get update
+
+    sudo apt-get install -y mongodb-org
+}
+
+install_mongodb_mac () {
+    xcode-select --install;
+    brew tap mongodb/brew;
+    brew update;
+    brew install mongodb-community@8.0;
+}
+
+if ! command -v mongod ; then
+    echo_title "INSTALL MONGODB"
+
+    if [ "$OS" = "Linux" ]; then
+        install_mongodb_ubuntu;
+        sudo systemctl start mongod;
+    elif [ "$OS" = "Mac" ]; then
+        install_mongodb_mac;
+        brew services start mongodb-community@8.0;
+    else echo "Unsupported OS: $OS"; exit 1;
+    fi;
+
+fi;

package/scripts/setup_mongodb_migrate.sh

@@ -0,0 +1,17 @@
+#!/bin/env bash
+
+#NOTE : this setup creates and populates our db WITHOUT users or authentication.
+# it is possible to add users and auth to a mongodb instance, but
+#   - it is a bit convoluted in itself
+#   - it is difficult to automate: we would need to
+#       - create a root user + a user for the app
+#       - create this app user by generating mongosh scripts from the user's .env files (so use Python to write custom JS)
+#       - update the mongodb conf file so that the systemd mongodb service uses auth login, which would need to parse YAML, so to use python
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+ROOT_DIR="$SCRIPT_DIR/../"
+
+cd "$ROOT_DIR"
+
+npm run migrate-apply
+

package/scripts/setup_node.sh

@@ -0,0 +1,15 @@
+#!/bin/env bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "$SCRIPT_DIR/utils.sh";
+
+if ! command -v npm &> /dev/null; then
+    echo_title "INSTALL NVM & NODE";
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash
+    nvm install node
+    npm install -g webpack webpack-cli
+fi
+
+echo_title "SETUP FASTIFY APP";
+cd "$ROOT_DIR";
+npm i;

package/scripts/utils.sh

@@ -0,0 +1,192 @@
+#!/bin/env bash
+
+# directory of the current script
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+# root of the app
+ROOT_DIR=$( dirname "$SCRIPT_DIR" )
+# src/ directory
+SRC_DIR="$ROOT_DIR/src"
+# migrations/ dir
+MIGRATIONS_DIR="$ROOT_DIR/migrations";
+# .env file
+ENV_FILE="$SRC_DIR/config/.env"
+
+
+color_echo() {
+    Color_Off="\033[0m"
+    Red="\033[1;91m"        # Red
+    Green="\033[1;92m"      # Green
+    Yellow="\033[1;93m"     # Yellow
+    Blue="\033[1;94m"       # Blue
+    Purple="\033[1;95m"     # Purple
+    Cyan="\033[1;96m"       # Cyan
+
+    case "$1" in
+        "green") echo -e "$Green$2$Color_Off";;
+        "red") echo -e "$Red$2$Color_Off";;
+        "blue") echo -e "$Blue$2$Color_Off";;
+        "yellow") echo -e "$Yellow$2$Color_Off";;
+        "purple") echo -e "$Purple$2$Color_Off";;
+        "cyan") echo -e "$Cyan$2$Color_Off";;
+        *) echo "$2";;
+    esac
+}
+
+echo_title(){
+    sep_line="========================================"
+    len_title=${#1}
+
+    if [ "$len_title" -gt 40 ]; then
+        sep_line=$(printf "%0.s=" $(seq 1 $len_title))
+        title="$1"
+    else
+        diff=$((38 - len_title))
+        half_diff=$((diff / 2))
+        sep=$(printf "%0.s=" $(seq 1 $half_diff))
+
+        if [ $((diff % 2)) -ne 0 ]; then
+            title="$sep $1 $sep="
+        else
+            title="$sep $1 $sep"
+        fi
+    fi
+
+    color_echo purple "\n\n$sep_line\n$title\n$sep_line"
+}
+
+get_os() {
+    unameOut="$(uname -s)"
+    case "${unameOut}" in
+        Linux*)     os=Linux;;
+        Darwin*)    os=Mac;;
+        CYGWIN*)    os=Cygwin;;
+        MINGW*)     os=MinGw;;
+        MSYS_NT*)   os=Git;;
+        *)          os="UNKNOWN:${unameOut}"
+    esac
+    echo "${os}"
+}
+
+export OS
+OS=$(get_os)
+
+# gets a password and validates it by running a dummy cmd.
+# parent process must call the function with `get_password || exit` to exit the script if `SUDO_PSW` is invalid
+get_password() {
+    if [ -z "$SUDO_PSW" ]; then
+        read -s -p "Enter your sudo password: " SUDO_PSW
+        echo
+        echo "$SUDO_PSW" | sudo -S whoami > /dev/null 2>&1
+        if [ $? -ne 0 ]; then
+            echo "Invalid sudo password. Exiting..."
+            return 1
+        fi
+        return 0
+    fi
+}
+
+# the sed at the end removes trailing non-alphanumeric chars.
+generate_random_string() {
+    echo "$(openssl rand -base64 32 | tr -d '/\n' | sed -r -e "s/[^a-zA-Z0-9]+$//")"
+}
+
+get_env_value() {
+    param=$1
+    env_file=$2
+    value=$(awk -F= -v param="$param" '/^[^#]/ && $1 == param {gsub(/"/, "", $2); print $2}' "$env_file")
+    echo "$value"
+}
+
+get_env_desc() {
+    current_line="$1"
+    prev_line="$2"
+    desc=""
+    if [[ $prev_line =~ ^# ]]; then
+        desc=$(echo "$prev_line" | sed 's/^#\s*//')
+    fi
+    echo "$desc"
+}
+
+ask() {
+    options=("yes" "no")
+    color_echo blue "$1"
+    answer=$(printf "%s\n" "${options[@]}" | fzy)
+    echo ""
+    if [ "$answer" = "no" ]; then
+        exit 1
+    fi
+}
+
+# float arithmetic comparison is not supported by bash and we need to use `bc`
+# usage: if float_comparison "a >= b"; then... ; fi
+float_comparison () {
+    expr="$1"
+    (( $(echo "$expr" |bc -l) ));
+}
+
+# sed replacements in place
+# `sed -i` can't be used in the same way with Linux and Mac: it's `sed -i` on Linux, `sed -i ""` on Mac
+sed_repl_inplace() {
+    sed_expr="$1"
+    file="$2"
+
+    if [ "$OS" = "Linux" ]; then
+        sed -i -e "$sed_expr" "$file"
+    else
+        sed -i "" -e "$sed_expr" "$file"
+    fi
+}
+
+# sudo does not inherit from bash functions so this is a copy of "sed_repl_inplace" with sudo privileges (see: https://stackoverflow.com/a/9448969)
+sudo_sed_repl_inplace() {
+    sed_expr="$1"
+    file="$2"
+
+    if [ "$OS" = "Linux" ]; then
+        [ -n "$SUDO_PSW" ] && echo "$SUDO_PSW" | sudo -S sed -i -e "$sed_expr" "$file" || sudo sed -i -e "$sed_expr" "$file"
+    else
+        [ -n "$SUDO_PSW" ] && echo "$SUDO_PSW" | sudo -S sed -i "" -e "$sed_expr" "$file" || sudo sed -i "" -e "$sed_expr" "$file"
+    fi
+}
+
+# ed replacements to a new file
+sed_repl_newfile() {
+    sed_expr="$1"
+    infile="$2"
+    outfile="$3"
+
+    sed "$sed_expr" "$infile" | tee "$outfile" > /dev/null
+}
+
+sudo_sed_repl_newfile() {
+    sed_expr="$1"
+    infile="$2"
+    outfile="$3"
+
+    sudo sed "$sed_expr" "$infile" | sudo tee "$outfile" > /dev/null
+}
+
+run_script() {
+    local script_name="$1"
+    local description="$2"
+    local SCRIPT_DIR=${3:-${SCRIPT_DIR}}
+    options=("yes" "no")
+
+    color_echo blue "Do you want to run $description?"
+    answer=$(printf "%s\n" "${options[@]}" | fzy)
+    echo ""
+    if [ "$answer" = "yes" ]; then
+        bash "$SCRIPT_DIR/$script_name" \
+        && color_echo green "$description completed successfully" \
+        || color_echo red "$description failed with exit code. Continuing..."
+    else
+        color_echo cyan "Skipping $description"
+    fi
+    echo ""
+}
+
+start_mongod() {
+    if ! systemctl is-active --quiet mongod;
+    then sudo systemctl start mongod;
+    fi;
+}