aiiinotate 0.2.9 → 0.2.11

package/cli/migrate.js

@@ -61,6 +61,7 @@ function migrateMake(migrationName) {
 
 /** apply all pending migrations */
 function migrateApply() {
+  // NOTE: if we don't use a specific version of migrate-mongo, we get docker errors
   migrationConfigs.map((mc) => execSync(`npx -- migrate-mongo@^12.1.3 up -f ${mc}`));
 }
 

package/cli/utils/mongoClient.js

@@ -6,9 +6,7 @@ import { MongoClient } from "mongodb";
  */
 function loadMongoClient() {
   try {
-    const client = new MongoClient(
-      process.env.MONGODB_CONNSTRING,
-    );
+    const client = new MongoClient(process.env.MONGODB_CONNSTRING);
     client.db(process.env.MONGODB_DB);
     return client;
   } catch (err) {

package/docker/Dockerfile

@@ -1,8 +1,8 @@
 # syntax=docker/dockerfile:1
-FROM node:23.11 AS aiiinotate
+FROM node:23.11
 
 # aiiinotate port
-ARG PORT=4444
+ARG PORT
 ENV PORT=${PORT}
 
 # set up environment
@@ -11,15 +11,17 @@ SHELL ["/bin/bash", "-c"]
 
 # root of the app in the docker container
 WORKDIR /aiiinotate
-# copy the .env in the docker container
-COPY ./config/.env /aiiinotate/.env
-# install the app as an NPM library
+# copy the docker .env in the docker container
+COPY ./docker/.env /aiiinotate/.env
+# for debug: install `iproute2` which provides CLI `ss` to monitor active ports
+RUN apt update && apt install iproute2 -y
+# install the app as an NPM library. we do a global install as it saves us from issues with `$PATH`.
 RUN npm i -g aiiinotate
 
-RUN npm i -g migrate-mongo@^12.1.3
-# migrate the database
-RUN  aiiinotate --env=/aiiinotate/.env -- migrate apply
 # expose the used port
 EXPOSE $PORT
-# start the app
-CMD aiiinotate --env=/aiiinotate/.env -- serve prod
+
+# run the migrations and start the app.
+# NOTE migrations must be done in CMD: they need the mongo service to be running.
+CMD aiiinotate --env=/aiiinotate/.env -- migrate apply && \
+    aiiinotate --env=/aiiinotate/.env -- serve prod;

package/docker/README.md

@@ -0,0 +1,62 @@
+# Docker usage
+
+---
+
+## Usage
+
+Note that all `docker` commands must be run from the `docker/` directory !
+
+1. Clone the repo and `cd` in it
+
+```bash
+git clone git@github.com:Aikon-platform/aiiinotate.git
+cd aiiinotate
+```
+
+2. Create a `.env` file at `config/.env` (from the root of the project)
+
+```bash
+cp ./config/.env.template ./config/.env
+vim ./config/.env  # do your edits
+```
+
+3. Build the containers
+
+```bash
+cd docker
+bash docker.sh build
+```
+
+4. Start the containers
+
+```bash
+bash docker.sh start
+```
+
+---
+
+## Troubleshooting and useful commands
+
+Access `mongosh` within the Mongo container:
+
+```bash
+sudo docker exec -it docker-mongo-1 mongosh
+```
+
+Check running ports in the Web container:
+
+```bash
+sudo docker exec -it docker-web-1 ss -tnl
+```
+
+View globally installed NPM packages in the Web container:
+
+```bash
+sudo docker exec -it docker-web-1 npm list -g --depth=0
+```
+
+CURL the Web container
+
+```bash
+sudo docker exec -it docker-web-1 curl http://0.0.0.0:4444   # change 4444 by your $APP_PORT
+```

package/docker/docker-compose.yaml

@@ -1,8 +1,16 @@
+x-proxy-settings: &proxy-settings
+  HTTP_PROXY: ${HTTP_PROXY:-}
+  HTTPS_PROXY: ${HTTPS_PROXY:-}
+  NO_PROXY: "localhost,127.0.0.1,mongo,web,.aiiinotate_network"
+
 services:
   mongo:
     image: mongo:8
     restart: always
-    # environment:
+    networks:
+      - aiiinotate_network
+    environment:
+      <<: *proxy-settings
     #   MONGO_INITDB_ROOT_USERNAME: root
     #   MONGO_INITDB_ROOT_PASSWORD: example
 
@@ -15,7 +23,15 @@ services:
     env_file:
       - ../config/.env
     ports:
-      - ${APP_PORT}:${APP_PORT}
+      - "${APP_PORT}:${APP_PORT}"
     depends_on:
       - mongo
+    networks:
+      - aiiinotate_network
+    environment:
+      <<: *proxy-settings
     restart: always
+
+networks:
+  aiiinotate_network:
+    driver: bridge

package/docker/docker.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+set -e;
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+# basic / dev config
+ENV_CONFIG="$SCRIPT_DIR/../config/.env"
+# docker specific config, created by `env_to_docker`
+ENV_DOCKER="$SCRIPT_DIR/.env"
+
+get_os() {
+    unameOut="$(uname -s)"
+    case "${unameOut}" in
+        Linux*)     os=Linux;;
+        Darwin*)    os=Mac;;
+        CYGWIN*)    os=Cygwin;;
+        MINGW*)     os=MinGw;;
+        MSYS_NT*)   os=Git;;
+        *)          os="UNKNOWN:${unameOut}"
+    esac
+    echo "${os}"
+}
+OS=$(get_os)
+
+# mac+linux compatible file replacements
+sed_repl () {
+    sed_expr="$1"
+    file="$2"
+    if [ "$OS" = "Linux" ];
+    then sed -i -e "$sed_expr" "$file";
+    else sed -i "" -e "$sed_expr" "$file";
+    fi
+}
+
+# copy config/.env file to docker/.env and adapt the file to work with docker.
+env_to_docker() {
+    # check the env file is found
+    if [ ! -f "$ENV_CONFIG" ];
+    then
+        echo ".env file not found at at '$ENV_CONFIG'. exiting...";
+        return 1;  # will exit when used with `set -e`
+    fi;
+
+    # NOTE that the MongoDB host in Docker MUST BE the name of the Mongo docker service (defined in docker-compose)
+    cp "$ENV_CONFIG" "$ENV_DOCKER";
+    sed_repl s~^MONGODB_HOST=.*$~MONGODB_HOST=mongo~ "$ENV_DOCKER";
+    sed_repl s~^APP_HOST=.*~APP_HOST=0.0.0.0~ "$ENV_DOCKER";
+}
+env_to_docker;
+
+build_containers () {
+    sudo docker compose --env-file "$ENV_DOCKER" build --no-cache;
+}
+
+start_containers() {
+    sudo docker compose --env-file "$ENV_DOCKER" up --force-recreate;
+}
+
+stop_containers() {
+    sudo docker compose --env-file "$ENV_DOCKER" down;
+}
+
+case "$1" in
+    start)
+        start_containers
+        ;;
+    stop)
+        stop_containers
+        ;;
+    build)
+        stop_containers
+        build_containers
+        start_containers
+        ;;
+    *)
+        echo "Usage: $0 {build|start|stop}"
+        exit 1
+        ;;
+esac;
+

package/docs/notes_quirks_troubleshooting.md

@@ -0,0 +1,143 @@
+# Notes, quirks and troubleshooting
+
+---
+
+## Notes
+
+### Uniqueness
+
+As of writing (09.10.25), there are no uniqueness constraints on annotations. There is only a uniqueness constraint on collection `manifest2` on field `manifest2.@id` (the ID of a manifest).
+
+Ideally, we would want to avoid having duplicate annotations in the database. This is more complicated in practice: at least for `annotions2`, an annotation's `@id` field is re-generated and a random part (an UUID) is generated at insert time. This means that, when trying to store the same annotation (with the same `@id`), the `@id` is changed, and so a different value is inserted.
+
+This means that we can't have a uniqueness constraint on `@id` or `id` fields of annotations. Another option would be to have a uniqueness constraint on annotation targets (no 2 annotations can annotate the same region), but this behaviour seems brittle in practice, so it's not yet implemented.
+
+### Concurrency
+
+For clients, concurrency/parrallelization (i.e., with JS `Promise.all()`) on insert/update should be avoided because it can cause a data race: several processes can attempt to write the same thing in parrallel.
+
+For example, when inserting annotations, the manifests related to each annotation are inserted in parrallel. Since this is a side effect, 2 processes may unknowingly try to insert the same manifest in the database, which causes a uniqueness constraint to fail. This error can be hard to debug, so it's best to avoid concurrency at write time.
+
+---
+
+## Dev quirks
+
+Sometimes, node/fastify can be weird. When scratching your head at dev errors, look here:)
+
+### Error swallowing at app build
+
+Errors that happen when a plugin is being registered will cause the app's startup to fail, without necessarily throwing an error.
+
+This is especially true for test cases that build the fastify app:
+
+```
+test
+=> build fastify app
+=> build step fails silently
+=> test fails
+```
+
+#### Troubleshooting
+
+- normal behaviour: when a runtime error happens, the failing test will log the error on the console
+- what this error looks like:
+    - tests fail **very quickly**,
+    - without throwing an error, seemingly without even running the test suite
+    - the proces doesn't exit, although all tests have failed
+- when `npm run test` fails like this, run `npm run start`. See if normal startup throws an error
+    - NOTE: normal startup not throwing does NOT mean that the build step necessarily worked
+
+#### Possible help/solutions
+
+- find a way to stisfyingly use `try...catch` at plugin registration.
+- look into these issues: [2694](https://github.com/fastify/fastify/issues/2694)
+    - in particular, it may be an issue specific to async plugins ?
+
+### Route response schema definition
+
+For some reason, route schema definition is much less flexible for responses than for queries.
+
+#### The problem
+
+**Query schemas**: In queries (`schema.params` or `schema.querystring`), fastify is very permissive. You can use unresolved schemas (with `$ref`), save entire schemas as JsonSchemas ...
+
+**Response schemas**: Response schemas have more constraints.
+- **response schemas are defined at route level** and cannot be stored as full JsonSchemas:
+    ```js
+    // this will be an invalid response schema: it is trying to store a complete response schema as a JsonSchema
+    fastify.addSchema({
+        $id: makeSchemaUri("routeResponsePost"),
+        properties: {
+            200: { ... },
+            500: { ... },
+        }
+    })
+
+    // this is a fixed version: an object containing schemas, not a full JsonSchema, to be used inside a Route definition
+    schema: {
+        response: {
+            200: { ... },
+            500: { ... }
+        }
+    }
+    // if you want to reuse a schema, store it as a JS object and import it.
+    ```
+- **unresolved response schemas (`$ref`) are forbidden**. You cannot use `$ref`. In the app, use `fastify.schemasResolver` to resolve the schema to a plain `JsonSchema` without `$ref`.
+
+#### The fix
+
+In short, here's **how to use shared schemas in responses**:
+1. Define payload schemas for different response cases:
+    ```js
+    // in case of a POST success
+    fastify.addSchema({
+        $id: makeSchemaUri("routeResponseInsert"),
+        type: "object",
+        // ...properties
+    });
+
+    // in case of a POST error
+    fastify.addSchema({
+        $id: makeSchemaUri("routeResponseError"),
+        type: "object",
+        // ...properties
+    });
+    ```
+2. Resolve schemas and use in responses
+    ```js
+    const routeResponseInsert = fastify.getSchema("...");
+    const routeResponseError = fastify.getSchema("...");
+    fastify.post(
+        "/annotations/:iiifPresentationVersion/create",
+        {
+            schema: {
+                body: routeAnnotations2Or3Schema,
+                response: {
+                    200: routeResponseInsert,
+                    500: routeResponseError
+               }
+            }
+        },
+        async (req, rep) => {}
+    )
+    ```
+
+### `migrate revert-all && migrate apply`
+
+#### The problem
+
+Sometimes, you want to remove all migrations and then rerun them. This will cause an error like:
+
+```bash
+{
+    ...,
+    errmsg: 'namespace aiiinotate.manifests2 already exists, but with different options: { uuid: UUID("65ca87c4-ddce-4c2d-a6f9-08ff6b70c0a7"), validationLevel: "strict", validationAction: "error" }',
+    code: 48,
+    codeName: 'NamespaceExists',
+    ...
+}
+```
+
+#### The fix
+
+I figure it's an error due to cache or something like that. Either wait or manually delete the collection or the entire database using `mongosh`.

package/docs/progress.md

@@ -36,124 +36,3 @@ Routes are only implemented with IIIF Presentation API 2.x, not with the 3.0 ver
 - fetching and inserting manifests related to an annotation when using inserting annotations.
 
 ---
-
-## Notes
-
-### Uniqueness
-
-As of writing (09.10.25), there are no uniqueness constraints on annotations. There is only a uniqueness constraint on collection `manifest2` on field `manifest2.@id` (the ID of a manifest).
-
-Ideally, we would want to avoid having duplicate annotations in the database. This is more complicated in practice: at least for `annotions2`, an annotation's `@id` field is re-generated and a random part (an UUID) is generated at insert time. This means that, when trying to store the same annotation (with the same `@id`), the `@id` is changed, and so a different value is inserted.
-
-This means that we can't have a uniqueness constraint on `@id` or `id` fields of annotations. Another option would be to have a uniqueness constraint on annotation targets (no 2 annotations can annotate the same region), but this behaviour seems brittle in practice, so it's not yet implemented.
-
-### Concurrency
-
-For clients, concurrency/parrallelization (i.e., with JS `Promise.all()`) on insert/update should be avoided because it can cause a data race: several processes can attempt to write the same thing in parrallel.
-
-For example, when inserting annotations, the manifests related to each annotation are inserted in parrallel. Since this is a side effect, 2 processes may unknowingly try to insert the same manifest in the database, which causes a uniqueness constraint to fail. This error can be hard to debug, so it's best to avoid concurrency at write time.
-
----
-
-## Dev quirks
-
-Sometimes, node/fastify can be weird. When scratching your head at dev errors, look here:)
-
-### Error swallowing at app build
-
-Errors that happen when a plugin is being registered will cause the app's startup to fail, without necessarily throwing an error.
-
-This is especially true for test cases that build the fastify app:
-
-```
-test
-=> build fastify app
-=> build step fails silently
-=> test fails
-```
-
-#### Troubleshooting
-
-- normal behaviour: when a runtime error happens, the failing test will log the error on the console
-- what this error looks like:
-    - tests fail **very quickly**,
-    - without throwing an error, seemingly without even running the test suite
-    - the proces doesn't exit, although all tests have failed
-- when `npm run test` fails like this, run `npm run start`. See if normal startup throws an error
-    - NOTE: normal startup not throwing does NOT mean that the build step necessarily worked
-
-#### Possible help/solutions
-
-- find a way to stisfyingly use `try...catch` at plugin registration.
-- look into these issues: [2694](https://github.com/fastify/fastify/issues/2694)
-    - in particular, it may be an issue specific to async plugins ?
-
-### Route response schema definition
-
-For some reason, route schema definition is much less flexible for responses than for queries.
-
-#### The problem
-
-**Query schemas**: In queries (`schema.params` or `schema.querystring`), fastify is very permissive. You can use unresolved schemas (with `$ref`), save entire schemas as JsonSchemas ...
-
-**Response schemas**: Response schemas have more constraints.
-- **response schemas are defined at route level** and cannot be stored as full JsonSchemas:
-    ```js
-    // this will be an invalid response schema: it is trying to store a complete response schema as a JsonSchema
-    fastify.addSchema({
-        $id: makeSchemaUri("routeResponsePost"),
-        properties: {
-            200: { ... },
-            500: { ... },
-        }
-    })
-
-    // this is a fixed version: an object containing schemas, not a full JsonSchema, to be used inside a Route definition
-    schema: {
-        response: {
-            200: { ... },
-            500: { ... }
-        }
-    }
-    // if you want to reuse a schema, store it as a JS object and import it.
-    ```
-- **unresolved response schemas (`$ref`) are forbidden**. You cannot use `$ref`. In the app, use `fastify.schemasResolver` to resolve the schema to a plain `JsonSchema` without `$ref`.
-
-#### The fix
-
-In short, here's **how to use shared schemas in responses**:
-1. Define payload schemas for different response cases:
-    ```js
-    // in case of a POST success
-    fastify.addSchema({
-        $id: makeSchemaUri("routeResponseInsert"),
-        type: "object",
-        // ...properties
-    });
-
-    // in case of a POST error
-    fastify.addSchema({
-        $id: makeSchemaUri("routeResponseError"),
-        type: "object",
-        // ...properties
-    });
-    ```
-2.
-2. Resolve schemas and use in responses
-    ```js
-    const routeResponseInsert = fastify.getSchema("...");
-    const routeResponseError = fastify.getSchema("...");
-    fastify.post(
-        "/annotations/:iiifPresentationVersion/create",
-        {
-            schema: {
-                body: routeAnnotations2Or3Schema,
-                response: {
-                    200: routeResponseInsert,
-                    500: routeResponseError
-               }
-            }
-        },
-        async (req, rep) => {}
-    )
-    ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "aiiinotate",
-  "version": "0.2.9",
+  "version": "0.2.11",
   "description": "a fast IIIF-compliant annotation server",
   "main": "./cli/index.js",
   "type": "module",
@@ -12,9 +12,9 @@
   },
   "scripts": {
     "cli": "node ./cli/index.js --env=./config/.env",
-    "setup": "npm run cli migrate apply",
-    "start": "npm run cli serve dev",
-    "test": "dotenvx run -f ./config/.env -- node --test --test-isolation=none",
+    "setup": "sudo systemctl start mongod && npm run cli migrate apply",
+    "start": "sudo systemctl start mongod && npm run cli serve dev",
+    "test": "sudo systemctl start mongod && dotenvx run -f ./config/.env -- node --test --test-isolation=none",
     "lint": "npx eslint --fix",
     "migrate": "npm run cli -- migrate"
   },

package/src/data/annotations/routes.test.js

@@ -41,7 +41,7 @@ test("test annotation Routes", async (t) => {
 
   // NOTE: it is necessary to run the app because internally there are fetches to external data.
   try {
-    await fastify.listen({ port: process.env.APP_PORT });
+    await fastify.listen({ port: process.env.APP_PORT, host: process.env.APP_HOST });
   } catch (err) {
     console.log("FASTIFY ERROR", err);
     throw err;

package/src/data/manifests/manifests2.test.js

@@ -27,7 +27,7 @@ test("test Manifests2 module", async (t) => {
 
   // NOTE: it is necessary to run the app because internally there are fetches to external data.
   try {
-    await fastify.listen({ port: process.env.APP_PORT });
+    await fastify.listen({ port: process.env.APP_PORT, host: process.env.APP_HOST });
   } catch (err) {
     console.log("FASTIFY ERROR", err);
     throw err;

package/src/data/manifests/routes.test.js

@@ -30,7 +30,7 @@ test("test manifests Routes", async (t) => {
 
   // NOTE: it is necessary to run the app because internally there are fetches to external data.
   try {
-    await fastify.listen({ port: process.env.APP_PORT });
+    await fastify.listen({ port: process.env.APP_PORT, host: process.env.APP_HOST });
   } catch (err) {
     console.log("FASTIFY ERROR", err);
     throw err;

package/src/data/routes.test.js

@@ -26,7 +26,7 @@ test("test common routes", async (t) => {
 
   // NOTE: it is necessary to run the app because internally there are fetches to external data.
   try {
-    await fastify.listen({ port: process.env.APP_PORT });
+    await fastify.listen({ port: process.env.APP_PORT, host: process.env.APP_HOST });
   } catch (err) {
     console.log("FASTIFY ERROR", err);
     throw err;

package/src/server.js

@@ -3,6 +3,7 @@
  */
 
 import build from "#src/app.js";
+import { visibleLog } from "#utils/utils.js";
 
 /**
  * @param {import("#types").RerveModeType} serveMode
@@ -14,8 +15,10 @@ async function server (serveMode) {
 
   const fastify = await build(serveMode);
 
+  visibleLog([process.env.APP_HOST, process.env.APP_PORT]);
+
   try {
-    fastify.listen({ port: process.env.APP_PORT });
+    fastify.listen({ port: process.env.APP_PORT, host: process.env.APP_HOST });
   } catch(err) {
     fastify.log.error(err);
     process.exit(1);