@paulhectork/aiiinotate 0.12.0

package/cli/utils/fastifyClient.js

@@ -0,0 +1,86 @@
+import build from "#src/app.js";
+
+/** @typedef {import("#types").FastifyInstanceType} FastifyInstanceType */
+/** @typedef {import("#types").FastifyReplyType} FastifyReplyType */
+/** @typedef {import("#types").InsertResponseType} InsertResponseType */
+
+/**
+ * a client to interact with the fastify app.
+ *
+ * @example
+ *  import FastifyClient from "#cli/utils/fastifyClient.js";
+ *  const fastify = new FastifyClient();
+ *  // load the fastify instance asynchronously
+ *  await fastify.build();
+ *  // run operations
+ *  await fastify.importAnnotationList(...);
+ */
+class FastifyClient {
+  constructor () { }
+
+  // NOTE: instanciating this.fastify is asynchronous and a constructor must be synchronous,
+  // so we use a builder to set this.fastify, following this pattern: https://stackoverflow.com/a/43433773
+  async build() {
+    /** @type {FastifyInstanceType} */
+    this.fastify = await build("default");
+    // TODO find way to actually completely disable logging because it makes the CLI UI uglyyy !
+    // log only fastify errors
+    this.fastify.log.level = "error";
+  }
+
+  async stop() {
+    await this.fastify.close();
+    this.fastify.log.info("fastify instance successfully closed.");
+  }
+
+  /**
+   * @param {string} route
+   * @returns {Promise<FastifyReplyType>}
+   */
+  injectGet(route) {
+    return this.fastify.inject({
+      method: "GET",
+      url: route,
+    })
+  }
+
+  /**
+   * @param {string} route
+   * @param {object} payload
+   * @returns {Promise<FastifyReplyType>}
+   */
+  injectPost(route, payload) {
+    return this.fastify.inject({
+      method: "POST",
+      url: route,
+      payload: payload
+    })
+  }
+
+  async importAnnotationPage(annotationPage) {
+    // TODO
+  }
+
+  /**
+   * function to interact with all annotation createMany + manifest create routes.
+   * @type {(iiifVersion: 2|3, datatype: "manifest"|"annotation") => (data: object) => [Number, Promise<object>] }
+   */
+  importData(iiifVersion, datatype) {
+    if (![ "2","3" ].includes(`${iiifVersion}`)) {
+      throw new Error(`fastifyClient.importData: "iiifVersion" must by 2 or 3, got "${iiifVersion}"`);
+    }
+    if (![ "manifest","annotation" ].includes(datatype)) {
+      throw new Error(`fastifyClient.importData: "datatype" must by "manifest" or "annotation", got "${datatype}"`);
+    }
+
+    const routeSuffix = datatype==="manifest" ? "create" : "createMany";
+    const route = `/${datatype}s/${iiifVersion}/${routeSuffix}`;
+
+    return async (data) => {
+      const r = await this.injectPost(route, data);
+      return [ r.statusCode, r.json() ];
+    }
+  }
+}
+
+export default FastifyClient;

package/cli/utils/io.js

@@ -0,0 +1,225 @@
+import path from "path";
+import fs from "fs";
+
+import ProgressBar from "#cli/utils/progressbar.js";
+
+/** @typedef {import("fs").WriteStream} WriteStreamType */
+
+const cwd = process.cwd();  // directory the script is run from
+const getCwd = () => process.cwd();
+
+/**
+ * convert a filepath to absolute if it is relative.
+ * @param {string} f
+ * @returns {string}
+ */
+const toAbsPath = (f) => path.isAbsolute(f) ? f : path.join(cwd, f);
+
+/**
+ * @returns {[string, boolean]} - [<absolute file path>, <success?>]
+ */
+const fileOk = (f) => {
+  f = toAbsPath(f);
+  try {
+    fs.accessSync(f, fs.constants.R_OK);  // will throw if there's a problem opening.
+    return [ f, fs.lstatSync(f).isFile() ];
+  } catch (e) {
+    return [ f, false ]
+  }
+}
+
+/** @returns {[PathLike, boolean]} */
+const dirOk = (d) => {
+  d = toAbsPath(d);
+  try {
+    fs.accessSync(d, fs.constants.R_OK);  // will throw if there's an error
+    return [ d, fs.lstatSync(d).isDirectory() ]
+  } catch (e) {
+    return [ d, false ]
+  }
+}
+
+/**
+ * @param {string} f
+ * @return {string?}
+ */
+const fileRead = (f) => {
+  f = toAbsPath(f);
+  try {
+    return fs.readFileSync(f, { encoding: "utf8" })
+  } catch (err) {
+    console.error(`io.fileRead: could not read file: ${f}`);
+  }
+}
+
+/**
+ * take an input array of filepaths. convert the paths to absolute, and check that the files exist
+ * the cli exits if any of the files don't exist
+ * @param {string[]} fileArr
+ * @returns { string[] } array of absolute filepaths
+ */
+function fileArrayValidate (fileArr) {
+  // array of [<absolute filepath>, <success opening file?>]
+  fileArr = fileArr.map(fileOk);
+  const successCount = fileArr.filter(x => x[1]).length;
+  if (successCount!==fileArr.length) {
+    console.error(
+      `io.fileArrayValidate: ${fileArr.length - successCount} files could not be accessed. exiting...`,
+      fileArr.filter(x => !x[1]).map(x => x[0])
+    );
+    process.exit(1);
+  }
+  return fileArr.map(x => x[0]);
+}
+
+/**
+ * `file` is a path to a file containing paths to other files (1 file per line).
+ * parse the file and return an array of absolute paths to files.
+ * @param {str} file
+ * @returns {Promise<string[]>}
+ */
+async function parseImportInputFile(file) {
+  // ensure input file exists
+  const [ fileAbs, ok ] = fileOk(file);
+  if (!ok) {
+    console.error(`could not read import file: ${file}. exiting...`);
+    process.exit(1);
+  }
+
+  // read `file` split it by lines, remove empty lines
+  const fileArr =
+    fileRead(fileAbs)
+      .split("\n")
+      .filter(l => !l.match(/^\s*$/g));
+  return [ ...new Set(fileArrayValidate(fileArr)) ];
+}
+
+const fileWrite = (f, data) => {
+  try {
+    fs.writeFileSync(f, data, "utf-8");
+  } catch (e) {
+    throw new Error(`Error writing to file: ${f} because of error: ${e}`);
+  }
+}
+
+/**
+ * NOTE: this will NOT work on collections and huge objects. use streamWriteJson to write from Mongo Cursors.
+ * @param {string|import("fs").PathLike} f
+ * @param {Array|object} data
+ * @returns {void}
+ */
+const fileWriteJson = (f, data) => {
+  data = JSON.stringify(data)
+  fileWrite(f, data);
+}
+
+/**
+ * wrapper for writer.write(data) that respects writer backpressure
+ *
+ * we wrap the .write() in a Promise that checks on the rturn of `writer.write`
+ * and drains the stream if necessary: if `writer.write()` returns
+ * false, the write buffer is full  and must be drained by the OS
+ * before continuing.
+ *
+ * this avoids:
+ * - writes that are silently dropped or reordered
+ * - corrupt/truncated JSON with no error thrown
+ *
+ * @type {(writer: WriteStreamType) => (data: string) => Promise<void> }
+ */
+const writeOrWait = (writer) =>
+  (data) =>
+    new Promise((resolve, reject) => {
+      const ok = writer.write(data, (error) => {
+        if (error) return reject(error);
+      });
+      if (ok) {
+        resolve();  // buffer has room, keep going
+      } else {
+        writer.once("drain", resolve);  // buffer full, wait for drain
+      }
+    });
+
+/**
+ * WriteStream.write is actually asyncrhonous => for the last line,
+ * we need to use `writer.end` to ensure everything has been written to file.
+ *
+ * @returns {Promise<void>}
+ */
+const endStream = (writer, data) =>
+  new Promise((resolve, reject) => {
+    writer.end(data, (error) => {
+      if (error) return reject(error);
+      resolve();
+    });
+  });
+
+/**
+ * given a file path `fp` and a FindCursor `cursor`,
+ * write all documents in `cursor` to a file.
+ *
+ * if `totalCount` is defined, print a progress bar as well.
+ * else, just print the document number and update at each iteration.
+ *
+ * necessary to use a string when `cursor` stores a lot of documents
+ * (instead of a simple file-write):
+ * - cursor.toArray() uses TONS of memory and is slow
+ * - JSON.stringify(), used to write JSONs to file, has a maximum
+ *    size for arrays and will crash if stringifying huge arrays.
+ *
+ * to check that the output is a valid JSON, use:
+ * ```bash
+ * python3 -mjson.tool path/to/json > /dev/null && echo "ok" || echo "err"
+ * ````
+ *
+ * @param {string|PathLike} fp
+ * @param {MongoFindCursorType} cursor
+ * @param {number?} totalCount
+ * @returns {Promise}
+ */
+const writeCursorToJson = async (fp, cursor, totalCount) => {
+  const writer = fs.createWriteStream(fp);
+  const onceWriter = writeOrWait(writer);
+
+  let pb;
+  if (totalCount) {
+    pb = new ProgressBar({ desc: "writing documents to file", total: totalCount });
+  } else {
+    console.log("");
+  }
+
+  // we are writing an array => manually write the opening "[".
+  await onceWriter("[");
+  let i = 0
+  for await (const doc of cursor) {
+    i += 1;
+    // if previous items were written to file, write a "," separator.
+    if (i!=1) await onceWriter(", ");
+    if (pb) {
+      pb.update(i);
+    } else {
+      process.stdout.clearLine(0);
+      process.stdout.cursorTo(0);
+      process.stdout.write(`writing document #${i} to file`);
+    }
+    await onceWriter(JSON.stringify(doc));
+  }
+  // close the array with a "]" and end the stream.
+  await endStream(writer, "]");
+
+  if (!pb && i>0) console.log("");
+  return totalCount;
+}
+
+
+export {
+  fileRead,
+  fileOk,
+  dirOk,
+  fileWrite,
+  fileWriteJson,
+  getCwd,
+  fileArrayValidate,
+  parseImportInputFile,
+  writeCursorToJson
+}

package/cli/utils/mongoClient.js

@@ -0,0 +1,21 @@
+import { MongoClient } from "mongodb";
+
+import { MONGODB_CONNSTRING, MONGODB_DB } from "#constants";
+
+/**
+ * load a mongo client and connect it to the database. exists if there's an error
+ * @returns {{ client: import("mongodb").MongoClient, db: import("mongodb").Db }}
+ */
+function loadMongoClient() {
+  try {
+    const
+      client = new MongoClient(MONGODB_CONNSTRING),
+      db = client.db(MONGODB_DB);
+    return { client, db };
+  } catch (err) {
+    console.error(`mongoClient: could not connect to DB because of error ${err.message}`);
+    process.exit(1);
+  }
+}
+
+export default loadMongoClient;

package/cli/utils/progressbar.js

@@ -0,0 +1,86 @@
+/**
+ * a progress bar
+ * @example
+ * const total = 100;
+ * const ms = 10;
+ * const pb = new ProgressBar({ desc: "Progressbar example", total: total });
+ * const sleep = (pb, i) => {
+ *     pb.update(i);
+ *     return new Promise((res, rej) => setTimeout(res, ms));
+ * };
+ * for (let i=0; i<=total; i++) {
+ *     await sleep(pb, i);
+ * }
+ */
+class ProgressBar {
+  constructor(options = {
+    desc: "Progress",
+    total: 100
+  }) {
+    this.desc = options.desc
+    this.total = options.total
+    this.current = 0;
+    this.startTime = Date.now();
+  }
+
+  update(current) {
+    this.current = Math.min(current, this.total);
+    this.render();
+  }
+
+  render() {
+    const terminalWidth = process.stdout.columns || 80;
+
+    const percent = this.total === 0 ? 100 : (this.current / this.total) * 100;
+    const percentStr = percent.toFixed(1) + "%";
+
+    // calculate elapsed and estimated time
+    const elapsedTime = Date.now() - this.startTime;
+    const elapsedSec = Math.floor(elapsedTime / 1000);
+    const rate = this.current / (elapsedTime / 1000);
+    const remainingTime = rate > 0 ? Math.floor((this.total - this.current) / rate) : 0;
+
+    const timeStr = `${this.formatTime(elapsedSec)}/${this.formatTime(remainingTime)}`;
+    const countStr = `${this.current}/${this.total}`;
+
+    // calculate available space for bar
+    const prefix = `${this.desc} [`;
+    const suffix = `] ${percentStr} ${countStr} ${timeStr}`;
+    const availableWidth = terminalWidth - prefix.length - suffix.length;
+
+    if (availableWidth < 10) {
+      process.stdout.write(`\r${this.desc}: ${percentStr}`);
+      return;
+    }
+
+    // build progress bar
+    const filledWidth = Math.floor((this.current / this.total) * availableWidth);
+    const emptyWidth = availableWidth - filledWidth;
+    const bar = "█".repeat(filledWidth) + "░".repeat(emptyWidth);
+
+    // build and display complete line
+    const line = prefix + bar + suffix;
+    process.stdout.write("\r" + line);
+
+    // new line when complete
+    if (this.current === this.total) {
+      process.stdout.write("\n");
+    }
+  }
+
+  formatTime(seconds) {
+    const h = Math.floor(seconds / 3600);
+    const m = Math.floor((seconds % 3600) / 60);
+    const s = seconds % 60;
+
+    if (h > 0) {
+      return `${h}h${m}m`;
+    } else if (m > 0) {
+      return `${m}m${s}s`;
+    } else {
+      return `${s}s`;
+    }
+  }
+}
+
+export default ProgressBar

package/cli/xywhToInt.js

@@ -0,0 +1,99 @@
+import { Command, Option, Argument } from "commander";
+
+import loadMongoClient from "#cli/utils/mongoClient.js";
+import ProgressBar from "#cli/utils/progressbar.js";
+import { maybeToArray, xywhToInt } from "#src/utils/utils.js";
+
+/** @typedef {import("mongodb").Db} Db */
+
+/**
+ * bash command to get faulty annotations from a docker aiiinotate instance:
+ * ```
+ * docker exec docker-mongo-1 mongosh vhs_aiiinotate --eval \
+ *      "JSON.stringify(db.annotations2.find({ 'on.xywh': { \$type: 'double' } }, { '_id': 0 }).toArray(), null, 2);" \
+ *      > mogno_export.json
+ * ```
+ *
+ * bash command to import annotations exported above in a local mongo instance:
+ * ```
+ * mongoimport \
+ *   --host localhost \
+ *   --db aiiinotate \
+ *   --collection annotations2 \
+ *   --file ./mogno_export.json \
+ *   --jsonArray
+ * ```
+ */
+
+/**
+ *
+ * @param {Db} db - the mongo database
+ * @param {object} annotation - the annotation to update
+ * @param {boolean} dryRun - if `false`, don't update the annotation in database
+ */
+async function updateAnnotation(db, annotation, dryRun) {
+  let [ annotationTargetArray, converted ] = maybeToArray(annotation.on, true);
+
+  annotationTargetArray = annotationTargetArray.map((target) => {
+    target.xywh = xywhToInt(target.xywh);
+    console.log(target.xywh);
+    return target;
+  });
+  annotation.on = converted
+    ? annotationTargetArray[0]
+    : annotationTargetArray;
+
+  // console.log("post: ", annotation.on);
+  // we need to do 1 update / document, since the @id filter can select 1 document at a time.
+  if (!dryRun) {
+    await db.collection("annotations2").updateOne(
+      { "@id": annotation["@id"] },
+      { $set: annotation },
+      { upsert: false }
+    );
+  }
+  return
+}
+
+// NOTE: annotations have been exported from vhs and imported in local aiiinotate instance.
+async function action(options) {
+  const dryRun = options.dryRun || false;
+
+  const summary = { total: 0, ok: 0, error: 0 };
+
+  const { client, db } = loadMongoClient();
+  const filter = { "on.xywh": { $type: "double" } };
+  const toUpdateCount = await db.collection("annotations2").countDocuments(filter);
+  const annotationsCursor = db.collection("annotations2").find(filter);
+  summary.total = toUpdateCount;
+
+  const pb = new ProgressBar({ desc: "updating annotations", total: toUpdateCount });
+  let i = 0;
+  while (await annotationsCursor.hasNext()) {
+    i += 1;
+    pb.update(i);
+    const annotation = await annotationsCursor.next();
+    try {
+      await updateAnnotation(db, annotation, dryRun);
+      summary.ok += 1;
+    } catch (e) {
+      console.error(e);
+      summary.error += 1
+    }
+  }
+
+  console.log(`converting annotation.on.xywh to int (dry run=${dryRun}). results:`, summary);
+  await client.close()
+  process.exit(0);
+}
+
+function makeXywhToIntCommand() {
+  const dryRunOpt = new Option("-d, --dry-run", "dry run (don't update data)");
+
+  return new Command("xywh-to-int")
+    .description("convert all `db.annotations2.on.xywh` values from float to int")
+    .addOption(dryRunOpt)
+    .action(async (options, command) => { return await action(options) });
+}
+
+export default makeXywhToIntCommand;

package/config/.env.template

@@ -0,0 +1,48 @@
+#!/bin/env bash
+
+# MongoDB host
+MONGODB_HOST=127.0.0.1
+# MongoDB port
+MONGODB_PORT=27017
+# MongoDB database name
+MONGODB_DB=aiiinotate
+
+# HTTP port for the app
+AIIINOTATE_PORT=4000
+# URL for the app
+AIIINOTATE_HOST=127.0.0.1
+# HTTP scheme: HTTP or HTTPS. should be HTTP in dev and in docker
+AIIINOTATE_SCHEME=http
+
+# where to output logs. either: "stdout"|"file"|"stdout+file"|"off"
+AIIINOTATE_LOG_TARGET=stdout
+# directory to save logs to (if AIIINOTATE_LOG_TARGET enables logging to file)
+AIIINOTATE_LOG_DIR=
+# log level. "trace"|"debug"|"info"|"warn"|"error"|"fatal"
+AIIINOTATE_LOG_LEVEL=debug
+
+
+# max number of items to display per result page
+AIIINOTATE_PAGE_SIZE=5000
+# "true"|"false". strict error throwing when inserting annotations. equivalent
+# to `throwOnCanvasIndexError=true` (raise an error if an annotation's target
+# manifest can't be found) and `throwOnXywhError=true` (raise an error if you
+# can't extract a bounding box for an annotation's target). use in controlled
+# environments where you know precisely the structure of your annotations and
+# where you know that your manifests are accessible through HTTP.
+AIIINOTATE_STRICT_MODE=false
+
+# IGNORE
+AIIINOTATE_BASE_URL="$AIIINOTATE_SCHEME://$AIIINOTATE_HOST:$AIIINOTATE_PORT"
+# IGNORE
+MONGODB_CONNSTRING="mongodb://$MONGODB_HOST:$MONGODB_PORT/$MONGODB_DB"
+# IGNORE
+MONGODB_DB_TEST="${MONGODB_DB}_test"
+# IGNORE
+MONGODB_CONNSTRING_TEST="mongodb://$MONGODB_HOST:$MONGODB_PORT/$MONGODB_DB_TEST"
+
+# public URL root following `scheme://host`, used to build to root of the IDs of
+# aiiinotate data. In production, set this to your domain name. in dev, set to http://localhost, or anything you want.
+# it should be accessible through HTTP in production, but not necessarily:
+# the app doesn't access this URL in HTTP queries when running.
+AIIINOTATE_PUBLIC_URL="$AIIINOTATE_BASE_URL"

package/docker/Dockerfile

@@ -0,0 +1,39 @@
+# syntax=docker/dockerfile:1
+FROM node:23.11
+
+# aiiinotate port
+ARG PORT
+ENV PORT=${PORT}
+
+# path to the .env file
+ARG ENV_PATH
+ENV ENV_PATH="$ENV_PATH"
+
+# set up environment
+ENV DIR=/aiiinotate
+ENV TERM=linux
+ENV NPM_BIN=/aiiinotate/node_modules/.bin
+SHELL ["/bin/bash", "-c"]
+
+# root of the app in the docker container
+WORKDIR ${DIR}
+# copy the docker .env in the docker container
+COPY "$ENV_PATH" ${DIR}/.env
+
+# install iproute and curl for debug
+RUN apt-get update
+RUN apt install curl iproute2 -y
+
+# create a npm package in $DIR to run aiiinotate from
+RUN npm init -y
+# install the app as an NPM library. we do a global install as it saves us from issues with `$PATH`.
+RUN npm i aiiinotate
+
+# expose the used port
+EXPOSE ${PORT}
+
+# run the migrations and start the app.
+# ./node_modules/.bin/aiiinotate is to be able to use the `aiiinotate` cli without doing a global install of `aiiinotate`
+# NOTE migrations must be done in CMD: they need the mongo service to be running.
+CMD ${NPM_BIN}/dotenvx run -f ${DIR}/.env -- ${NPM_BIN}/aiiinotate migrate apply && \
+    ${NPM_BIN}/dotenvx run -f ${DIR}/.env -- ${NPM_BIN}/aiiinotate serve prod;

package/docker/README.md

@@ -0,0 +1,5 @@
+# Docker setup
+
+This folder contains a `docker-compose` for an aiiinotate+MongoDB integration, with volume persistence and a script to import data.
+
+View the documentation for aiiinotate Docker setup [here](https://github.com/Aikon-platform/aiiinotate/blob/main/docs/docker.md)

package/docker/docker-compose.yaml

@@ -0,0 +1,49 @@
+x-proxy-settings: &proxy-settings
+  HTTP_PROXY: ${HTTP_PROXY:-}
+  HTTPS_PROXY: ${HTTPS_PROXY:-}
+  NO_PROXY: "localhost,127.0.0.1,mongo,web,.aiiinotate_network"
+
+services:
+  mongo:
+    image: mongo:8
+    restart: always
+    networks:
+      - aiiinotate_network
+    volumes:
+      - mongodata:/data/db
+    environment:
+      <<: *proxy-settings
+    healthcheck:
+      test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  aiiinotate:
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile
+      args:
+        PORT: ${AIIINOTATE_PORT}
+        ENV_PATH: ../docker/.env
+    # NOTE: be careful to point to docker/.env, not to config/.env, otherwise you will have very, very annoying errors.
+    env_file:
+      - ../docker/.env
+    ports:
+      - "${AIIINOTATE_PORT}:${AIIINOTATE_PORT}"
+    depends_on:
+      mongo:
+        condition: service_healthy
+    networks:
+      - aiiinotate_network
+    environment:
+      <<: *proxy-settings
+    restart: always
+
+
+volumes:
+  mongodata:
+
+networks:
+  aiiinotate_network:
+    driver: bridge