@paulhectork/aiiinotate 0.12.0

package/README.md

@@ -0,0 +1,206 @@
+# aiiinotate
+
+aiiinotate is a **fast and lightweight annotation server for IIIF**.
+
+- **aiiinotate relies on** `nodejs/fastify` and `mongodb`
+- **provides a REST API** to read/write/update/delete IIIF annotations and index manifests.
+- **is distributed as an NPM package**, can be used through NPM or Docker
+- **is built for scalability and speed**: [in benchmarks](https://github.com/paulhectork/aiiinotate-benchmark) aiiinotate stores millions of annotations and its response times are always between $$\frac{1}{10}$$ and $$\frac{1}{100}$$ seconds
+
+NOTE: currently, only annotations following the IIIF presentation API 2.0 and 2.1 are supported.
+
+---
+
+## API
+
+See the [docs on the aiiinotate API](https://github.com/Aikon-platform/aiiinotate/blob/dev/docs/api.md).
+
+---
+
+## PROD USAGE
+
+### Install
+
+1. **install mongodb** (see [dev installation script for help](./scripts/setup_mongodb.sh) and checkout the [official installation guide](https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-ubuntu/#std-label-install-mdb-community-ubuntu))
+
+2. **install aiiinotate**
+```bash
+npm install aiiinotate
+```
+
+### Env definition
+
+#### Basic definition
+
+Copy [`config/.env.template`](./config/.env.template) to `.env` and edit it.
+
+#### Runtime env sourcing
+
+`aiiinotate` runs in a subproicess and won't inherit variables from a plain bash `source` call. Use either of these instead:
+
+1. **`dotenvx` (recommended)**:
+
+```bash
+npx dotenvx run -f /path/to/.env -- aiiinotate <command>
+```
+
+2. **manual export**:
+
+```bash
+set -a && source /path/to/.env && set +a
+aiiinotate <command>
+```
+
+For clarity, we omit env sourcing from the below commands.
+
+### Setup the app
+
+1. **start `mongod`**
+
+```bash
+sudo systemctl start mongod
+```
+
+2. **create and configure the database**
+
+```bash
+aiiinotate migrate apply
+```
+
+### Usage
+
+All commands are accessible through a CLI (`./src/cli`).
+
+#### Run the app
+
+```bash
+aiiinotate serve prod
+```
+
+#### Run the CLI
+
+The base command is:
+
+```bash
+aiiinotate -- <command>
+```
+
+It will give full access to the CLI interface of Aiiinotate. Run `aiiinotate --help` for more info.
+
+Use the CLI to:
+- import data
+- export data
+- apply and manage migrations
+
+For more information, see [the CLI docs](https://github.com/Aikon-platform/aiiinotate/blob/main/docs/cli.md).
+
+---
+
+## DOCKER USAGE
+
+See the docs [here](https://github.com/Aikon-platform/aiiinotate/blob/dev/docs/docker.md)
+
+For a Mirador integration, see [the reference implementation](github.com/paulhectork/mirador-aiiinotate/tree/main) (aiiinotate + MongoDB + Mirador 4 + MAE bundled in a single `docker-compose`)
+
+---
+
+## DEV USAGE
+
+### Install
+
+```bash
+# clone the repo
+git clone git@github.com:Aikon-platform/aiiinotate.git
+
+# move inside it
+cd aiiinotate
+
+# install mongodb
+bash ./scripts/setup_mongodb.sh
+
+# install node
+bash ./scripts/setup_node.sh
+
+# install dependencies
+npm i
+```
+
+### Setup
+
+After installing, some setup must be done
+
+1. **setup your `.env`** file after [`config/.env.template`](./config/.env.template) and place it at `./config/.env`.
+
+2. **start `mongod`**
+
+```bash
+sudo systemctl start mongod
+```
+
+3. **configure the database**
+
+```bash
+npm run migrate apply
+```
+
+### Usage
+
+Remember to have your `mongodb` service running: `sudo systemctl start mongod` !
+
+#### Start the app
+
+```bash
+# reload enabled
+npm run dev
+```
+
+#### Test the app
+
+Note that the tests will probably fail if you set the env variable `AIIINOTATE_STRICT_MODE` to `true`.
+
+```bash
+npm run test
+```
+
+#### Run the CLI
+
+See [the CLI docs](https://github.com/Aikon-platform/aiiinotate/blob/dev/docs/cli.md) for more info: 
+
+```bash
+npm run cli
+```
+
+Use the CLI to:
+- import data
+- export data
+- apply and manage database migrations
+
+--- 
+
+## Test coverage
+
+aiiinotate is well tested: **~90% test coverage** on all files ! 
+
+```
+ℹ ----------------------------------------------------------------------------------------
+ℹ file       | line % | branch % | funcs % | uncovered lines
+ℹ ----------------------------------------------------------------------------------------
+ℹ all files  |  90.02 |    79.43 |   78.73 |
+ℹ ----------------------------------------------------------------------------------------
+```
+
+---
+
+## Scalability
+
+In [benchmarks](https://github.com/paulhectork/aiiinotate-benchmark), aiiinotate response times are between 1/100th and 1/10th of a second with up to 10,000,000 annotations. 
+
+![benchmark results](./docs/includes/report_benchmark_aiiinotate_2026-05-28-02:50:48_7steps.png)
+
+See [scalability.md](./docs/scalability.md) for more information.
+
+---
+
+## License
+
+GNU GPL 3.0.

package/cli/export.js

@@ -0,0 +1,118 @@
+import path from "node:path";
+
+import { Command, Option, Argument } from "commander";
+
+import loadMongoClient from "#cli/utils/mongoClient.js";
+import { dirOk, getCwd, writeCursorToJson } from "#cli/utils/io.js";
+
+/** @typedef {import("#types").MongoDbType} MongoDbType */
+/** @typedef {import("#types").MongoFindCursorType} MongoFindCursorType */
+/** @typedef {import("fs").PathLike} PathLike */
+
+
+const exportableCollections = [ "all", "annotations2", "annotations3", "manifests2", "manifests3" ];
+const actualCollections = exportableCollections.filter(c => c!=="all");
+
+/** @type {(outDir: string) => string} */
+const getOutDir = (outDir) => {
+  let success;
+  if (outDir) {
+    [ outDir, success ] = dirOk(outDir);
+    if (!success) {
+      console.error(`Error opening directory "${outDir}". Are you sure it exists ?`);
+      process.exit(1);
+    }
+  } else {
+    outDir = getCwd();
+  }
+  return outDir;
+}
+
+/** @type {() => (collectionName: string) => string|PathLike} */
+const outFileNameFunc = () => {
+  // we currify this function so that, if a user exports all collections,
+  // output file nmaes share the same timestamp.
+  const timestamp = (new Date).toISOString().slice(0,-2);
+  return (outDir, collectionName) =>
+    path.join(outDir, `${timestamp}-aiiinotate-${collectionName}.json`);
+}
+const outFileName = outFileNameFunc();
+
+/**
+ * if `count`, returns a function to count all documents in a collection.
+ * else, returns a function to find all documents in a collection.
+ * @type {(db:MongoClient) => (count: boolean) => (collectionName: string) => MongoFindCursorType|Promise<number>}
+ */
+const exportCmdFunc = (db) =>
+  (count) =>
+    (collectionName) => {
+      const collection = db.collection(collectionName);
+      return count
+        ? collection.countDocuments()
+        : collection.find().project({ _id: 0 })
+
+    }
+
+/**
+ * export a single collection to a file
+ * @param {MongoClient} db
+ * @param {string} outDir
+ * @returns {(collctionName: string) => Promise<void>}
+ */
+const exportCollectionFunc = (db, outDir) => {
+  const exportCmd = exportCmdFunc(db)(false);
+  const countFunc = exportCmdFunc(db)(true);
+
+  return async (collectionName) => {
+    const out = outFileName(outDir, collectionName);
+    const collectionCursor = exportCmd(collectionName);
+    const totalCount = await countFunc(collectionName);
+    try {
+      await writeCursorToJson(out, collectionCursor, totalCount);
+      console.log(`Wrote export of collection "${collectionName}" (${totalCount} documents) to "${out}"\n`)
+    } catch(e) {
+      console.error(`Error writing export of collection "${collectionName}" to "${out}": ${e}`);
+      process.exit(1);
+    }
+  }
+}
+
+/**
+ * @param {string} collection
+ * @param {object} options
+ */
+async function action(collection, options) {
+  const { client, db } = loadMongoClient();
+  const outDir = getOutDir(options.output);
+  const collectionNameArray =collection === "all" ? actualCollections : [ collection ];
+  const exportCollection = exportCollectionFunc(db, outDir);
+
+  console.log(`\nExporting collection(s): ${collectionNameArray}\n`)
+
+  // fetch and write to file
+  for (const collectionName of collectionNameArray) {
+    await exportCollection(collectionName);
+  }
+
+  console.log(`\nFinished exporting collection(s): ${collectionNameArray} to "${outDir}".`)
+  await client.close();
+  process.exit(0);
+}
+
+/** define the cli */
+function makeExportCommand() {
+
+  const collectionArg =
+    new Argument("collection", `collection to export import: one of ${exportableCollections}`)
+      .choices(exportableCollections);
+
+  const outOpt = new Option("-o, --output <directory>", "Output directory (defaults to current working directory");
+
+  return new Command("export")
+    .description("export aiiinotate data")
+    .addArgument(collectionArg)
+    .addOption(outOpt)
+    .action(async (collection, options, command) => await action(collection, options))
+}
+
+export default makeExportCommand;

package/cli/import.js

@@ -0,0 +1,111 @@
+import { Command, Option, Argument } from "commander";
+
+import { fileRead, parseImportInputFile } from "#cli/utils/io.js";
+import FastifyClient from "#cli/utils/fastifyClient.js";
+import ProgressBar from "#cli/utils/progressbar.js";
+import logger from "#utils/logger.js";
+import { inspectObj } from "#utils/utils.js";
+
+/** @typedef {import("#types").FastifyInstanceType} FastifyInstanceType */
+
+////////////////////////////////////////
+
+const notImplementedExit = (method) => {
+  logger.error(`\n\nERROR: import is not implemented '${method}'`);
+  process.exit(1);
+}
+
+const parseNumber = (x) => Number(x);
+
+////////////////////////////////////////
+
+/**
+ * @param {FastifyClient} fastifyClient
+ * @param {string[]} fileArr - array of full paths to annotationLists to insert.
+ */
+async function importData(importFunc, datatype, fileArr) {
+
+  let totalImports = 0
+  const
+    pb = new ProgressBar({ desc: `importing ${datatype}s`, total: fileArr.length }),
+    importErrors = [];
+
+  for (let [ i, file ] of fileArr.entries()) {
+    i += 1
+    const
+      data = JSON.parse(fileRead(file)),
+      [ statusCode, resultPromise ] = await importFunc(data),
+      result = await resultPromise;
+    if (statusCode <= 299) {
+      totalImports += result.insertedCount;
+    } else {
+      console.log(result);
+      importErrors.push(file);
+    }
+    pb.update(i)
+  }
+  return [ totalImports, importErrors ]
+}
+
+////////////////////////////////////////
+
+/**
+ * run the cli
+ * @param {import('commander').Command} command
+ * @param {"manifest"|"annotation"} datatype
+ * @param {object} options
+ */
+async function action(command, datatype, options) {
+  /** @type {2 | 3} */
+  const iiifVersion = options.iiifVersion;
+  /** @type {string[]} */
+  const inputFile = options.file;
+
+  if (iiifVersion===3) {
+    notImplementedExit("CLI imports for IIIF presentation V3 is not implemented");
+  }
+
+  const fastifyClient = new FastifyClient();
+  await fastifyClient.build();
+  const importFunc = fastifyClient.importData(iiifVersion, datatype);
+
+  // run
+  const fileArr = await parseImportInputFile(inputFile);
+  const [ totalImports, importErrors ] = await importData(importFunc, datatype, fileArr);
+
+  if (importErrors.length) {
+    logger.info(`There were problems importing ${datatype}s from the following ${importErrors.length} files: ${inspectObj(importErrors, -1)}`)
+  }
+  logger.info(`Imported ${totalImports} ${datatype}s into aiiinotate !`);
+  await fastifyClient.stop(); // TODO app keeps idling after that ....
+  return
+}
+
+/////////////////////////////////////////
+
+/** define the cli */
+function makeImportCommand() {
+
+  const datatypeArg =
+    new Argument("datatype", "type of data to import: manifests or annotations")
+      .choices([ "manifest", "annotation" ]);
+
+  const versionOpt =
+    new Option("-i, --iiif-version <version>", "IIIF version")
+      .choices([ "2", "3" ])
+      .argParser(parseNumber)
+      .makeOptionMandatory();
+
+  const filesOpt =
+    new Option("-f, --file <file>", "file containing paths to AnnotationLists, AnnotationPages or Manifests to process (1 line per path)")
+      .makeOptionMandatory();
+
+  return new Command("import")
+    .description("import data into aiiinotate")
+    .addArgument(datatypeArg)
+    .addOption(filesOpt)
+    .addOption(versionOpt)
+    .action((datatype, options, command) => action(command, datatype, options))
+}
+
+export default makeImportCommand;

package/cli/index.js

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+
+/**
+ * command line interface. run through the package.json.
+ * usage: npm run cli import -- [args] [opts]
+ *
+ * NOTE: node recommends only creating 1 mongoclient per app
+ * when possible for performance reasons => we create a global mongoClient
+ * here, and then pass it down to all the other scripts.
+ */
+
+import { Command, Option } from "commander";
+
+// import dotenvx from "dotenvx";
+
+import makeImportCommand from "#cli/import.js";
+import makeExportCommand from "#cli/export.js";
+import makeMigrateCommand from "#cli/migrate.js";
+import makeServeCommand from "#cli/serve.js";
+import makeXywhToIntCommand from "#cli/xywhToInt.js";
+
+function makeCli() {
+
+  const desc =
+    "Command line interface for aiiinotate.\n\n"
+    + `All commands are accessible through this CLI: starting the app,
+    managing and running migrations, importing and exporting data.
+    Run individual commands to see command-specific help.
+    `.replace(/^\s+/gm, "");
+
+  // NOTE: before running, it is necessary to load env variables.
+  const cli = new Command();
+  cli
+    .name("aiiinotate")
+    .description(desc)
+    .addCommand(makeServeCommand())
+    .addCommand(makeImportCommand())
+    .addCommand(makeExportCommand())
+    .addCommand(makeMigrateCommand())
+    .addCommand(makeXywhToIntCommand());
+
+  cli.parse(process.argv);
+}
+
+makeCli();

package/cli/migrate.js

@@ -0,0 +1,131 @@
+/**
+ * run and apply migrations.
+ *
+ * the commands here are wrappers for migrate-mongo.
+ * the big particularity is that we handle 2 databases in parrallel:
+ * a dev/prod database and a test database (that will be populated
+ * by running tests, emptied after running the tests)
+ * in turn, we need to apply migrations in parrallel to both databases.
+ *
+ */
+import path from "node:path";
+import fs from "node:fs";
+import { fileURLToPath } from "node:url";
+import { execSync } from "node:child_process"
+
+import { Command, Option, Argument } from "commander";
+
+/** @typedef {"make"|"apply"|"revert"|"revert-all"} MigrateOpType */
+const allowedMigrateOp = [ "make", "apply", "revert", "revert-all" ];
+
+const
+  // path to current dirctory
+  dirCli = path.dirname(fileURLToPath(import.meta.url)),
+  dirRoot = path.resolve(dirCli, ".."),
+  dirMigrations = path.resolve(dirRoot, "migrations"),
+  dirMigrationsScripts = path.resolve(dirMigrations, "migrationScripts"),
+  migrationsConfigMain = path.resolve(dirMigrations, "migrate-mongo-config-main.js"),
+  migrationsConfigTest = path.resolve(dirMigrations, "migrate-mongo-config-test.js"),
+  migrationConfigs = [ migrationsConfigMain, migrationsConfigTest ];
+
+/** return a date in YYYYMMDDhhmmss format */
+function formatDate(date) {
+  function pad2(n) {  // always returns a string
+    return (n < 10 ? "0" : "") + n;
+  }
+
+  return date.getFullYear() +
+    pad2(date.getMonth() + 1) +
+    pad2(date.getDate()) +
+    pad2(date.getHours()) +
+    pad2(date.getMinutes()) +
+    pad2(date.getSeconds());
+}
+
+/**
+ * create a single migration file
+ * @param {string} migrationName
+ */
+function migrateMake(migrationName) {
+  if (migrationName == null) {
+    throw new Error(`migration name must be a string. got ${migrationName}`);
+  }
+  fs.copyFileSync(
+    path.resolve(dirMigrations, "migrationTemplate.js"),
+    path.resolve(dirMigrationsScripts, `${formatDate(new Date())}-${migrationName}.js`)
+  );
+}
+
+/** apply all pending migrations */
+function migrateApply() {
+  // NOTE: if we don't use a specific version of migrate-mongo, we get docker errors
+  migrationConfigs.map((mc) => execSync(`npx -- migrate-mongo@^12.1.3 up -f ${mc}`));
+}
+
+/** revert the last migration */
+function migrateRevert() {
+  migrationConfigs.map((mc) => execSync(`npx migrate-mongo@^12.1.3 down -f ${mc}`));
+}
+
+/** revert all migrations */
+function migrateRevertAll() {
+  // there are as many migrations as there are files in `dirMigrationsScripts`
+  // => revert one migration per migration file
+  // do this for each migration file (prod and test database).
+  migrationConfigs.map((mc) =>
+    fs.readdirSync(dirMigrationsScripts).map((_) =>
+      execSync(`npx migrate-mongo@^12.1.3 down -f ${mc}`)
+    )
+  )
+}
+
+/**
+ * run the cli
+ * @param {import('commander').Command} command
+ * @param {MigrateOpType} mongoClient
+ * @param {object} options
+ */
+async function action(command, migrationOp, options) {
+  const { migrationName } = options;
+
+  switch (migrationOp) {
+    case ("make"):
+      migrateMake(migrationName);
+      break;
+    case ("apply"):
+      migrateApply();
+      break;
+    case ("revert"):
+      migrateRevert();
+      break;
+    case ("revert-all"):
+      migrateRevertAll();
+      break;
+  }
+}
+
+function makeMigrateCommand() {
+  const migrationOpArg =
+    new Argument("<migration-op>", "name of migration operation")
+      .choices(allowedMigrateOp);
+
+  const migrationNameOpt =
+    new Option("-n, --migration-name <name>", "name of migration (for 'make' argument)");
+
+  const command = new Command("migrate");
+  return command
+    .description("run database migrations")
+    .addArgument(migrationOpArg)
+    .addOption(migrationNameOpt)
+    .action((migrationOp, options, command) => {
+      if (
+        migrationOp == "make"
+        && (options.migrationName === undefined || options.migrationName === "")
+      ) {
+        command.error("migration operation \"apply\" requires option \"-n, --migration-name <name>\"", { exitCode: 1 });
+      }
+      action(command, migrationOp, options)
+    })
+}
+
+export default makeMigrateCommand;

package/cli/serve.js

@@ -0,0 +1,29 @@
+import serve from "#src/server.js";
+
+import { Command, Option, Argument } from "commander";
+
+/** @typedef {import("#types").ServeModeType} ServeModeType */
+
+const serveModeValues = [ "dev", "prod" ];
+
+/**
+ * @param {import('commander').Command} command
+ * @param {ServeModeType} serveMode
+ */
+async function action(command, serveMode) {
+  await serve(serveMode);
+}
+
+function makeServeCommand() {
+
+  const serveModeArg =
+    new Argument("<run-mode>", "mode with which to run the app")
+      .choices(serveModeValues);
+
+  return new Command("serve")
+    .description("run Aiiinotate. <run-mode>")
+    .addArgument(serveModeArg)
+    .action((serveMode, command) => action(command, serveMode));
+}
+
+export default makeServeCommand;